intel_powerclamp.c 21 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815
  1. /*
  2. * intel_powerclamp.c - package c-state idle injection
  3. *
  4. * Copyright (c) 2012, Intel Corporation.
  5. *
  6. * Authors:
  7. * Arjan van de Ven <arjan@linux.intel.com>
  8. * Jacob Pan <jacob.jun.pan@linux.intel.com>
  9. *
  10. * This program is free software; you can redistribute it and/or modify it
  11. * under the terms and conditions of the GNU General Public License,
  12. * version 2, as published by the Free Software Foundation.
  13. *
  14. * This program is distributed in the hope it will be useful, but WITHOUT
  15. * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  16. * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
  17. * more details.
  18. *
  19. * You should have received a copy of the GNU General Public License along with
  20. * this program; if not, write to the Free Software Foundation, Inc.,
  21. * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
  22. *
  23. *
  24. * TODO:
  25. * 1. better handle wakeup from external interrupts, currently a fixed
  26. * compensation is added to clamping duration when excessive amount
  27. * of wakeups are observed during idle time. the reason is that in
  28. * case of external interrupts without need for ack, clamping down
  29. * cpu in non-irq context does not reduce irq. for majority of the
  30. * cases, clamping down cpu does help reduce irq as well, we should
  31. * be able to differentiate the two cases and give a quantitative
  32. * solution for the irqs that we can control. perhaps based on
  33. * get_cpu_iowait_time_us()
  34. *
  35. * 2. synchronization with other hw blocks
  36. *
  37. *
  38. */
  39. #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  40. #include <linux/module.h>
  41. #include <linux/kernel.h>
  42. #include <linux/delay.h>
  43. #include <linux/kthread.h>
  44. #include <linux/cpu.h>
  45. #include <linux/thermal.h>
  46. #include <linux/slab.h>
  47. #include <linux/tick.h>
  48. #include <linux/debugfs.h>
  49. #include <linux/seq_file.h>
  50. #include <linux/sched/rt.h>
  51. #include <uapi/linux/sched/types.h>
  52. #include <asm/nmi.h>
  53. #include <asm/msr.h>
  54. #include <asm/mwait.h>
  55. #include <asm/cpu_device_id.h>
  56. #include <asm/hardirq.h>
  57. #define MAX_TARGET_RATIO (50U)
  58. /* For each undisturbed clamping period (no extra wake ups during idle time),
  59. * we increment the confidence counter for the given target ratio.
  60. * CONFIDENCE_OK defines the level where runtime calibration results are
  61. * valid.
  62. */
  63. #define CONFIDENCE_OK (3)
  64. /* Default idle injection duration, driver adjust sleep time to meet target
  65. * idle ratio. Similar to frequency modulation.
  66. */
  67. #define DEFAULT_DURATION_JIFFIES (6)
  68. static unsigned int target_mwait;
  69. static struct dentry *debug_dir;
  70. /* user selected target */
  71. static unsigned int set_target_ratio;
  72. static unsigned int current_ratio;
  73. static bool should_skip;
  74. static bool reduce_irq;
  75. static atomic_t idle_wakeup_counter;
  76. static unsigned int control_cpu; /* The cpu assigned to collect stat and update
  77. * control parameters. default to BSP but BSP
  78. * can be offlined.
  79. */
  80. static bool clamping;
  81. static const struct sched_param sparam = {
  82. .sched_priority = MAX_USER_RT_PRIO / 2,
  83. };
  84. struct powerclamp_worker_data {
  85. struct kthread_worker *worker;
  86. struct kthread_work balancing_work;
  87. struct kthread_delayed_work idle_injection_work;
  88. unsigned int cpu;
  89. unsigned int count;
  90. unsigned int guard;
  91. unsigned int window_size_now;
  92. unsigned int target_ratio;
  93. unsigned int duration_jiffies;
  94. bool clamping;
  95. };
  96. static struct powerclamp_worker_data __percpu *worker_data;
  97. static struct thermal_cooling_device *cooling_dev;
  98. static unsigned long *cpu_clamping_mask; /* bit map for tracking per cpu
  99. * clamping kthread worker
  100. */
  101. static unsigned int duration;
  102. static unsigned int pkg_cstate_ratio_cur;
  103. static unsigned int window_size;
  104. static int duration_set(const char *arg, const struct kernel_param *kp)
  105. {
  106. int ret = 0;
  107. unsigned long new_duration;
  108. ret = kstrtoul(arg, 10, &new_duration);
  109. if (ret)
  110. goto exit;
  111. if (new_duration > 25 || new_duration < 6) {
  112. pr_err("Out of recommended range %lu, between 6-25ms\n",
  113. new_duration);
  114. ret = -EINVAL;
  115. }
  116. duration = clamp(new_duration, 6ul, 25ul);
  117. smp_mb();
  118. exit:
  119. return ret;
  120. }
  121. static const struct kernel_param_ops duration_ops = {
  122. .set = duration_set,
  123. .get = param_get_int,
  124. };
  125. module_param_cb(duration, &duration_ops, &duration, 0644);
  126. MODULE_PARM_DESC(duration, "forced idle time for each attempt in msec.");
  127. struct powerclamp_calibration_data {
  128. unsigned long confidence; /* used for calibration, basically a counter
  129. * gets incremented each time a clamping
  130. * period is completed without extra wakeups
  131. * once that counter is reached given level,
  132. * compensation is deemed usable.
  133. */
  134. unsigned long steady_comp; /* steady state compensation used when
  135. * no extra wakeups occurred.
  136. */
  137. unsigned long dynamic_comp; /* compensate excessive wakeup from idle
  138. * mostly from external interrupts.
  139. */
  140. };
  141. static struct powerclamp_calibration_data cal_data[MAX_TARGET_RATIO];
  142. static int window_size_set(const char *arg, const struct kernel_param *kp)
  143. {
  144. int ret = 0;
  145. unsigned long new_window_size;
  146. ret = kstrtoul(arg, 10, &new_window_size);
  147. if (ret)
  148. goto exit_win;
  149. if (new_window_size > 10 || new_window_size < 2) {
  150. pr_err("Out of recommended window size %lu, between 2-10\n",
  151. new_window_size);
  152. ret = -EINVAL;
  153. }
  154. window_size = clamp(new_window_size, 2ul, 10ul);
  155. smp_mb();
  156. exit_win:
  157. return ret;
  158. }
  159. static const struct kernel_param_ops window_size_ops = {
  160. .set = window_size_set,
  161. .get = param_get_int,
  162. };
  163. module_param_cb(window_size, &window_size_ops, &window_size, 0644);
  164. MODULE_PARM_DESC(window_size, "sliding window in number of clamping cycles\n"
  165. "\tpowerclamp controls idle ratio within this window. larger\n"
  166. "\twindow size results in slower response time but more smooth\n"
  167. "\tclamping results. default to 2.");
  168. static void find_target_mwait(void)
  169. {
  170. unsigned int eax, ebx, ecx, edx;
  171. unsigned int highest_cstate = 0;
  172. unsigned int highest_subcstate = 0;
  173. int i;
  174. if (boot_cpu_data.cpuid_level < CPUID_MWAIT_LEAF)
  175. return;
  176. cpuid(CPUID_MWAIT_LEAF, &eax, &ebx, &ecx, &edx);
  177. if (!(ecx & CPUID5_ECX_EXTENSIONS_SUPPORTED) ||
  178. !(ecx & CPUID5_ECX_INTERRUPT_BREAK))
  179. return;
  180. edx >>= MWAIT_SUBSTATE_SIZE;
  181. for (i = 0; i < 7 && edx; i++, edx >>= MWAIT_SUBSTATE_SIZE) {
  182. if (edx & MWAIT_SUBSTATE_MASK) {
  183. highest_cstate = i;
  184. highest_subcstate = edx & MWAIT_SUBSTATE_MASK;
  185. }
  186. }
  187. target_mwait = (highest_cstate << MWAIT_SUBSTATE_SIZE) |
  188. (highest_subcstate - 1);
  189. }
  190. struct pkg_cstate_info {
  191. bool skip;
  192. int msr_index;
  193. int cstate_id;
  194. };
  195. #define PKG_CSTATE_INIT(id) { \
  196. .msr_index = MSR_PKG_C##id##_RESIDENCY, \
  197. .cstate_id = id \
  198. }
  199. static struct pkg_cstate_info pkg_cstates[] = {
  200. PKG_CSTATE_INIT(2),
  201. PKG_CSTATE_INIT(3),
  202. PKG_CSTATE_INIT(6),
  203. PKG_CSTATE_INIT(7),
  204. PKG_CSTATE_INIT(8),
  205. PKG_CSTATE_INIT(9),
  206. PKG_CSTATE_INIT(10),
  207. {NULL},
  208. };
  209. static bool has_pkg_state_counter(void)
  210. {
  211. u64 val;
  212. struct pkg_cstate_info *info = pkg_cstates;
  213. /* check if any one of the counter msrs exists */
  214. while (info->msr_index) {
  215. if (!rdmsrl_safe(info->msr_index, &val))
  216. return true;
  217. info++;
  218. }
  219. return false;
  220. }
  221. static u64 pkg_state_counter(void)
  222. {
  223. u64 val;
  224. u64 count = 0;
  225. struct pkg_cstate_info *info = pkg_cstates;
  226. while (info->msr_index) {
  227. if (!info->skip) {
  228. if (!rdmsrl_safe(info->msr_index, &val))
  229. count += val;
  230. else
  231. info->skip = true;
  232. }
  233. info++;
  234. }
  235. return count;
  236. }
  237. static unsigned int get_compensation(int ratio)
  238. {
  239. unsigned int comp = 0;
  240. /* we only use compensation if all adjacent ones are good */
  241. if (ratio == 1 &&
  242. cal_data[ratio].confidence >= CONFIDENCE_OK &&
  243. cal_data[ratio + 1].confidence >= CONFIDENCE_OK &&
  244. cal_data[ratio + 2].confidence >= CONFIDENCE_OK) {
  245. comp = (cal_data[ratio].steady_comp +
  246. cal_data[ratio + 1].steady_comp +
  247. cal_data[ratio + 2].steady_comp) / 3;
  248. } else if (ratio == MAX_TARGET_RATIO - 1 &&
  249. cal_data[ratio].confidence >= CONFIDENCE_OK &&
  250. cal_data[ratio - 1].confidence >= CONFIDENCE_OK &&
  251. cal_data[ratio - 2].confidence >= CONFIDENCE_OK) {
  252. comp = (cal_data[ratio].steady_comp +
  253. cal_data[ratio - 1].steady_comp +
  254. cal_data[ratio - 2].steady_comp) / 3;
  255. } else if (cal_data[ratio].confidence >= CONFIDENCE_OK &&
  256. cal_data[ratio - 1].confidence >= CONFIDENCE_OK &&
  257. cal_data[ratio + 1].confidence >= CONFIDENCE_OK) {
  258. comp = (cal_data[ratio].steady_comp +
  259. cal_data[ratio - 1].steady_comp +
  260. cal_data[ratio + 1].steady_comp) / 3;
  261. }
  262. /* REVISIT: simple penalty of double idle injection */
  263. if (reduce_irq)
  264. comp = ratio;
  265. /* do not exceed limit */
  266. if (comp + ratio >= MAX_TARGET_RATIO)
  267. comp = MAX_TARGET_RATIO - ratio - 1;
  268. return comp;
  269. }
  270. static void adjust_compensation(int target_ratio, unsigned int win)
  271. {
  272. int delta;
  273. struct powerclamp_calibration_data *d = &cal_data[target_ratio];
  274. /*
  275. * adjust compensations if confidence level has not been reached or
  276. * there are too many wakeups during the last idle injection period, we
  277. * cannot trust the data for compensation.
  278. */
  279. if (d->confidence >= CONFIDENCE_OK ||
  280. atomic_read(&idle_wakeup_counter) >
  281. win * num_online_cpus())
  282. return;
  283. delta = set_target_ratio - current_ratio;
  284. /* filter out bad data */
  285. if (delta >= 0 && delta <= (1+target_ratio/10)) {
  286. if (d->steady_comp)
  287. d->steady_comp =
  288. roundup(delta+d->steady_comp, 2)/2;
  289. else
  290. d->steady_comp = delta;
  291. d->confidence++;
  292. }
  293. }
  294. static bool powerclamp_adjust_controls(unsigned int target_ratio,
  295. unsigned int guard, unsigned int win)
  296. {
  297. static u64 msr_last, tsc_last;
  298. u64 msr_now, tsc_now;
  299. u64 val64;
  300. /* check result for the last window */
  301. msr_now = pkg_state_counter();
  302. tsc_now = rdtsc();
  303. /* calculate pkg cstate vs tsc ratio */
  304. if (!msr_last || !tsc_last)
  305. current_ratio = 1;
  306. else if (tsc_now-tsc_last) {
  307. val64 = 100*(msr_now-msr_last);
  308. do_div(val64, (tsc_now-tsc_last));
  309. current_ratio = val64;
  310. }
  311. /* update record */
  312. msr_last = msr_now;
  313. tsc_last = tsc_now;
  314. adjust_compensation(target_ratio, win);
  315. /*
  316. * too many external interrupts, set flag such
  317. * that we can take measure later.
  318. */
  319. reduce_irq = atomic_read(&idle_wakeup_counter) >=
  320. 2 * win * num_online_cpus();
  321. atomic_set(&idle_wakeup_counter, 0);
  322. /* if we are above target+guard, skip */
  323. return set_target_ratio + guard <= current_ratio;
  324. }
  325. static void clamp_balancing_func(struct kthread_work *work)
  326. {
  327. struct powerclamp_worker_data *w_data;
  328. int sleeptime;
  329. unsigned long target_jiffies;
  330. unsigned int compensated_ratio;
  331. int interval; /* jiffies to sleep for each attempt */
  332. w_data = container_of(work, struct powerclamp_worker_data,
  333. balancing_work);
  334. /*
  335. * make sure user selected ratio does not take effect until
  336. * the next round. adjust target_ratio if user has changed
  337. * target such that we can converge quickly.
  338. */
  339. w_data->target_ratio = READ_ONCE(set_target_ratio);
  340. w_data->guard = 1 + w_data->target_ratio / 20;
  341. w_data->window_size_now = window_size;
  342. w_data->duration_jiffies = msecs_to_jiffies(duration);
  343. w_data->count++;
  344. /*
  345. * systems may have different ability to enter package level
  346. * c-states, thus we need to compensate the injected idle ratio
  347. * to achieve the actual target reported by the HW.
  348. */
  349. compensated_ratio = w_data->target_ratio +
  350. get_compensation(w_data->target_ratio);
  351. if (compensated_ratio <= 0)
  352. compensated_ratio = 1;
  353. interval = w_data->duration_jiffies * 100 / compensated_ratio;
  354. /* align idle time */
  355. target_jiffies = roundup(jiffies, interval);
  356. sleeptime = target_jiffies - jiffies;
  357. if (sleeptime <= 0)
  358. sleeptime = 1;
  359. if (clamping && w_data->clamping && cpu_online(w_data->cpu))
  360. kthread_queue_delayed_work(w_data->worker,
  361. &w_data->idle_injection_work,
  362. sleeptime);
  363. }
  364. static void clamp_idle_injection_func(struct kthread_work *work)
  365. {
  366. struct powerclamp_worker_data *w_data;
  367. w_data = container_of(work, struct powerclamp_worker_data,
  368. idle_injection_work.work);
  369. /*
  370. * only elected controlling cpu can collect stats and update
  371. * control parameters.
  372. */
  373. if (w_data->cpu == control_cpu &&
  374. !(w_data->count % w_data->window_size_now)) {
  375. should_skip =
  376. powerclamp_adjust_controls(w_data->target_ratio,
  377. w_data->guard,
  378. w_data->window_size_now);
  379. smp_mb();
  380. }
  381. if (should_skip)
  382. goto balance;
  383. play_idle(jiffies_to_msecs(w_data->duration_jiffies));
  384. balance:
  385. if (clamping && w_data->clamping && cpu_online(w_data->cpu))
  386. kthread_queue_work(w_data->worker, &w_data->balancing_work);
  387. }
  388. /*
  389. * 1 HZ polling while clamping is active, useful for userspace
  390. * to monitor actual idle ratio.
  391. */
  392. static void poll_pkg_cstate(struct work_struct *dummy);
  393. static DECLARE_DELAYED_WORK(poll_pkg_cstate_work, poll_pkg_cstate);
  394. static void poll_pkg_cstate(struct work_struct *dummy)
  395. {
  396. static u64 msr_last;
  397. static u64 tsc_last;
  398. u64 msr_now;
  399. u64 tsc_now;
  400. u64 val64;
  401. msr_now = pkg_state_counter();
  402. tsc_now = rdtsc();
  403. /* calculate pkg cstate vs tsc ratio */
  404. if (!msr_last || !tsc_last)
  405. pkg_cstate_ratio_cur = 1;
  406. else {
  407. if (tsc_now - tsc_last) {
  408. val64 = 100 * (msr_now - msr_last);
  409. do_div(val64, (tsc_now - tsc_last));
  410. pkg_cstate_ratio_cur = val64;
  411. }
  412. }
  413. /* update record */
  414. msr_last = msr_now;
  415. tsc_last = tsc_now;
  416. if (true == clamping)
  417. schedule_delayed_work(&poll_pkg_cstate_work, HZ);
  418. }
  419. static void start_power_clamp_worker(unsigned long cpu)
  420. {
  421. struct powerclamp_worker_data *w_data = per_cpu_ptr(worker_data, cpu);
  422. struct kthread_worker *worker;
  423. worker = kthread_create_worker_on_cpu(cpu, 0, "kidle_inj/%ld", cpu);
  424. if (IS_ERR(worker))
  425. return;
  426. w_data->worker = worker;
  427. w_data->count = 0;
  428. w_data->cpu = cpu;
  429. w_data->clamping = true;
  430. set_bit(cpu, cpu_clamping_mask);
  431. sched_setscheduler(worker->task, SCHED_FIFO, &sparam);
  432. kthread_init_work(&w_data->balancing_work, clamp_balancing_func);
  433. kthread_init_delayed_work(&w_data->idle_injection_work,
  434. clamp_idle_injection_func);
  435. kthread_queue_work(w_data->worker, &w_data->balancing_work);
  436. }
  437. static void stop_power_clamp_worker(unsigned long cpu)
  438. {
  439. struct powerclamp_worker_data *w_data = per_cpu_ptr(worker_data, cpu);
  440. if (!w_data->worker)
  441. return;
  442. w_data->clamping = false;
  443. /*
  444. * Make sure that all works that get queued after this point see
  445. * the clamping disabled. The counter part is not needed because
  446. * there is an implicit memory barrier when the queued work
  447. * is proceed.
  448. */
  449. smp_wmb();
  450. kthread_cancel_work_sync(&w_data->balancing_work);
  451. kthread_cancel_delayed_work_sync(&w_data->idle_injection_work);
  452. /*
  453. * The balancing work still might be queued here because
  454. * the handling of the "clapming" variable, cancel, and queue
  455. * operations are not synchronized via a lock. But it is not
  456. * a big deal. The balancing work is fast and destroy kthread
  457. * will wait for it.
  458. */
  459. clear_bit(w_data->cpu, cpu_clamping_mask);
  460. kthread_destroy_worker(w_data->worker);
  461. w_data->worker = NULL;
  462. }
  463. static int start_power_clamp(void)
  464. {
  465. unsigned long cpu;
  466. set_target_ratio = clamp(set_target_ratio, 0U, MAX_TARGET_RATIO - 1);
  467. /* prevent cpu hotplug */
  468. get_online_cpus();
  469. /* prefer BSP */
  470. control_cpu = 0;
  471. if (!cpu_online(control_cpu))
  472. control_cpu = smp_processor_id();
  473. clamping = true;
  474. schedule_delayed_work(&poll_pkg_cstate_work, 0);
  475. /* start one kthread worker per online cpu */
  476. for_each_online_cpu(cpu) {
  477. start_power_clamp_worker(cpu);
  478. }
  479. put_online_cpus();
  480. return 0;
  481. }
  482. static void end_power_clamp(void)
  483. {
  484. int i;
  485. /*
  486. * Block requeuing in all the kthread workers. They will flush and
  487. * stop faster.
  488. */
  489. clamping = false;
  490. if (bitmap_weight(cpu_clamping_mask, num_possible_cpus())) {
  491. for_each_set_bit(i, cpu_clamping_mask, num_possible_cpus()) {
  492. pr_debug("clamping worker for cpu %d alive, destroy\n",
  493. i);
  494. stop_power_clamp_worker(i);
  495. }
  496. }
  497. }
  498. static int powerclamp_cpu_online(unsigned int cpu)
  499. {
  500. if (clamping == false)
  501. return 0;
  502. start_power_clamp_worker(cpu);
  503. /* prefer BSP as controlling CPU */
  504. if (cpu == 0) {
  505. control_cpu = 0;
  506. smp_mb();
  507. }
  508. return 0;
  509. }
  510. static int powerclamp_cpu_predown(unsigned int cpu)
  511. {
  512. if (clamping == false)
  513. return 0;
  514. stop_power_clamp_worker(cpu);
  515. if (cpu != control_cpu)
  516. return 0;
  517. control_cpu = cpumask_first(cpu_online_mask);
  518. if (control_cpu == cpu)
  519. control_cpu = cpumask_next(cpu, cpu_online_mask);
  520. smp_mb();
  521. return 0;
  522. }
  523. static int powerclamp_get_max_state(struct thermal_cooling_device *cdev,
  524. unsigned long *state)
  525. {
  526. *state = MAX_TARGET_RATIO;
  527. return 0;
  528. }
  529. static int powerclamp_get_cur_state(struct thermal_cooling_device *cdev,
  530. unsigned long *state)
  531. {
  532. if (true == clamping)
  533. *state = pkg_cstate_ratio_cur;
  534. else
  535. /* to save power, do not poll idle ratio while not clamping */
  536. *state = -1; /* indicates invalid state */
  537. return 0;
  538. }
  539. static int powerclamp_set_cur_state(struct thermal_cooling_device *cdev,
  540. unsigned long new_target_ratio)
  541. {
  542. int ret = 0;
  543. new_target_ratio = clamp(new_target_ratio, 0UL,
  544. (unsigned long) (MAX_TARGET_RATIO-1));
  545. if (set_target_ratio == 0 && new_target_ratio > 0) {
  546. pr_info("Start idle injection to reduce power\n");
  547. set_target_ratio = new_target_ratio;
  548. ret = start_power_clamp();
  549. goto exit_set;
  550. } else if (set_target_ratio > 0 && new_target_ratio == 0) {
  551. pr_info("Stop forced idle injection\n");
  552. end_power_clamp();
  553. set_target_ratio = 0;
  554. } else /* adjust currently running */ {
  555. set_target_ratio = new_target_ratio;
  556. /* make new set_target_ratio visible to other cpus */
  557. smp_mb();
  558. }
  559. exit_set:
  560. return ret;
  561. }
  562. /* bind to generic thermal layer as cooling device*/
  563. static struct thermal_cooling_device_ops powerclamp_cooling_ops = {
  564. .get_max_state = powerclamp_get_max_state,
  565. .get_cur_state = powerclamp_get_cur_state,
  566. .set_cur_state = powerclamp_set_cur_state,
  567. };
  568. static const struct x86_cpu_id __initconst intel_powerclamp_ids[] = {
  569. { X86_VENDOR_INTEL, X86_FAMILY_ANY, X86_MODEL_ANY, X86_FEATURE_MWAIT },
  570. {}
  571. };
  572. MODULE_DEVICE_TABLE(x86cpu, intel_powerclamp_ids);
  573. static int __init powerclamp_probe(void)
  574. {
  575. if (!x86_match_cpu(intel_powerclamp_ids)) {
  576. pr_err("CPU does not support MWAIT\n");
  577. return -ENODEV;
  578. }
  579. /* The goal for idle time alignment is to achieve package cstate. */
  580. if (!has_pkg_state_counter()) {
  581. pr_info("No package C-state available\n");
  582. return -ENODEV;
  583. }
  584. /* find the deepest mwait value */
  585. find_target_mwait();
  586. return 0;
  587. }
  588. static int powerclamp_debug_show(struct seq_file *m, void *unused)
  589. {
  590. int i = 0;
  591. seq_printf(m, "controlling cpu: %d\n", control_cpu);
  592. seq_printf(m, "pct confidence steady dynamic (compensation)\n");
  593. for (i = 0; i < MAX_TARGET_RATIO; i++) {
  594. seq_printf(m, "%d\t%lu\t%lu\t%lu\n",
  595. i,
  596. cal_data[i].confidence,
  597. cal_data[i].steady_comp,
  598. cal_data[i].dynamic_comp);
  599. }
  600. return 0;
  601. }
  602. static int powerclamp_debug_open(struct inode *inode,
  603. struct file *file)
  604. {
  605. return single_open(file, powerclamp_debug_show, inode->i_private);
  606. }
  607. static const struct file_operations powerclamp_debug_fops = {
  608. .open = powerclamp_debug_open,
  609. .read = seq_read,
  610. .llseek = seq_lseek,
  611. .release = single_release,
  612. .owner = THIS_MODULE,
  613. };
  614. static inline void powerclamp_create_debug_files(void)
  615. {
  616. debug_dir = debugfs_create_dir("intel_powerclamp", NULL);
  617. if (!debug_dir)
  618. return;
  619. if (!debugfs_create_file("powerclamp_calib", S_IRUGO, debug_dir,
  620. cal_data, &powerclamp_debug_fops))
  621. goto file_error;
  622. return;
  623. file_error:
  624. debugfs_remove_recursive(debug_dir);
  625. }
  626. static enum cpuhp_state hp_state;
  627. static int __init powerclamp_init(void)
  628. {
  629. int retval;
  630. int bitmap_size;
  631. bitmap_size = BITS_TO_LONGS(num_possible_cpus()) * sizeof(long);
  632. cpu_clamping_mask = kzalloc(bitmap_size, GFP_KERNEL);
  633. if (!cpu_clamping_mask)
  634. return -ENOMEM;
  635. /* probe cpu features and ids here */
  636. retval = powerclamp_probe();
  637. if (retval)
  638. goto exit_free;
  639. /* set default limit, maybe adjusted during runtime based on feedback */
  640. window_size = 2;
  641. retval = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN,
  642. "thermal/intel_powerclamp:online",
  643. powerclamp_cpu_online,
  644. powerclamp_cpu_predown);
  645. if (retval < 0)
  646. goto exit_free;
  647. hp_state = retval;
  648. worker_data = alloc_percpu(struct powerclamp_worker_data);
  649. if (!worker_data) {
  650. retval = -ENOMEM;
  651. goto exit_unregister;
  652. }
  653. cooling_dev = thermal_cooling_device_register("intel_powerclamp", NULL,
  654. &powerclamp_cooling_ops);
  655. if (IS_ERR(cooling_dev)) {
  656. retval = -ENODEV;
  657. goto exit_free_thread;
  658. }
  659. if (!duration)
  660. duration = jiffies_to_msecs(DEFAULT_DURATION_JIFFIES);
  661. powerclamp_create_debug_files();
  662. return 0;
  663. exit_free_thread:
  664. free_percpu(worker_data);
  665. exit_unregister:
  666. cpuhp_remove_state_nocalls(hp_state);
  667. exit_free:
  668. kfree(cpu_clamping_mask);
  669. return retval;
  670. }
  671. module_init(powerclamp_init);
  672. static void __exit powerclamp_exit(void)
  673. {
  674. end_power_clamp();
  675. cpuhp_remove_state_nocalls(hp_state);
  676. free_percpu(worker_data);
  677. thermal_cooling_device_unregister(cooling_dev);
  678. kfree(cpu_clamping_mask);
  679. cancel_delayed_work_sync(&poll_pkg_cstate_work);
  680. debugfs_remove_recursive(debug_dir);
  681. }
  682. module_exit(powerclamp_exit);
  683. MODULE_LICENSE("GPL");
  684. MODULE_AUTHOR("Arjan van de Ven <arjan@linux.intel.com>");
  685. MODULE_AUTHOR("Jacob Pan <jacob.jun.pan@linux.intel.com>");
  686. MODULE_DESCRIPTION("Package Level C-state Idle Injection for Intel CPUs");