trace_sched_wakeup.c 20 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * trace task wakeup timings
  4. *
  5. * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
  6. * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
  7. *
  8. * Based on code from the latency_tracer, that is:
  9. *
  10. * Copyright (C) 2004-2006 Ingo Molnar
  11. * Copyright (C) 2004 Nadia Yvette Chambers
  12. */
  13. #include <linux/module.h>
  14. #include <linux/kallsyms.h>
  15. #include <linux/uaccess.h>
  16. #include <linux/ftrace.h>
  17. #include <linux/sched/rt.h>
  18. #include <linux/sched/deadline.h>
  19. #include <trace/events/sched.h>
  20. #include "trace.h"
  21. static struct trace_array *wakeup_trace;
  22. static int __read_mostly tracer_enabled;
  23. static struct task_struct *wakeup_task;
  24. static int wakeup_cpu;
  25. static int wakeup_current_cpu;
  26. static unsigned wakeup_prio = -1;
  27. static bool wakeup_rt;
  28. static bool wakeup_dl;
  29. static bool tracing_dl;
  30. static arch_spinlock_t wakeup_lock =
  31. (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
  32. static void wakeup_reset(struct trace_array *tr);
  33. static void __wakeup_reset(struct trace_array *tr);
  34. static int start_func_tracer(struct trace_array *tr, int graph);
  35. static void stop_func_tracer(struct trace_array *tr, int graph);
  36. static int save_flags;
  37. #ifdef CONFIG_FUNCTION_GRAPH_TRACER
  38. # define is_graph(tr) ((tr)->trace_flags & TRACE_ITER_DISPLAY_GRAPH)
  39. #else
  40. # define is_graph(tr) false
  41. #endif
  42. #ifdef CONFIG_FUNCTION_TRACER
  43. static bool function_enabled;
  44. /*
  45. * Prologue for the wakeup function tracers.
  46. *
  47. * Returns 1 if it is OK to continue, and preemption
  48. * is disabled and data->disabled is incremented.
  49. * 0 if the trace is to be ignored, and preemption
  50. * is not disabled and data->disabled is
  51. * kept the same.
  52. *
  53. * Note, this function is also used outside this ifdef but
  54. * inside the #ifdef of the function graph tracer below.
  55. * This is OK, since the function graph tracer is
  56. * dependent on the function tracer.
  57. */
  58. static int
  59. func_prolog_preempt_disable(struct trace_array *tr,
  60. struct trace_array_cpu **data,
  61. unsigned int *trace_ctx)
  62. {
  63. long disabled;
  64. int cpu;
  65. if (likely(!wakeup_task))
  66. return 0;
  67. *trace_ctx = tracing_gen_ctx();
  68. preempt_disable_notrace();
  69. cpu = raw_smp_processor_id();
  70. if (cpu != wakeup_current_cpu)
  71. goto out_enable;
  72. *data = per_cpu_ptr(tr->array_buffer.data, cpu);
  73. disabled = atomic_inc_return(&(*data)->disabled);
  74. if (unlikely(disabled != 1))
  75. goto out;
  76. return 1;
  77. out:
  78. atomic_dec(&(*data)->disabled);
  79. out_enable:
  80. preempt_enable_notrace();
  81. return 0;
  82. }
  83. #ifdef CONFIG_FUNCTION_GRAPH_TRACER
  84. static int wakeup_display_graph(struct trace_array *tr, int set)
  85. {
  86. if (!(is_graph(tr) ^ set))
  87. return 0;
  88. stop_func_tracer(tr, !set);
  89. wakeup_reset(wakeup_trace);
  90. tr->max_latency = 0;
  91. return start_func_tracer(tr, set);
  92. }
  93. static int wakeup_graph_entry(struct ftrace_graph_ent *trace,
  94. struct fgraph_ops *gops)
  95. {
  96. struct trace_array *tr = wakeup_trace;
  97. struct trace_array_cpu *data;
  98. unsigned int trace_ctx;
  99. int ret = 0;
  100. if (ftrace_graph_ignore_func(gops, trace))
  101. return 0;
  102. /*
  103. * Do not trace a function if it's filtered by set_graph_notrace.
  104. * Make the index of ret stack negative to indicate that it should
  105. * ignore further functions. But it needs its own ret stack entry
  106. * to recover the original index in order to continue tracing after
  107. * returning from the function.
  108. */
  109. if (ftrace_graph_notrace_addr(trace->func))
  110. return 1;
  111. if (!func_prolog_preempt_disable(tr, &data, &trace_ctx))
  112. return 0;
  113. ret = __trace_graph_entry(tr, trace, trace_ctx);
  114. atomic_dec(&data->disabled);
  115. preempt_enable_notrace();
  116. return ret;
  117. }
  118. static void wakeup_graph_return(struct ftrace_graph_ret *trace,
  119. struct fgraph_ops *gops)
  120. {
  121. struct trace_array *tr = wakeup_trace;
  122. struct trace_array_cpu *data;
  123. unsigned int trace_ctx;
  124. ftrace_graph_addr_finish(gops, trace);
  125. if (!func_prolog_preempt_disable(tr, &data, &trace_ctx))
  126. return;
  127. __trace_graph_return(tr, trace, trace_ctx);
  128. atomic_dec(&data->disabled);
  129. preempt_enable_notrace();
  130. return;
  131. }
  132. static struct fgraph_ops fgraph_wakeup_ops = {
  133. .entryfunc = &wakeup_graph_entry,
  134. .retfunc = &wakeup_graph_return,
  135. };
  136. static void wakeup_trace_open(struct trace_iterator *iter)
  137. {
  138. if (is_graph(iter->tr))
  139. graph_trace_open(iter);
  140. else
  141. iter->private = NULL;
  142. }
  143. static void wakeup_trace_close(struct trace_iterator *iter)
  144. {
  145. if (iter->private)
  146. graph_trace_close(iter);
  147. }
  148. #define GRAPH_TRACER_FLAGS (TRACE_GRAPH_PRINT_PROC | \
  149. TRACE_GRAPH_PRINT_CPU | \
  150. TRACE_GRAPH_PRINT_REL_TIME | \
  151. TRACE_GRAPH_PRINT_DURATION | \
  152. TRACE_GRAPH_PRINT_OVERHEAD | \
  153. TRACE_GRAPH_PRINT_IRQS)
  154. static enum print_line_t wakeup_print_line(struct trace_iterator *iter)
  155. {
  156. /*
  157. * In graph mode call the graph tracer output function,
  158. * otherwise go with the TRACE_FN event handler
  159. */
  160. if (is_graph(iter->tr))
  161. return print_graph_function_flags(iter, GRAPH_TRACER_FLAGS);
  162. return TRACE_TYPE_UNHANDLED;
  163. }
  164. static void wakeup_print_header(struct seq_file *s)
  165. {
  166. if (is_graph(wakeup_trace))
  167. print_graph_headers_flags(s, GRAPH_TRACER_FLAGS);
  168. else
  169. trace_default_header(s);
  170. }
  171. #endif /* else CONFIG_FUNCTION_GRAPH_TRACER */
  172. /*
  173. * wakeup uses its own tracer function to keep the overhead down:
  174. */
  175. static void
  176. wakeup_tracer_call(unsigned long ip, unsigned long parent_ip,
  177. struct ftrace_ops *op, struct ftrace_regs *fregs)
  178. {
  179. struct trace_array *tr = wakeup_trace;
  180. struct trace_array_cpu *data;
  181. unsigned long flags;
  182. unsigned int trace_ctx;
  183. if (!func_prolog_preempt_disable(tr, &data, &trace_ctx))
  184. return;
  185. local_irq_save(flags);
  186. trace_function(tr, ip, parent_ip, trace_ctx);
  187. local_irq_restore(flags);
  188. atomic_dec(&data->disabled);
  189. preempt_enable_notrace();
  190. }
  191. static int register_wakeup_function(struct trace_array *tr, int graph, int set)
  192. {
  193. int ret;
  194. /* 'set' is set if TRACE_ITER_FUNCTION is about to be set */
  195. if (function_enabled || (!set && !(tr->trace_flags & TRACE_ITER_FUNCTION)))
  196. return 0;
  197. if (graph)
  198. ret = register_ftrace_graph(&fgraph_wakeup_ops);
  199. else
  200. ret = register_ftrace_function(tr->ops);
  201. if (!ret)
  202. function_enabled = true;
  203. return ret;
  204. }
  205. static void unregister_wakeup_function(struct trace_array *tr, int graph)
  206. {
  207. if (!function_enabled)
  208. return;
  209. if (graph)
  210. unregister_ftrace_graph(&fgraph_wakeup_ops);
  211. else
  212. unregister_ftrace_function(tr->ops);
  213. function_enabled = false;
  214. }
  215. static int wakeup_function_set(struct trace_array *tr, u32 mask, int set)
  216. {
  217. if (!(mask & TRACE_ITER_FUNCTION))
  218. return 0;
  219. if (set)
  220. register_wakeup_function(tr, is_graph(tr), 1);
  221. else
  222. unregister_wakeup_function(tr, is_graph(tr));
  223. return 1;
  224. }
  225. #else /* CONFIG_FUNCTION_TRACER */
  226. static int register_wakeup_function(struct trace_array *tr, int graph, int set)
  227. {
  228. return 0;
  229. }
  230. static void unregister_wakeup_function(struct trace_array *tr, int graph) { }
  231. static int wakeup_function_set(struct trace_array *tr, u32 mask, int set)
  232. {
  233. return 0;
  234. }
  235. #endif /* else CONFIG_FUNCTION_TRACER */
  236. #ifndef CONFIG_FUNCTION_GRAPH_TRACER
  237. static enum print_line_t wakeup_print_line(struct trace_iterator *iter)
  238. {
  239. return TRACE_TYPE_UNHANDLED;
  240. }
  241. static void wakeup_trace_open(struct trace_iterator *iter) { }
  242. static void wakeup_trace_close(struct trace_iterator *iter) { }
  243. static void wakeup_print_header(struct seq_file *s)
  244. {
  245. trace_default_header(s);
  246. }
  247. #endif /* !CONFIG_FUNCTION_GRAPH_TRACER */
  248. static void
  249. __trace_function(struct trace_array *tr,
  250. unsigned long ip, unsigned long parent_ip,
  251. unsigned int trace_ctx)
  252. {
  253. if (is_graph(tr))
  254. trace_graph_function(tr, ip, parent_ip, trace_ctx);
  255. else
  256. trace_function(tr, ip, parent_ip, trace_ctx);
  257. }
  258. static int wakeup_flag_changed(struct trace_array *tr, u32 mask, int set)
  259. {
  260. struct tracer *tracer = tr->current_trace;
  261. if (wakeup_function_set(tr, mask, set))
  262. return 0;
  263. #ifdef CONFIG_FUNCTION_GRAPH_TRACER
  264. if (mask & TRACE_ITER_DISPLAY_GRAPH)
  265. return wakeup_display_graph(tr, set);
  266. #endif
  267. return trace_keep_overwrite(tracer, mask, set);
  268. }
  269. static int start_func_tracer(struct trace_array *tr, int graph)
  270. {
  271. int ret;
  272. ret = register_wakeup_function(tr, graph, 0);
  273. if (!ret && tracing_is_enabled())
  274. tracer_enabled = 1;
  275. else
  276. tracer_enabled = 0;
  277. return ret;
  278. }
  279. static void stop_func_tracer(struct trace_array *tr, int graph)
  280. {
  281. tracer_enabled = 0;
  282. unregister_wakeup_function(tr, graph);
  283. }
  284. /*
  285. * Should this new latency be reported/recorded?
  286. */
  287. static bool report_latency(struct trace_array *tr, u64 delta)
  288. {
  289. if (tracing_thresh) {
  290. if (delta < tracing_thresh)
  291. return false;
  292. } else {
  293. if (delta <= tr->max_latency)
  294. return false;
  295. }
  296. return true;
  297. }
  298. static void
  299. probe_wakeup_migrate_task(void *ignore, struct task_struct *task, int cpu)
  300. {
  301. if (task != wakeup_task)
  302. return;
  303. wakeup_current_cpu = cpu;
  304. }
  305. static void
  306. tracing_sched_switch_trace(struct trace_array *tr,
  307. struct task_struct *prev,
  308. struct task_struct *next,
  309. unsigned int trace_ctx)
  310. {
  311. struct trace_event_call *call = &event_context_switch;
  312. struct trace_buffer *buffer = tr->array_buffer.buffer;
  313. struct ring_buffer_event *event;
  314. struct ctx_switch_entry *entry;
  315. event = trace_buffer_lock_reserve(buffer, TRACE_CTX,
  316. sizeof(*entry), trace_ctx);
  317. if (!event)
  318. return;
  319. entry = ring_buffer_event_data(event);
  320. entry->prev_pid = prev->pid;
  321. entry->prev_prio = prev->prio;
  322. entry->prev_state = task_state_index(prev);
  323. entry->next_pid = next->pid;
  324. entry->next_prio = next->prio;
  325. entry->next_state = task_state_index(next);
  326. entry->next_cpu = task_cpu(next);
  327. if (!call_filter_check_discard(call, entry, buffer, event))
  328. trace_buffer_unlock_commit(tr, buffer, event, trace_ctx);
  329. }
  330. static void
  331. tracing_sched_wakeup_trace(struct trace_array *tr,
  332. struct task_struct *wakee,
  333. struct task_struct *curr,
  334. unsigned int trace_ctx)
  335. {
  336. struct trace_event_call *call = &event_wakeup;
  337. struct ring_buffer_event *event;
  338. struct ctx_switch_entry *entry;
  339. struct trace_buffer *buffer = tr->array_buffer.buffer;
  340. event = trace_buffer_lock_reserve(buffer, TRACE_WAKE,
  341. sizeof(*entry), trace_ctx);
  342. if (!event)
  343. return;
  344. entry = ring_buffer_event_data(event);
  345. entry->prev_pid = curr->pid;
  346. entry->prev_prio = curr->prio;
  347. entry->prev_state = task_state_index(curr);
  348. entry->next_pid = wakee->pid;
  349. entry->next_prio = wakee->prio;
  350. entry->next_state = task_state_index(wakee);
  351. entry->next_cpu = task_cpu(wakee);
  352. if (!call_filter_check_discard(call, entry, buffer, event))
  353. trace_buffer_unlock_commit(tr, buffer, event, trace_ctx);
  354. }
  355. static void notrace
  356. probe_wakeup_sched_switch(void *ignore, bool preempt,
  357. struct task_struct *prev, struct task_struct *next,
  358. unsigned int prev_state)
  359. {
  360. struct trace_array_cpu *data;
  361. u64 T0, T1, delta;
  362. unsigned long flags;
  363. long disabled;
  364. int cpu;
  365. unsigned int trace_ctx;
  366. tracing_record_cmdline(prev);
  367. if (unlikely(!tracer_enabled))
  368. return;
  369. /*
  370. * When we start a new trace, we set wakeup_task to NULL
  371. * and then set tracer_enabled = 1. We want to make sure
  372. * that another CPU does not see the tracer_enabled = 1
  373. * and the wakeup_task with an older task, that might
  374. * actually be the same as next.
  375. */
  376. smp_rmb();
  377. if (next != wakeup_task)
  378. return;
  379. /* disable local data, not wakeup_cpu data */
  380. cpu = raw_smp_processor_id();
  381. disabled = atomic_inc_return(&per_cpu_ptr(wakeup_trace->array_buffer.data, cpu)->disabled);
  382. if (likely(disabled != 1))
  383. goto out;
  384. local_irq_save(flags);
  385. trace_ctx = tracing_gen_ctx_flags(flags);
  386. arch_spin_lock(&wakeup_lock);
  387. /* We could race with grabbing wakeup_lock */
  388. if (unlikely(!tracer_enabled || next != wakeup_task))
  389. goto out_unlock;
  390. /* The task we are waiting for is waking up */
  391. data = per_cpu_ptr(wakeup_trace->array_buffer.data, wakeup_cpu);
  392. __trace_function(wakeup_trace, CALLER_ADDR0, CALLER_ADDR1, trace_ctx);
  393. tracing_sched_switch_trace(wakeup_trace, prev, next, trace_ctx);
  394. __trace_stack(wakeup_trace, trace_ctx, 0);
  395. T0 = data->preempt_timestamp;
  396. T1 = ftrace_now(cpu);
  397. delta = T1-T0;
  398. if (!report_latency(wakeup_trace, delta))
  399. goto out_unlock;
  400. if (likely(!is_tracing_stopped())) {
  401. wakeup_trace->max_latency = delta;
  402. update_max_tr(wakeup_trace, wakeup_task, wakeup_cpu, NULL);
  403. }
  404. out_unlock:
  405. __wakeup_reset(wakeup_trace);
  406. arch_spin_unlock(&wakeup_lock);
  407. local_irq_restore(flags);
  408. out:
  409. atomic_dec(&per_cpu_ptr(wakeup_trace->array_buffer.data, cpu)->disabled);
  410. }
  411. static void __wakeup_reset(struct trace_array *tr)
  412. {
  413. wakeup_cpu = -1;
  414. wakeup_prio = -1;
  415. tracing_dl = false;
  416. if (wakeup_task)
  417. put_task_struct(wakeup_task);
  418. wakeup_task = NULL;
  419. }
  420. static void wakeup_reset(struct trace_array *tr)
  421. {
  422. unsigned long flags;
  423. tracing_reset_online_cpus(&tr->array_buffer);
  424. local_irq_save(flags);
  425. arch_spin_lock(&wakeup_lock);
  426. __wakeup_reset(tr);
  427. arch_spin_unlock(&wakeup_lock);
  428. local_irq_restore(flags);
  429. }
  430. static void
  431. probe_wakeup(void *ignore, struct task_struct *p)
  432. {
  433. struct trace_array_cpu *data;
  434. int cpu = smp_processor_id();
  435. long disabled;
  436. unsigned int trace_ctx;
  437. if (likely(!tracer_enabled))
  438. return;
  439. tracing_record_cmdline(p);
  440. tracing_record_cmdline(current);
  441. /*
  442. * Semantic is like this:
  443. * - wakeup tracer handles all tasks in the system, independently
  444. * from their scheduling class;
  445. * - wakeup_rt tracer handles tasks belonging to sched_dl and
  446. * sched_rt class;
  447. * - wakeup_dl handles tasks belonging to sched_dl class only.
  448. */
  449. if (tracing_dl || (wakeup_dl && !dl_task(p)) ||
  450. (wakeup_rt && !rt_or_dl_task(p)) ||
  451. (!dl_task(p) && (p->prio >= wakeup_prio || p->prio >= current->prio)))
  452. return;
  453. disabled = atomic_inc_return(&per_cpu_ptr(wakeup_trace->array_buffer.data, cpu)->disabled);
  454. if (unlikely(disabled != 1))
  455. goto out;
  456. trace_ctx = tracing_gen_ctx();
  457. /* interrupts should be off from try_to_wake_up */
  458. arch_spin_lock(&wakeup_lock);
  459. /* check for races. */
  460. if (!tracer_enabled || tracing_dl ||
  461. (!dl_task(p) && p->prio >= wakeup_prio))
  462. goto out_locked;
  463. /* reset the trace */
  464. __wakeup_reset(wakeup_trace);
  465. wakeup_cpu = task_cpu(p);
  466. wakeup_current_cpu = wakeup_cpu;
  467. wakeup_prio = p->prio;
  468. /*
  469. * Once you start tracing a -deadline task, don't bother tracing
  470. * another task until the first one wakes up.
  471. */
  472. if (dl_task(p))
  473. tracing_dl = true;
  474. else
  475. tracing_dl = false;
  476. wakeup_task = get_task_struct(p);
  477. data = per_cpu_ptr(wakeup_trace->array_buffer.data, wakeup_cpu);
  478. data->preempt_timestamp = ftrace_now(cpu);
  479. tracing_sched_wakeup_trace(wakeup_trace, p, current, trace_ctx);
  480. __trace_stack(wakeup_trace, trace_ctx, 0);
  481. /*
  482. * We must be careful in using CALLER_ADDR2. But since wake_up
  483. * is not called by an assembly function (where as schedule is)
  484. * it should be safe to use it here.
  485. */
  486. __trace_function(wakeup_trace, CALLER_ADDR1, CALLER_ADDR2, trace_ctx);
  487. out_locked:
  488. arch_spin_unlock(&wakeup_lock);
  489. out:
  490. atomic_dec(&per_cpu_ptr(wakeup_trace->array_buffer.data, cpu)->disabled);
  491. }
  492. static void start_wakeup_tracer(struct trace_array *tr)
  493. {
  494. int ret;
  495. ret = register_trace_sched_wakeup(probe_wakeup, NULL);
  496. if (ret) {
  497. pr_info("wakeup trace: Couldn't activate tracepoint"
  498. " probe to kernel_sched_wakeup\n");
  499. return;
  500. }
  501. ret = register_trace_sched_wakeup_new(probe_wakeup, NULL);
  502. if (ret) {
  503. pr_info("wakeup trace: Couldn't activate tracepoint"
  504. " probe to kernel_sched_wakeup_new\n");
  505. goto fail_deprobe;
  506. }
  507. ret = register_trace_sched_switch(probe_wakeup_sched_switch, NULL);
  508. if (ret) {
  509. pr_info("sched trace: Couldn't activate tracepoint"
  510. " probe to kernel_sched_switch\n");
  511. goto fail_deprobe_wake_new;
  512. }
  513. ret = register_trace_sched_migrate_task(probe_wakeup_migrate_task, NULL);
  514. if (ret) {
  515. pr_info("wakeup trace: Couldn't activate tracepoint"
  516. " probe to kernel_sched_migrate_task\n");
  517. goto fail_deprobe_sched_switch;
  518. }
  519. wakeup_reset(tr);
  520. /*
  521. * Don't let the tracer_enabled = 1 show up before
  522. * the wakeup_task is reset. This may be overkill since
  523. * wakeup_reset does a spin_unlock after setting the
  524. * wakeup_task to NULL, but I want to be safe.
  525. * This is a slow path anyway.
  526. */
  527. smp_wmb();
  528. if (start_func_tracer(tr, is_graph(tr)))
  529. printk(KERN_ERR "failed to start wakeup tracer\n");
  530. return;
  531. fail_deprobe_sched_switch:
  532. unregister_trace_sched_switch(probe_wakeup_sched_switch, NULL);
  533. fail_deprobe_wake_new:
  534. unregister_trace_sched_wakeup_new(probe_wakeup, NULL);
  535. fail_deprobe:
  536. unregister_trace_sched_wakeup(probe_wakeup, NULL);
  537. }
  538. static void stop_wakeup_tracer(struct trace_array *tr)
  539. {
  540. tracer_enabled = 0;
  541. stop_func_tracer(tr, is_graph(tr));
  542. unregister_trace_sched_switch(probe_wakeup_sched_switch, NULL);
  543. unregister_trace_sched_wakeup_new(probe_wakeup, NULL);
  544. unregister_trace_sched_wakeup(probe_wakeup, NULL);
  545. unregister_trace_sched_migrate_task(probe_wakeup_migrate_task, NULL);
  546. }
  547. static bool wakeup_busy;
  548. static int __wakeup_tracer_init(struct trace_array *tr)
  549. {
  550. save_flags = tr->trace_flags;
  551. /* non overwrite screws up the latency tracers */
  552. set_tracer_flag(tr, TRACE_ITER_OVERWRITE, 1);
  553. set_tracer_flag(tr, TRACE_ITER_LATENCY_FMT, 1);
  554. tr->max_latency = 0;
  555. wakeup_trace = tr;
  556. ftrace_init_array_ops(tr, wakeup_tracer_call);
  557. start_wakeup_tracer(tr);
  558. wakeup_busy = true;
  559. return 0;
  560. }
  561. static int wakeup_tracer_init(struct trace_array *tr)
  562. {
  563. if (wakeup_busy)
  564. return -EBUSY;
  565. wakeup_dl = false;
  566. wakeup_rt = false;
  567. return __wakeup_tracer_init(tr);
  568. }
  569. static int wakeup_rt_tracer_init(struct trace_array *tr)
  570. {
  571. if (wakeup_busy)
  572. return -EBUSY;
  573. wakeup_dl = false;
  574. wakeup_rt = true;
  575. return __wakeup_tracer_init(tr);
  576. }
  577. static int wakeup_dl_tracer_init(struct trace_array *tr)
  578. {
  579. if (wakeup_busy)
  580. return -EBUSY;
  581. wakeup_dl = true;
  582. wakeup_rt = false;
  583. return __wakeup_tracer_init(tr);
  584. }
  585. static void wakeup_tracer_reset(struct trace_array *tr)
  586. {
  587. int lat_flag = save_flags & TRACE_ITER_LATENCY_FMT;
  588. int overwrite_flag = save_flags & TRACE_ITER_OVERWRITE;
  589. stop_wakeup_tracer(tr);
  590. /* make sure we put back any tasks we are tracing */
  591. wakeup_reset(tr);
  592. set_tracer_flag(tr, TRACE_ITER_LATENCY_FMT, lat_flag);
  593. set_tracer_flag(tr, TRACE_ITER_OVERWRITE, overwrite_flag);
  594. ftrace_reset_array_ops(tr);
  595. wakeup_busy = false;
  596. }
  597. static void wakeup_tracer_start(struct trace_array *tr)
  598. {
  599. wakeup_reset(tr);
  600. tracer_enabled = 1;
  601. }
  602. static void wakeup_tracer_stop(struct trace_array *tr)
  603. {
  604. tracer_enabled = 0;
  605. }
  606. static struct tracer wakeup_tracer __read_mostly =
  607. {
  608. .name = "wakeup",
  609. .init = wakeup_tracer_init,
  610. .reset = wakeup_tracer_reset,
  611. .start = wakeup_tracer_start,
  612. .stop = wakeup_tracer_stop,
  613. .print_max = true,
  614. .print_header = wakeup_print_header,
  615. .print_line = wakeup_print_line,
  616. .flag_changed = wakeup_flag_changed,
  617. #ifdef CONFIG_FTRACE_SELFTEST
  618. .selftest = trace_selftest_startup_wakeup,
  619. #endif
  620. .open = wakeup_trace_open,
  621. .close = wakeup_trace_close,
  622. .allow_instances = true,
  623. .use_max_tr = true,
  624. };
  625. static struct tracer wakeup_rt_tracer __read_mostly =
  626. {
  627. .name = "wakeup_rt",
  628. .init = wakeup_rt_tracer_init,
  629. .reset = wakeup_tracer_reset,
  630. .start = wakeup_tracer_start,
  631. .stop = wakeup_tracer_stop,
  632. .print_max = true,
  633. .print_header = wakeup_print_header,
  634. .print_line = wakeup_print_line,
  635. .flag_changed = wakeup_flag_changed,
  636. #ifdef CONFIG_FTRACE_SELFTEST
  637. .selftest = trace_selftest_startup_wakeup,
  638. #endif
  639. .open = wakeup_trace_open,
  640. .close = wakeup_trace_close,
  641. .allow_instances = true,
  642. .use_max_tr = true,
  643. };
  644. static struct tracer wakeup_dl_tracer __read_mostly =
  645. {
  646. .name = "wakeup_dl",
  647. .init = wakeup_dl_tracer_init,
  648. .reset = wakeup_tracer_reset,
  649. .start = wakeup_tracer_start,
  650. .stop = wakeup_tracer_stop,
  651. .print_max = true,
  652. .print_header = wakeup_print_header,
  653. .print_line = wakeup_print_line,
  654. .flag_changed = wakeup_flag_changed,
  655. #ifdef CONFIG_FTRACE_SELFTEST
  656. .selftest = trace_selftest_startup_wakeup,
  657. #endif
  658. .open = wakeup_trace_open,
  659. .close = wakeup_trace_close,
  660. .allow_instances = true,
  661. .use_max_tr = true,
  662. };
  663. __init static int init_wakeup_tracer(void)
  664. {
  665. int ret;
  666. ret = register_tracer(&wakeup_tracer);
  667. if (ret)
  668. return ret;
  669. ret = register_tracer(&wakeup_rt_tracer);
  670. if (ret)
  671. return ret;
  672. ret = register_tracer(&wakeup_dl_tracer);
  673. if (ret)
  674. return ret;
  675. return 0;
  676. }
  677. core_initcall(init_wakeup_tracer);