trace.h 61 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957
  1. // SPDX-License-Identifier: GPL-2.0
  2. #ifndef _LINUX_KERNEL_TRACE_H
  3. #define _LINUX_KERNEL_TRACE_H
  4. #include <linux/fs.h>
  5. #include <linux/atomic.h>
  6. #include <linux/sched.h>
  7. #include <linux/clocksource.h>
  8. #include <linux/ring_buffer.h>
  9. #include <linux/mmiotrace.h>
  10. #include <linux/tracepoint.h>
  11. #include <linux/ftrace.h>
  12. #include <linux/hw_breakpoint.h>
  13. #include <linux/trace_seq.h>
  14. #include <linux/trace_events.h>
  15. #include <linux/compiler.h>
  16. #include <linux/trace_seq.h>
  17. #include <linux/glob.h>
  18. #ifdef CONFIG_FTRACE_SYSCALLS
  19. #include <asm/unistd.h> /* For NR_SYSCALLS */
  20. #include <asm/syscall.h> /* some archs define it here */
  21. #endif
  22. enum trace_type {
  23. __TRACE_FIRST_TYPE = 0,
  24. TRACE_FN,
  25. TRACE_CTX,
  26. TRACE_WAKE,
  27. TRACE_STACK,
  28. TRACE_PRINT,
  29. TRACE_BPRINT,
  30. TRACE_MMIO_RW,
  31. TRACE_MMIO_MAP,
  32. TRACE_BRANCH,
  33. TRACE_GRAPH_RET,
  34. TRACE_GRAPH_ENT,
  35. TRACE_USER_STACK,
  36. TRACE_BLK,
  37. TRACE_BPUTS,
  38. TRACE_HWLAT,
  39. TRACE_RAW_DATA,
  40. __TRACE_LAST_TYPE,
  41. };
  42. #undef __field
  43. #define __field(type, item) type item;
  44. #undef __field_struct
  45. #define __field_struct(type, item) __field(type, item)
  46. #undef __field_desc
  47. #define __field_desc(type, container, item)
  48. #undef __array
  49. #define __array(type, item, size) type item[size];
  50. #undef __array_desc
  51. #define __array_desc(type, container, item, size)
  52. #undef __dynamic_array
  53. #define __dynamic_array(type, item) type item[];
  54. #undef F_STRUCT
  55. #define F_STRUCT(args...) args
  56. #undef FTRACE_ENTRY
  57. #define FTRACE_ENTRY(name, struct_name, id, tstruct, print, filter) \
  58. struct struct_name { \
  59. struct trace_entry ent; \
  60. tstruct \
  61. }
  62. #undef FTRACE_ENTRY_DUP
  63. #define FTRACE_ENTRY_DUP(name, name_struct, id, tstruct, printk, filter)
  64. #undef FTRACE_ENTRY_REG
  65. #define FTRACE_ENTRY_REG(name, struct_name, id, tstruct, print, \
  66. filter, regfn) \
  67. FTRACE_ENTRY(name, struct_name, id, PARAMS(tstruct), PARAMS(print), \
  68. filter)
  69. #undef FTRACE_ENTRY_PACKED
  70. #define FTRACE_ENTRY_PACKED(name, struct_name, id, tstruct, print, \
  71. filter) \
  72. FTRACE_ENTRY(name, struct_name, id, PARAMS(tstruct), PARAMS(print), \
  73. filter) __packed
  74. #include "trace_entries.h"
  75. /*
  76. * syscalls are special, and need special handling, this is why
  77. * they are not included in trace_entries.h
  78. */
  79. struct syscall_trace_enter {
  80. struct trace_entry ent;
  81. int nr;
  82. unsigned long args[];
  83. };
  84. struct syscall_trace_exit {
  85. struct trace_entry ent;
  86. int nr;
  87. long ret;
  88. };
  89. struct kprobe_trace_entry_head {
  90. struct trace_entry ent;
  91. unsigned long ip;
  92. };
  93. struct kretprobe_trace_entry_head {
  94. struct trace_entry ent;
  95. unsigned long func;
  96. unsigned long ret_ip;
  97. };
  98. /*
  99. * trace_flag_type is an enumeration that holds different
  100. * states when a trace occurs. These are:
  101. * IRQS_OFF - interrupts were disabled
  102. * IRQS_NOSUPPORT - arch does not support irqs_disabled_flags
  103. * NEED_RESCHED - reschedule is requested
  104. * HARDIRQ - inside an interrupt handler
  105. * SOFTIRQ - inside a softirq handler
  106. */
  107. enum trace_flag_type {
  108. TRACE_FLAG_IRQS_OFF = 0x01,
  109. TRACE_FLAG_IRQS_NOSUPPORT = 0x02,
  110. TRACE_FLAG_NEED_RESCHED = 0x04,
  111. TRACE_FLAG_HARDIRQ = 0x08,
  112. TRACE_FLAG_SOFTIRQ = 0x10,
  113. TRACE_FLAG_PREEMPT_RESCHED = 0x20,
  114. TRACE_FLAG_NMI = 0x40,
  115. };
  116. #define TRACE_BUF_SIZE 1024
  117. struct trace_array;
  118. /*
  119. * The CPU trace array - it consists of thousands of trace entries
  120. * plus some other descriptor data: (for example which task started
  121. * the trace, etc.)
  122. */
  123. struct trace_array_cpu {
  124. atomic_t disabled;
  125. void *buffer_page; /* ring buffer spare */
  126. unsigned long entries;
  127. unsigned long saved_latency;
  128. unsigned long critical_start;
  129. unsigned long critical_end;
  130. unsigned long critical_sequence;
  131. unsigned long nice;
  132. unsigned long policy;
  133. unsigned long rt_priority;
  134. unsigned long skipped_entries;
  135. u64 preempt_timestamp;
  136. pid_t pid;
  137. kuid_t uid;
  138. char comm[TASK_COMM_LEN];
  139. bool ignore_pid;
  140. #ifdef CONFIG_FUNCTION_TRACER
  141. bool ftrace_ignore_pid;
  142. #endif
  143. };
  144. struct tracer;
  145. struct trace_option_dentry;
  146. struct trace_buffer {
  147. struct trace_array *tr;
  148. struct ring_buffer *buffer;
  149. struct trace_array_cpu __percpu *data;
  150. u64 time_start;
  151. int cpu;
  152. };
  153. #define TRACE_FLAGS_MAX_SIZE 32
  154. struct trace_options {
  155. struct tracer *tracer;
  156. struct trace_option_dentry *topts;
  157. };
  158. struct trace_pid_list {
  159. int pid_max;
  160. unsigned long *pids;
  161. };
  162. /*
  163. * The trace array - an array of per-CPU trace arrays. This is the
  164. * highest level data structure that individual tracers deal with.
  165. * They have on/off state as well:
  166. */
  167. struct trace_array {
  168. struct list_head list;
  169. char *name;
  170. struct trace_buffer trace_buffer;
  171. #ifdef CONFIG_TRACER_MAX_TRACE
  172. /*
  173. * The max_buffer is used to snapshot the trace when a maximum
  174. * latency is reached, or when the user initiates a snapshot.
  175. * Some tracers will use this to store a maximum trace while
  176. * it continues examining live traces.
  177. *
  178. * The buffers for the max_buffer are set up the same as the trace_buffer
  179. * When a snapshot is taken, the buffer of the max_buffer is swapped
  180. * with the buffer of the trace_buffer and the buffers are reset for
  181. * the trace_buffer so the tracing can continue.
  182. */
  183. struct trace_buffer max_buffer;
  184. bool allocated_snapshot;
  185. #endif
  186. #if defined(CONFIG_TRACER_MAX_TRACE) || defined(CONFIG_HWLAT_TRACER)
  187. unsigned long max_latency;
  188. #endif
  189. struct trace_pid_list __rcu *filtered_pids;
  190. /*
  191. * max_lock is used to protect the swapping of buffers
  192. * when taking a max snapshot. The buffers themselves are
  193. * protected by per_cpu spinlocks. But the action of the swap
  194. * needs its own lock.
  195. *
  196. * This is defined as a arch_spinlock_t in order to help
  197. * with performance when lockdep debugging is enabled.
  198. *
  199. * It is also used in other places outside the update_max_tr
  200. * so it needs to be defined outside of the
  201. * CONFIG_TRACER_MAX_TRACE.
  202. */
  203. arch_spinlock_t max_lock;
  204. int buffer_disabled;
  205. #ifdef CONFIG_FTRACE_SYSCALLS
  206. int sys_refcount_enter;
  207. int sys_refcount_exit;
  208. struct trace_event_file __rcu *enter_syscall_files[NR_syscalls];
  209. struct trace_event_file __rcu *exit_syscall_files[NR_syscalls];
  210. #endif
  211. int stop_count;
  212. int clock_id;
  213. int nr_topts;
  214. bool clear_trace;
  215. struct tracer *current_trace;
  216. unsigned int trace_flags;
  217. unsigned char trace_flags_index[TRACE_FLAGS_MAX_SIZE];
  218. unsigned int flags;
  219. raw_spinlock_t start_lock;
  220. struct dentry *dir;
  221. struct dentry *options;
  222. struct dentry *percpu_dir;
  223. struct dentry *event_dir;
  224. struct trace_options *topts;
  225. struct list_head systems;
  226. struct list_head events;
  227. struct trace_event_file *trace_marker_file;
  228. cpumask_var_t tracing_cpumask; /* only trace on set CPUs */
  229. int ref;
  230. #ifdef CONFIG_FUNCTION_TRACER
  231. struct ftrace_ops *ops;
  232. struct trace_pid_list __rcu *function_pids;
  233. #ifdef CONFIG_DYNAMIC_FTRACE
  234. /* All of these are protected by the ftrace_lock */
  235. struct list_head func_probes;
  236. struct list_head mod_trace;
  237. struct list_head mod_notrace;
  238. #endif
  239. /* function tracing enabled */
  240. int function_enabled;
  241. #endif
  242. int time_stamp_abs_ref;
  243. struct list_head hist_vars;
  244. };
  245. enum {
  246. TRACE_ARRAY_FL_GLOBAL = (1 << 0)
  247. };
  248. extern struct list_head ftrace_trace_arrays;
  249. extern struct mutex trace_types_lock;
  250. extern int trace_array_get(struct trace_array *tr);
  251. extern void trace_array_put(struct trace_array *tr);
  252. extern int tracing_set_time_stamp_abs(struct trace_array *tr, bool abs);
  253. extern int tracing_set_clock(struct trace_array *tr, const char *clockstr);
  254. extern bool trace_clock_in_ns(struct trace_array *tr);
  255. /*
  256. * The global tracer (top) should be the first trace array added,
  257. * but we check the flag anyway.
  258. */
  259. static inline struct trace_array *top_trace_array(void)
  260. {
  261. struct trace_array *tr;
  262. if (list_empty(&ftrace_trace_arrays))
  263. return NULL;
  264. tr = list_entry(ftrace_trace_arrays.prev,
  265. typeof(*tr), list);
  266. WARN_ON(!(tr->flags & TRACE_ARRAY_FL_GLOBAL));
  267. return tr;
  268. }
  269. #define FTRACE_CMP_TYPE(var, type) \
  270. __builtin_types_compatible_p(typeof(var), type *)
  271. #undef IF_ASSIGN
  272. #define IF_ASSIGN(var, entry, etype, id) \
  273. if (FTRACE_CMP_TYPE(var, etype)) { \
  274. var = (typeof(var))(entry); \
  275. WARN_ON(id && (entry)->type != id); \
  276. break; \
  277. }
  278. /* Will cause compile errors if type is not found. */
  279. extern void __ftrace_bad_type(void);
  280. /*
  281. * The trace_assign_type is a verifier that the entry type is
  282. * the same as the type being assigned. To add new types simply
  283. * add a line with the following format:
  284. *
  285. * IF_ASSIGN(var, ent, type, id);
  286. *
  287. * Where "type" is the trace type that includes the trace_entry
  288. * as the "ent" item. And "id" is the trace identifier that is
  289. * used in the trace_type enum.
  290. *
  291. * If the type can have more than one id, then use zero.
  292. */
  293. #define trace_assign_type(var, ent) \
  294. do { \
  295. IF_ASSIGN(var, ent, struct ftrace_entry, TRACE_FN); \
  296. IF_ASSIGN(var, ent, struct ctx_switch_entry, 0); \
  297. IF_ASSIGN(var, ent, struct stack_entry, TRACE_STACK); \
  298. IF_ASSIGN(var, ent, struct userstack_entry, TRACE_USER_STACK);\
  299. IF_ASSIGN(var, ent, struct print_entry, TRACE_PRINT); \
  300. IF_ASSIGN(var, ent, struct bprint_entry, TRACE_BPRINT); \
  301. IF_ASSIGN(var, ent, struct bputs_entry, TRACE_BPUTS); \
  302. IF_ASSIGN(var, ent, struct hwlat_entry, TRACE_HWLAT); \
  303. IF_ASSIGN(var, ent, struct raw_data_entry, TRACE_RAW_DATA);\
  304. IF_ASSIGN(var, ent, struct trace_mmiotrace_rw, \
  305. TRACE_MMIO_RW); \
  306. IF_ASSIGN(var, ent, struct trace_mmiotrace_map, \
  307. TRACE_MMIO_MAP); \
  308. IF_ASSIGN(var, ent, struct trace_branch, TRACE_BRANCH); \
  309. IF_ASSIGN(var, ent, struct ftrace_graph_ent_entry, \
  310. TRACE_GRAPH_ENT); \
  311. IF_ASSIGN(var, ent, struct ftrace_graph_ret_entry, \
  312. TRACE_GRAPH_RET); \
  313. __ftrace_bad_type(); \
  314. } while (0)
  315. /*
  316. * An option specific to a tracer. This is a boolean value.
  317. * The bit is the bit index that sets its value on the
  318. * flags value in struct tracer_flags.
  319. */
  320. struct tracer_opt {
  321. const char *name; /* Will appear on the trace_options file */
  322. u32 bit; /* Mask assigned in val field in tracer_flags */
  323. };
  324. /*
  325. * The set of specific options for a tracer. Your tracer
  326. * have to set the initial value of the flags val.
  327. */
  328. struct tracer_flags {
  329. u32 val;
  330. struct tracer_opt *opts;
  331. struct tracer *trace;
  332. };
  333. /* Makes more easy to define a tracer opt */
  334. #define TRACER_OPT(s, b) .name = #s, .bit = b
  335. struct trace_option_dentry {
  336. struct tracer_opt *opt;
  337. struct tracer_flags *flags;
  338. struct trace_array *tr;
  339. struct dentry *entry;
  340. };
  341. /**
  342. * struct tracer - a specific tracer and its callbacks to interact with tracefs
  343. * @name: the name chosen to select it on the available_tracers file
  344. * @init: called when one switches to this tracer (echo name > current_tracer)
  345. * @reset: called when one switches to another tracer
  346. * @start: called when tracing is unpaused (echo 1 > tracing_on)
  347. * @stop: called when tracing is paused (echo 0 > tracing_on)
  348. * @update_thresh: called when tracing_thresh is updated
  349. * @open: called when the trace file is opened
  350. * @pipe_open: called when the trace_pipe file is opened
  351. * @close: called when the trace file is released
  352. * @pipe_close: called when the trace_pipe file is released
  353. * @read: override the default read callback on trace_pipe
  354. * @splice_read: override the default splice_read callback on trace_pipe
  355. * @selftest: selftest to run on boot (see trace_selftest.c)
  356. * @print_headers: override the first lines that describe your columns
  357. * @print_line: callback that prints a trace
  358. * @set_flag: signals one of your private flags changed (trace_options file)
  359. * @flags: your private flags
  360. */
  361. struct tracer {
  362. const char *name;
  363. int (*init)(struct trace_array *tr);
  364. void (*reset)(struct trace_array *tr);
  365. void (*start)(struct trace_array *tr);
  366. void (*stop)(struct trace_array *tr);
  367. int (*update_thresh)(struct trace_array *tr);
  368. void (*open)(struct trace_iterator *iter);
  369. void (*pipe_open)(struct trace_iterator *iter);
  370. void (*close)(struct trace_iterator *iter);
  371. void (*pipe_close)(struct trace_iterator *iter);
  372. ssize_t (*read)(struct trace_iterator *iter,
  373. struct file *filp, char __user *ubuf,
  374. size_t cnt, loff_t *ppos);
  375. ssize_t (*splice_read)(struct trace_iterator *iter,
  376. struct file *filp,
  377. loff_t *ppos,
  378. struct pipe_inode_info *pipe,
  379. size_t len,
  380. unsigned int flags);
  381. #ifdef CONFIG_FTRACE_STARTUP_TEST
  382. int (*selftest)(struct tracer *trace,
  383. struct trace_array *tr);
  384. #endif
  385. void (*print_header)(struct seq_file *m);
  386. enum print_line_t (*print_line)(struct trace_iterator *iter);
  387. /* If you handled the flag setting, return 0 */
  388. int (*set_flag)(struct trace_array *tr,
  389. u32 old_flags, u32 bit, int set);
  390. /* Return 0 if OK with change, else return non-zero */
  391. int (*flag_changed)(struct trace_array *tr,
  392. u32 mask, int set);
  393. struct tracer *next;
  394. struct tracer_flags *flags;
  395. int enabled;
  396. int ref;
  397. bool print_max;
  398. bool allow_instances;
  399. #ifdef CONFIG_TRACER_MAX_TRACE
  400. bool use_max_tr;
  401. #endif
  402. /* True if tracer cannot be enabled in kernel param */
  403. bool noboot;
  404. };
  405. /* Only current can touch trace_recursion */
  406. /*
  407. * For function tracing recursion:
  408. * The order of these bits are important.
  409. *
  410. * When function tracing occurs, the following steps are made:
  411. * If arch does not support a ftrace feature:
  412. * call internal function (uses INTERNAL bits) which calls...
  413. * If callback is registered to the "global" list, the list
  414. * function is called and recursion checks the GLOBAL bits.
  415. * then this function calls...
  416. * The function callback, which can use the FTRACE bits to
  417. * check for recursion.
  418. *
  419. * Now if the arch does not suppport a feature, and it calls
  420. * the global list function which calls the ftrace callback
  421. * all three of these steps will do a recursion protection.
  422. * There's no reason to do one if the previous caller already
  423. * did. The recursion that we are protecting against will
  424. * go through the same steps again.
  425. *
  426. * To prevent the multiple recursion checks, if a recursion
  427. * bit is set that is higher than the MAX bit of the current
  428. * check, then we know that the check was made by the previous
  429. * caller, and we can skip the current check.
  430. */
  431. enum {
  432. TRACE_BUFFER_BIT,
  433. TRACE_BUFFER_NMI_BIT,
  434. TRACE_BUFFER_IRQ_BIT,
  435. TRACE_BUFFER_SIRQ_BIT,
  436. /* Start of function recursion bits */
  437. TRACE_FTRACE_BIT,
  438. TRACE_FTRACE_NMI_BIT,
  439. TRACE_FTRACE_IRQ_BIT,
  440. TRACE_FTRACE_SIRQ_BIT,
  441. /* INTERNAL_BITs must be greater than FTRACE_BITs */
  442. TRACE_INTERNAL_BIT,
  443. TRACE_INTERNAL_NMI_BIT,
  444. TRACE_INTERNAL_IRQ_BIT,
  445. TRACE_INTERNAL_SIRQ_BIT,
  446. TRACE_BRANCH_BIT,
  447. /*
  448. * Abuse of the trace_recursion.
  449. * As we need a way to maintain state if we are tracing the function
  450. * graph in irq because we want to trace a particular function that
  451. * was called in irq context but we have irq tracing off. Since this
  452. * can only be modified by current, we can reuse trace_recursion.
  453. */
  454. TRACE_IRQ_BIT,
  455. /* Set if the function is in the set_graph_function file */
  456. TRACE_GRAPH_BIT,
  457. /*
  458. * In the very unlikely case that an interrupt came in
  459. * at a start of graph tracing, and we want to trace
  460. * the function in that interrupt, the depth can be greater
  461. * than zero, because of the preempted start of a previous
  462. * trace. In an even more unlikely case, depth could be 2
  463. * if a softirq interrupted the start of graph tracing,
  464. * followed by an interrupt preempting a start of graph
  465. * tracing in the softirq, and depth can even be 3
  466. * if an NMI came in at the start of an interrupt function
  467. * that preempted a softirq start of a function that
  468. * preempted normal context!!!! Luckily, it can't be
  469. * greater than 3, so the next two bits are a mask
  470. * of what the depth is when we set TRACE_GRAPH_BIT
  471. */
  472. TRACE_GRAPH_DEPTH_START_BIT,
  473. TRACE_GRAPH_DEPTH_END_BIT,
  474. /*
  475. * When transitioning between context, the preempt_count() may
  476. * not be correct. Allow for a single recursion to cover this case.
  477. */
  478. TRACE_TRANSITION_BIT,
  479. };
  480. #define trace_recursion_set(bit) do { (current)->trace_recursion |= (1<<(bit)); } while (0)
  481. #define trace_recursion_clear(bit) do { (current)->trace_recursion &= ~(1<<(bit)); } while (0)
  482. #define trace_recursion_test(bit) ((current)->trace_recursion & (1<<(bit)))
  483. #define trace_recursion_depth() \
  484. (((current)->trace_recursion >> TRACE_GRAPH_DEPTH_START_BIT) & 3)
  485. #define trace_recursion_set_depth(depth) \
  486. do { \
  487. current->trace_recursion &= \
  488. ~(3 << TRACE_GRAPH_DEPTH_START_BIT); \
  489. current->trace_recursion |= \
  490. ((depth) & 3) << TRACE_GRAPH_DEPTH_START_BIT; \
  491. } while (0)
  492. #define TRACE_CONTEXT_BITS 4
  493. #define TRACE_FTRACE_START TRACE_FTRACE_BIT
  494. #define TRACE_FTRACE_MAX ((1 << (TRACE_FTRACE_START + TRACE_CONTEXT_BITS)) - 1)
  495. #define TRACE_LIST_START TRACE_INTERNAL_BIT
  496. #define TRACE_LIST_MAX ((1 << (TRACE_LIST_START + TRACE_CONTEXT_BITS)) - 1)
  497. #define TRACE_CONTEXT_MASK TRACE_LIST_MAX
  498. static __always_inline int trace_get_context_bit(void)
  499. {
  500. int bit;
  501. if (in_interrupt()) {
  502. if (in_nmi())
  503. bit = 0;
  504. else if (in_irq())
  505. bit = 1;
  506. else
  507. bit = 2;
  508. } else
  509. bit = 3;
  510. return bit;
  511. }
  512. static __always_inline int trace_test_and_set_recursion(int start, int max)
  513. {
  514. unsigned int val = current->trace_recursion;
  515. int bit;
  516. /* A previous recursion check was made */
  517. if ((val & TRACE_CONTEXT_MASK) > max)
  518. return 0;
  519. bit = trace_get_context_bit() + start;
  520. if (unlikely(val & (1 << bit))) {
  521. /*
  522. * It could be that preempt_count has not been updated during
  523. * a switch between contexts. Allow for a single recursion.
  524. */
  525. bit = TRACE_TRANSITION_BIT;
  526. if (trace_recursion_test(bit))
  527. return -1;
  528. trace_recursion_set(bit);
  529. barrier();
  530. return bit + 1;
  531. }
  532. /* Normal check passed, clear the transition to allow it again */
  533. trace_recursion_clear(TRACE_TRANSITION_BIT);
  534. val |= 1 << bit;
  535. current->trace_recursion = val;
  536. barrier();
  537. return bit + 1;
  538. }
  539. static __always_inline void trace_clear_recursion(int bit)
  540. {
  541. unsigned int val = current->trace_recursion;
  542. if (!bit)
  543. return;
  544. bit--;
  545. bit = 1 << bit;
  546. val &= ~bit;
  547. barrier();
  548. current->trace_recursion = val;
  549. }
  550. static inline struct ring_buffer_iter *
  551. trace_buffer_iter(struct trace_iterator *iter, int cpu)
  552. {
  553. return iter->buffer_iter ? iter->buffer_iter[cpu] : NULL;
  554. }
  555. int tracer_init(struct tracer *t, struct trace_array *tr);
  556. int tracing_is_enabled(void);
  557. void tracing_reset(struct trace_buffer *buf, int cpu);
  558. void tracing_reset_online_cpus(struct trace_buffer *buf);
  559. void tracing_reset_current(int cpu);
  560. void tracing_reset_all_online_cpus(void);
  561. int tracing_open_generic(struct inode *inode, struct file *filp);
  562. bool tracing_is_disabled(void);
  563. bool tracer_tracing_is_on(struct trace_array *tr);
  564. void tracer_tracing_on(struct trace_array *tr);
  565. void tracer_tracing_off(struct trace_array *tr);
  566. struct dentry *trace_create_file(const char *name,
  567. umode_t mode,
  568. struct dentry *parent,
  569. void *data,
  570. const struct file_operations *fops);
  571. struct dentry *tracing_init_dentry(void);
  572. struct ring_buffer_event;
  573. struct ring_buffer_event *
  574. trace_buffer_lock_reserve(struct ring_buffer *buffer,
  575. int type,
  576. unsigned long len,
  577. unsigned long flags,
  578. int pc);
  579. struct trace_entry *tracing_get_trace_entry(struct trace_array *tr,
  580. struct trace_array_cpu *data);
  581. struct trace_entry *trace_find_next_entry(struct trace_iterator *iter,
  582. int *ent_cpu, u64 *ent_ts);
  583. void trace_buffer_unlock_commit_nostack(struct ring_buffer *buffer,
  584. struct ring_buffer_event *event);
  585. int trace_empty(struct trace_iterator *iter);
  586. void *trace_find_next_entry_inc(struct trace_iterator *iter);
  587. void trace_init_global_iter(struct trace_iterator *iter);
  588. void tracing_iter_reset(struct trace_iterator *iter, int cpu);
  589. void trace_function(struct trace_array *tr,
  590. unsigned long ip,
  591. unsigned long parent_ip,
  592. unsigned long flags, int pc);
  593. void trace_graph_function(struct trace_array *tr,
  594. unsigned long ip,
  595. unsigned long parent_ip,
  596. unsigned long flags, int pc);
  597. void trace_latency_header(struct seq_file *m);
  598. void trace_default_header(struct seq_file *m);
  599. void print_trace_header(struct seq_file *m, struct trace_iterator *iter);
  600. int trace_empty(struct trace_iterator *iter);
  601. void trace_graph_return(struct ftrace_graph_ret *trace);
  602. int trace_graph_entry(struct ftrace_graph_ent *trace);
  603. void set_graph_array(struct trace_array *tr);
  604. void tracing_start_cmdline_record(void);
  605. void tracing_stop_cmdline_record(void);
  606. void tracing_start_tgid_record(void);
  607. void tracing_stop_tgid_record(void);
  608. int register_tracer(struct tracer *type);
  609. int is_tracing_stopped(void);
  610. loff_t tracing_lseek(struct file *file, loff_t offset, int whence);
  611. extern cpumask_var_t __read_mostly tracing_buffer_mask;
  612. #define for_each_tracing_cpu(cpu) \
  613. for_each_cpu(cpu, tracing_buffer_mask)
  614. extern unsigned long nsecs_to_usecs(unsigned long nsecs);
  615. extern unsigned long tracing_thresh;
  616. /* PID filtering */
  617. extern int pid_max;
  618. bool trace_find_filtered_pid(struct trace_pid_list *filtered_pids,
  619. pid_t search_pid);
  620. bool trace_ignore_this_task(struct trace_pid_list *filtered_pids,
  621. struct task_struct *task);
  622. void trace_filter_add_remove_task(struct trace_pid_list *pid_list,
  623. struct task_struct *self,
  624. struct task_struct *task);
  625. void *trace_pid_next(struct trace_pid_list *pid_list, void *v, loff_t *pos);
  626. void *trace_pid_start(struct trace_pid_list *pid_list, loff_t *pos);
  627. int trace_pid_show(struct seq_file *m, void *v);
  628. void trace_free_pid_list(struct trace_pid_list *pid_list);
  629. int trace_pid_write(struct trace_pid_list *filtered_pids,
  630. struct trace_pid_list **new_pid_list,
  631. const char __user *ubuf, size_t cnt);
  632. #ifdef CONFIG_TRACER_MAX_TRACE
  633. void update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu);
  634. void update_max_tr_single(struct trace_array *tr,
  635. struct task_struct *tsk, int cpu);
  636. #endif /* CONFIG_TRACER_MAX_TRACE */
  637. #ifdef CONFIG_STACKTRACE
  638. void ftrace_trace_userstack(struct trace_array *tr,
  639. struct ring_buffer *buffer, unsigned long flags,
  640. int pc);
  641. void __trace_stack(struct trace_array *tr, unsigned long flags, int skip,
  642. int pc);
  643. #else
  644. static inline void ftrace_trace_userstack(struct trace_array *tr,
  645. struct ring_buffer *buffer,
  646. unsigned long flags, int pc)
  647. {
  648. }
  649. static inline void __trace_stack(struct trace_array *tr, unsigned long flags,
  650. int skip, int pc)
  651. {
  652. }
  653. #endif /* CONFIG_STACKTRACE */
  654. extern u64 ftrace_now(int cpu);
  655. extern void trace_find_cmdline(int pid, char comm[]);
  656. extern int trace_find_tgid(int pid);
  657. extern void trace_event_follow_fork(struct trace_array *tr, bool enable);
  658. #ifdef CONFIG_DYNAMIC_FTRACE
  659. extern unsigned long ftrace_update_tot_cnt;
  660. void ftrace_init_trace_array(struct trace_array *tr);
  661. #else
  662. static inline void ftrace_init_trace_array(struct trace_array *tr) { }
  663. #endif
  664. #define DYN_FTRACE_TEST_NAME trace_selftest_dynamic_test_func
  665. extern int DYN_FTRACE_TEST_NAME(void);
  666. #define DYN_FTRACE_TEST_NAME2 trace_selftest_dynamic_test_func2
  667. extern int DYN_FTRACE_TEST_NAME2(void);
  668. extern bool ring_buffer_expanded;
  669. extern bool tracing_selftest_disabled;
  670. #ifdef CONFIG_FTRACE_STARTUP_TEST
  671. extern int trace_selftest_startup_function(struct tracer *trace,
  672. struct trace_array *tr);
  673. extern int trace_selftest_startup_function_graph(struct tracer *trace,
  674. struct trace_array *tr);
  675. extern int trace_selftest_startup_irqsoff(struct tracer *trace,
  676. struct trace_array *tr);
  677. extern int trace_selftest_startup_preemptoff(struct tracer *trace,
  678. struct trace_array *tr);
  679. extern int trace_selftest_startup_preemptirqsoff(struct tracer *trace,
  680. struct trace_array *tr);
  681. extern int trace_selftest_startup_wakeup(struct tracer *trace,
  682. struct trace_array *tr);
  683. extern int trace_selftest_startup_nop(struct tracer *trace,
  684. struct trace_array *tr);
  685. extern int trace_selftest_startup_branch(struct tracer *trace,
  686. struct trace_array *tr);
  687. /*
  688. * Tracer data references selftest functions that only occur
  689. * on boot up. These can be __init functions. Thus, when selftests
  690. * are enabled, then the tracers need to reference __init functions.
  691. */
  692. #define __tracer_data __refdata
  693. #else
  694. /* Tracers are seldom changed. Optimize when selftests are disabled. */
  695. #define __tracer_data __read_mostly
  696. #endif /* CONFIG_FTRACE_STARTUP_TEST */
  697. extern void *head_page(struct trace_array_cpu *data);
  698. extern unsigned long long ns2usecs(u64 nsec);
  699. extern int
  700. trace_vbprintk(unsigned long ip, const char *fmt, va_list args);
  701. extern int
  702. trace_vprintk(unsigned long ip, const char *fmt, va_list args);
  703. extern int
  704. trace_array_vprintk(struct trace_array *tr,
  705. unsigned long ip, const char *fmt, va_list args);
  706. int trace_array_printk(struct trace_array *tr,
  707. unsigned long ip, const char *fmt, ...);
  708. int trace_array_printk_buf(struct ring_buffer *buffer,
  709. unsigned long ip, const char *fmt, ...);
  710. void trace_printk_seq(struct trace_seq *s);
  711. enum print_line_t print_trace_line(struct trace_iterator *iter);
  712. extern char trace_find_mark(unsigned long long duration);
  713. struct ftrace_hash;
  714. struct ftrace_mod_load {
  715. struct list_head list;
  716. char *func;
  717. char *module;
  718. int enable;
  719. };
  720. enum {
  721. FTRACE_HASH_FL_MOD = (1 << 0),
  722. };
  723. struct ftrace_hash {
  724. unsigned long size_bits;
  725. struct hlist_head *buckets;
  726. unsigned long count;
  727. unsigned long flags;
  728. struct rcu_head rcu;
  729. };
  730. struct ftrace_func_entry *
  731. ftrace_lookup_ip(struct ftrace_hash *hash, unsigned long ip);
  732. static __always_inline bool ftrace_hash_empty(struct ftrace_hash *hash)
  733. {
  734. return !hash || !(hash->count || (hash->flags & FTRACE_HASH_FL_MOD));
  735. }
  736. /* Standard output formatting function used for function return traces */
  737. #ifdef CONFIG_FUNCTION_GRAPH_TRACER
  738. /* Flag options */
  739. #define TRACE_GRAPH_PRINT_OVERRUN 0x1
  740. #define TRACE_GRAPH_PRINT_CPU 0x2
  741. #define TRACE_GRAPH_PRINT_OVERHEAD 0x4
  742. #define TRACE_GRAPH_PRINT_PROC 0x8
  743. #define TRACE_GRAPH_PRINT_DURATION 0x10
  744. #define TRACE_GRAPH_PRINT_ABS_TIME 0x20
  745. #define TRACE_GRAPH_PRINT_IRQS 0x40
  746. #define TRACE_GRAPH_PRINT_TAIL 0x80
  747. #define TRACE_GRAPH_SLEEP_TIME 0x100
  748. #define TRACE_GRAPH_GRAPH_TIME 0x200
  749. #define TRACE_GRAPH_PRINT_FILL_SHIFT 28
  750. #define TRACE_GRAPH_PRINT_FILL_MASK (0x3 << TRACE_GRAPH_PRINT_FILL_SHIFT)
  751. extern void ftrace_graph_sleep_time_control(bool enable);
  752. extern void ftrace_graph_graph_time_control(bool enable);
  753. extern enum print_line_t
  754. print_graph_function_flags(struct trace_iterator *iter, u32 flags);
  755. extern void print_graph_headers_flags(struct seq_file *s, u32 flags);
  756. extern void
  757. trace_print_graph_duration(unsigned long long duration, struct trace_seq *s);
  758. extern void graph_trace_open(struct trace_iterator *iter);
  759. extern void graph_trace_close(struct trace_iterator *iter);
  760. extern int __trace_graph_entry(struct trace_array *tr,
  761. struct ftrace_graph_ent *trace,
  762. unsigned long flags, int pc);
  763. extern void __trace_graph_return(struct trace_array *tr,
  764. struct ftrace_graph_ret *trace,
  765. unsigned long flags, int pc);
  766. #ifdef CONFIG_DYNAMIC_FTRACE
  767. extern struct ftrace_hash __rcu *ftrace_graph_hash;
  768. extern struct ftrace_hash __rcu *ftrace_graph_notrace_hash;
  769. static inline int ftrace_graph_addr(struct ftrace_graph_ent *trace)
  770. {
  771. unsigned long addr = trace->func;
  772. int ret = 0;
  773. struct ftrace_hash *hash;
  774. preempt_disable_notrace();
  775. /*
  776. * Have to open code "rcu_dereference_sched()" because the
  777. * function graph tracer can be called when RCU is not
  778. * "watching".
  779. * Protected with schedule_on_each_cpu(ftrace_sync)
  780. */
  781. hash = rcu_dereference_protected(ftrace_graph_hash, !preemptible());
  782. if (ftrace_hash_empty(hash)) {
  783. ret = 1;
  784. goto out;
  785. }
  786. if (ftrace_lookup_ip(hash, addr)) {
  787. /*
  788. * This needs to be cleared on the return functions
  789. * when the depth is zero.
  790. */
  791. trace_recursion_set(TRACE_GRAPH_BIT);
  792. trace_recursion_set_depth(trace->depth);
  793. /*
  794. * If no irqs are to be traced, but a set_graph_function
  795. * is set, and called by an interrupt handler, we still
  796. * want to trace it.
  797. */
  798. if (in_irq())
  799. trace_recursion_set(TRACE_IRQ_BIT);
  800. else
  801. trace_recursion_clear(TRACE_IRQ_BIT);
  802. ret = 1;
  803. }
  804. out:
  805. preempt_enable_notrace();
  806. return ret;
  807. }
  808. static inline void ftrace_graph_addr_finish(struct ftrace_graph_ret *trace)
  809. {
  810. if (trace_recursion_test(TRACE_GRAPH_BIT) &&
  811. trace->depth == trace_recursion_depth())
  812. trace_recursion_clear(TRACE_GRAPH_BIT);
  813. }
  814. static inline int ftrace_graph_notrace_addr(unsigned long addr)
  815. {
  816. int ret = 0;
  817. struct ftrace_hash *notrace_hash;
  818. preempt_disable_notrace();
  819. /*
  820. * Have to open code "rcu_dereference_sched()" because the
  821. * function graph tracer can be called when RCU is not
  822. * "watching".
  823. * Protected with schedule_on_each_cpu(ftrace_sync)
  824. */
  825. notrace_hash = rcu_dereference_protected(ftrace_graph_notrace_hash,
  826. !preemptible());
  827. if (ftrace_lookup_ip(notrace_hash, addr))
  828. ret = 1;
  829. preempt_enable_notrace();
  830. return ret;
  831. }
  832. #else
  833. static inline int ftrace_graph_addr(struct ftrace_graph_ent *trace)
  834. {
  835. return 1;
  836. }
  837. static inline int ftrace_graph_notrace_addr(unsigned long addr)
  838. {
  839. return 0;
  840. }
  841. static inline void ftrace_graph_addr_finish(struct ftrace_graph_ret *trace)
  842. { }
  843. #endif /* CONFIG_DYNAMIC_FTRACE */
  844. extern unsigned int fgraph_max_depth;
  845. static inline bool ftrace_graph_ignore_func(struct ftrace_graph_ent *trace)
  846. {
  847. /* trace it when it is-nested-in or is a function enabled. */
  848. return !(trace_recursion_test(TRACE_GRAPH_BIT) ||
  849. ftrace_graph_addr(trace)) ||
  850. (trace->depth < 0) ||
  851. (fgraph_max_depth && trace->depth >= fgraph_max_depth);
  852. }
  853. #else /* CONFIG_FUNCTION_GRAPH_TRACER */
  854. static inline enum print_line_t
  855. print_graph_function_flags(struct trace_iterator *iter, u32 flags)
  856. {
  857. return TRACE_TYPE_UNHANDLED;
  858. }
  859. #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
  860. extern struct list_head ftrace_pids;
  861. #ifdef CONFIG_FUNCTION_TRACER
  862. struct ftrace_func_command {
  863. struct list_head list;
  864. char *name;
  865. int (*func)(struct trace_array *tr,
  866. struct ftrace_hash *hash,
  867. char *func, char *cmd,
  868. char *params, int enable);
  869. };
  870. extern bool ftrace_filter_param __initdata;
  871. static inline int ftrace_trace_task(struct trace_array *tr)
  872. {
  873. return !this_cpu_read(tr->trace_buffer.data->ftrace_ignore_pid);
  874. }
  875. extern int ftrace_is_dead(void);
  876. int ftrace_create_function_files(struct trace_array *tr,
  877. struct dentry *parent);
  878. void ftrace_destroy_function_files(struct trace_array *tr);
  879. void ftrace_init_global_array_ops(struct trace_array *tr);
  880. void ftrace_init_array_ops(struct trace_array *tr, ftrace_func_t func);
  881. void ftrace_reset_array_ops(struct trace_array *tr);
  882. void ftrace_init_tracefs(struct trace_array *tr, struct dentry *d_tracer);
  883. void ftrace_init_tracefs_toplevel(struct trace_array *tr,
  884. struct dentry *d_tracer);
  885. void ftrace_clear_pids(struct trace_array *tr);
  886. int init_function_trace(void);
  887. void ftrace_pid_follow_fork(struct trace_array *tr, bool enable);
  888. #else
  889. static inline int ftrace_trace_task(struct trace_array *tr)
  890. {
  891. return 1;
  892. }
  893. static inline int ftrace_is_dead(void) { return 0; }
  894. static inline int
  895. ftrace_create_function_files(struct trace_array *tr,
  896. struct dentry *parent)
  897. {
  898. return 0;
  899. }
  900. static inline void ftrace_destroy_function_files(struct trace_array *tr) { }
  901. static inline __init void
  902. ftrace_init_global_array_ops(struct trace_array *tr) { }
  903. static inline void ftrace_reset_array_ops(struct trace_array *tr) { }
  904. static inline void ftrace_init_tracefs(struct trace_array *tr, struct dentry *d) { }
  905. static inline void ftrace_init_tracefs_toplevel(struct trace_array *tr, struct dentry *d) { }
  906. static inline void ftrace_clear_pids(struct trace_array *tr) { }
  907. static inline int init_function_trace(void) { return 0; }
  908. static inline void ftrace_pid_follow_fork(struct trace_array *tr, bool enable) { }
  909. /* ftace_func_t type is not defined, use macro instead of static inline */
  910. #define ftrace_init_array_ops(tr, func) do { } while (0)
  911. #endif /* CONFIG_FUNCTION_TRACER */
  912. #if defined(CONFIG_FUNCTION_TRACER) && defined(CONFIG_DYNAMIC_FTRACE)
  913. struct ftrace_probe_ops {
  914. void (*func)(unsigned long ip,
  915. unsigned long parent_ip,
  916. struct trace_array *tr,
  917. struct ftrace_probe_ops *ops,
  918. void *data);
  919. int (*init)(struct ftrace_probe_ops *ops,
  920. struct trace_array *tr,
  921. unsigned long ip, void *init_data,
  922. void **data);
  923. void (*free)(struct ftrace_probe_ops *ops,
  924. struct trace_array *tr,
  925. unsigned long ip, void *data);
  926. int (*print)(struct seq_file *m,
  927. unsigned long ip,
  928. struct ftrace_probe_ops *ops,
  929. void *data);
  930. };
  931. struct ftrace_func_mapper;
  932. typedef int (*ftrace_mapper_func)(void *data);
  933. struct ftrace_func_mapper *allocate_ftrace_func_mapper(void);
  934. void **ftrace_func_mapper_find_ip(struct ftrace_func_mapper *mapper,
  935. unsigned long ip);
  936. int ftrace_func_mapper_add_ip(struct ftrace_func_mapper *mapper,
  937. unsigned long ip, void *data);
  938. void *ftrace_func_mapper_remove_ip(struct ftrace_func_mapper *mapper,
  939. unsigned long ip);
  940. void free_ftrace_func_mapper(struct ftrace_func_mapper *mapper,
  941. ftrace_mapper_func free_func);
  942. extern int
  943. register_ftrace_function_probe(char *glob, struct trace_array *tr,
  944. struct ftrace_probe_ops *ops, void *data);
  945. extern int
  946. unregister_ftrace_function_probe_func(char *glob, struct trace_array *tr,
  947. struct ftrace_probe_ops *ops);
  948. extern void clear_ftrace_function_probes(struct trace_array *tr);
  949. int register_ftrace_command(struct ftrace_func_command *cmd);
  950. int unregister_ftrace_command(struct ftrace_func_command *cmd);
  951. void ftrace_create_filter_files(struct ftrace_ops *ops,
  952. struct dentry *parent);
  953. void ftrace_destroy_filter_files(struct ftrace_ops *ops);
  954. #else
  955. struct ftrace_func_command;
  956. static inline __init int register_ftrace_command(struct ftrace_func_command *cmd)
  957. {
  958. return -EINVAL;
  959. }
  960. static inline __init int unregister_ftrace_command(char *cmd_name)
  961. {
  962. return -EINVAL;
  963. }
  964. static inline void clear_ftrace_function_probes(struct trace_array *tr)
  965. {
  966. }
  967. /*
  968. * The ops parameter passed in is usually undefined.
  969. * This must be a macro.
  970. */
  971. #define ftrace_create_filter_files(ops, parent) do { } while (0)
  972. #define ftrace_destroy_filter_files(ops) do { } while (0)
  973. #endif /* CONFIG_FUNCTION_TRACER && CONFIG_DYNAMIC_FTRACE */
  974. bool ftrace_event_is_function(struct trace_event_call *call);
  975. /*
  976. * struct trace_parser - servers for reading the user input separated by spaces
  977. * @cont: set if the input is not complete - no final space char was found
  978. * @buffer: holds the parsed user input
  979. * @idx: user input length
  980. * @size: buffer size
  981. */
  982. struct trace_parser {
  983. bool cont;
  984. char *buffer;
  985. unsigned idx;
  986. unsigned size;
  987. };
  988. static inline bool trace_parser_loaded(struct trace_parser *parser)
  989. {
  990. return (parser->idx != 0);
  991. }
  992. static inline bool trace_parser_cont(struct trace_parser *parser)
  993. {
  994. return parser->cont;
  995. }
  996. static inline void trace_parser_clear(struct trace_parser *parser)
  997. {
  998. parser->cont = false;
  999. parser->idx = 0;
  1000. }
  1001. extern int trace_parser_get_init(struct trace_parser *parser, int size);
  1002. extern void trace_parser_put(struct trace_parser *parser);
  1003. extern int trace_get_user(struct trace_parser *parser, const char __user *ubuf,
  1004. size_t cnt, loff_t *ppos);
  1005. /*
  1006. * Only create function graph options if function graph is configured.
  1007. */
  1008. #ifdef CONFIG_FUNCTION_GRAPH_TRACER
  1009. # define FGRAPH_FLAGS \
  1010. C(DISPLAY_GRAPH, "display-graph"),
  1011. #else
  1012. # define FGRAPH_FLAGS
  1013. #endif
  1014. #ifdef CONFIG_BRANCH_TRACER
  1015. # define BRANCH_FLAGS \
  1016. C(BRANCH, "branch"),
  1017. #else
  1018. # define BRANCH_FLAGS
  1019. #endif
  1020. #ifdef CONFIG_FUNCTION_TRACER
  1021. # define FUNCTION_FLAGS \
  1022. C(FUNCTION, "function-trace"), \
  1023. C(FUNC_FORK, "function-fork"),
  1024. # define FUNCTION_DEFAULT_FLAGS TRACE_ITER_FUNCTION
  1025. #else
  1026. # define FUNCTION_FLAGS
  1027. # define FUNCTION_DEFAULT_FLAGS 0UL
  1028. # define TRACE_ITER_FUNC_FORK 0UL
  1029. #endif
  1030. #ifdef CONFIG_STACKTRACE
  1031. # define STACK_FLAGS \
  1032. C(STACKTRACE, "stacktrace"),
  1033. #else
  1034. # define STACK_FLAGS
  1035. #endif
  1036. /*
  1037. * trace_iterator_flags is an enumeration that defines bit
  1038. * positions into trace_flags that controls the output.
  1039. *
  1040. * NOTE: These bits must match the trace_options array in
  1041. * trace.c (this macro guarantees it).
  1042. */
  1043. #define TRACE_FLAGS \
  1044. C(PRINT_PARENT, "print-parent"), \
  1045. C(SYM_OFFSET, "sym-offset"), \
  1046. C(SYM_ADDR, "sym-addr"), \
  1047. C(VERBOSE, "verbose"), \
  1048. C(RAW, "raw"), \
  1049. C(HEX, "hex"), \
  1050. C(BIN, "bin"), \
  1051. C(BLOCK, "block"), \
  1052. C(PRINTK, "trace_printk"), \
  1053. C(ANNOTATE, "annotate"), \
  1054. C(USERSTACKTRACE, "userstacktrace"), \
  1055. C(SYM_USEROBJ, "sym-userobj"), \
  1056. C(PRINTK_MSGONLY, "printk-msg-only"), \
  1057. C(CONTEXT_INFO, "context-info"), /* Print pid/cpu/time */ \
  1058. C(LATENCY_FMT, "latency-format"), \
  1059. C(RECORD_CMD, "record-cmd"), \
  1060. C(RECORD_TGID, "record-tgid"), \
  1061. C(OVERWRITE, "overwrite"), \
  1062. C(STOP_ON_FREE, "disable_on_free"), \
  1063. C(IRQ_INFO, "irq-info"), \
  1064. C(MARKERS, "markers"), \
  1065. C(EVENT_FORK, "event-fork"), \
  1066. FUNCTION_FLAGS \
  1067. FGRAPH_FLAGS \
  1068. STACK_FLAGS \
  1069. BRANCH_FLAGS
  1070. /*
  1071. * By defining C, we can make TRACE_FLAGS a list of bit names
  1072. * that will define the bits for the flag masks.
  1073. */
  1074. #undef C
  1075. #define C(a, b) TRACE_ITER_##a##_BIT
  1076. enum trace_iterator_bits {
  1077. TRACE_FLAGS
  1078. /* Make sure we don't go more than we have bits for */
  1079. TRACE_ITER_LAST_BIT
  1080. };
  1081. /*
  1082. * By redefining C, we can make TRACE_FLAGS a list of masks that
  1083. * use the bits as defined above.
  1084. */
  1085. #undef C
  1086. #define C(a, b) TRACE_ITER_##a = (1 << TRACE_ITER_##a##_BIT)
  1087. enum trace_iterator_flags { TRACE_FLAGS };
  1088. /*
  1089. * TRACE_ITER_SYM_MASK masks the options in trace_flags that
  1090. * control the output of kernel symbols.
  1091. */
  1092. #define TRACE_ITER_SYM_MASK \
  1093. (TRACE_ITER_PRINT_PARENT|TRACE_ITER_SYM_OFFSET|TRACE_ITER_SYM_ADDR)
  1094. extern struct tracer nop_trace;
  1095. #ifdef CONFIG_BRANCH_TRACER
  1096. extern int enable_branch_tracing(struct trace_array *tr);
  1097. extern void disable_branch_tracing(void);
  1098. static inline int trace_branch_enable(struct trace_array *tr)
  1099. {
  1100. if (tr->trace_flags & TRACE_ITER_BRANCH)
  1101. return enable_branch_tracing(tr);
  1102. return 0;
  1103. }
  1104. static inline void trace_branch_disable(void)
  1105. {
  1106. /* due to races, always disable */
  1107. disable_branch_tracing();
  1108. }
  1109. #else
  1110. static inline int trace_branch_enable(struct trace_array *tr)
  1111. {
  1112. return 0;
  1113. }
  1114. static inline void trace_branch_disable(void)
  1115. {
  1116. }
  1117. #endif /* CONFIG_BRANCH_TRACER */
  1118. /* set ring buffers to default size if not already done so */
  1119. int tracing_update_buffers(void);
  1120. struct ftrace_event_field {
  1121. struct list_head link;
  1122. const char *name;
  1123. const char *type;
  1124. int filter_type;
  1125. int offset;
  1126. int size;
  1127. int is_signed;
  1128. };
  1129. struct prog_entry;
  1130. struct event_filter {
  1131. struct prog_entry __rcu *prog;
  1132. char *filter_string;
  1133. };
  1134. struct event_subsystem {
  1135. struct list_head list;
  1136. const char *name;
  1137. struct event_filter *filter;
  1138. int ref_count;
  1139. };
  1140. struct trace_subsystem_dir {
  1141. struct list_head list;
  1142. struct event_subsystem *subsystem;
  1143. struct trace_array *tr;
  1144. struct dentry *entry;
  1145. int ref_count;
  1146. int nr_events;
  1147. };
  1148. extern int call_filter_check_discard(struct trace_event_call *call, void *rec,
  1149. struct ring_buffer *buffer,
  1150. struct ring_buffer_event *event);
  1151. void trace_buffer_unlock_commit_regs(struct trace_array *tr,
  1152. struct ring_buffer *buffer,
  1153. struct ring_buffer_event *event,
  1154. unsigned long flags, int pc,
  1155. struct pt_regs *regs);
  1156. static inline void trace_buffer_unlock_commit(struct trace_array *tr,
  1157. struct ring_buffer *buffer,
  1158. struct ring_buffer_event *event,
  1159. unsigned long flags, int pc)
  1160. {
  1161. trace_buffer_unlock_commit_regs(tr, buffer, event, flags, pc, NULL);
  1162. }
  1163. DECLARE_PER_CPU(struct ring_buffer_event *, trace_buffered_event);
  1164. DECLARE_PER_CPU(int, trace_buffered_event_cnt);
  1165. void trace_buffered_event_disable(void);
  1166. void trace_buffered_event_enable(void);
  1167. static inline void
  1168. __trace_event_discard_commit(struct ring_buffer *buffer,
  1169. struct ring_buffer_event *event)
  1170. {
  1171. if (this_cpu_read(trace_buffered_event) == event) {
  1172. /* Simply release the temp buffer */
  1173. this_cpu_dec(trace_buffered_event_cnt);
  1174. return;
  1175. }
  1176. ring_buffer_discard_commit(buffer, event);
  1177. }
  1178. /*
  1179. * Helper function for event_trigger_unlock_commit{_regs}().
  1180. * If there are event triggers attached to this event that requires
  1181. * filtering against its fields, then they wil be called as the
  1182. * entry already holds the field information of the current event.
  1183. *
  1184. * It also checks if the event should be discarded or not.
  1185. * It is to be discarded if the event is soft disabled and the
  1186. * event was only recorded to process triggers, or if the event
  1187. * filter is active and this event did not match the filters.
  1188. *
  1189. * Returns true if the event is discarded, false otherwise.
  1190. */
  1191. static inline bool
  1192. __event_trigger_test_discard(struct trace_event_file *file,
  1193. struct ring_buffer *buffer,
  1194. struct ring_buffer_event *event,
  1195. void *entry,
  1196. enum event_trigger_type *tt)
  1197. {
  1198. unsigned long eflags = file->flags;
  1199. if (eflags & EVENT_FILE_FL_TRIGGER_COND)
  1200. *tt = event_triggers_call(file, entry, event);
  1201. if (test_bit(EVENT_FILE_FL_SOFT_DISABLED_BIT, &file->flags) ||
  1202. (unlikely(file->flags & EVENT_FILE_FL_FILTERED) &&
  1203. !filter_match_preds(file->filter, entry))) {
  1204. __trace_event_discard_commit(buffer, event);
  1205. return true;
  1206. }
  1207. return false;
  1208. }
  1209. /**
  1210. * event_trigger_unlock_commit - handle triggers and finish event commit
  1211. * @file: The file pointer assoctiated to the event
  1212. * @buffer: The ring buffer that the event is being written to
  1213. * @event: The event meta data in the ring buffer
  1214. * @entry: The event itself
  1215. * @irq_flags: The state of the interrupts at the start of the event
  1216. * @pc: The state of the preempt count at the start of the event.
  1217. *
  1218. * This is a helper function to handle triggers that require data
  1219. * from the event itself. It also tests the event against filters and
  1220. * if the event is soft disabled and should be discarded.
  1221. */
  1222. static inline void
  1223. event_trigger_unlock_commit(struct trace_event_file *file,
  1224. struct ring_buffer *buffer,
  1225. struct ring_buffer_event *event,
  1226. void *entry, unsigned long irq_flags, int pc)
  1227. {
  1228. enum event_trigger_type tt = ETT_NONE;
  1229. if (!__event_trigger_test_discard(file, buffer, event, entry, &tt))
  1230. trace_buffer_unlock_commit(file->tr, buffer, event, irq_flags, pc);
  1231. if (tt)
  1232. event_triggers_post_call(file, tt);
  1233. }
  1234. /**
  1235. * event_trigger_unlock_commit_regs - handle triggers and finish event commit
  1236. * @file: The file pointer assoctiated to the event
  1237. * @buffer: The ring buffer that the event is being written to
  1238. * @event: The event meta data in the ring buffer
  1239. * @entry: The event itself
  1240. * @irq_flags: The state of the interrupts at the start of the event
  1241. * @pc: The state of the preempt count at the start of the event.
  1242. *
  1243. * This is a helper function to handle triggers that require data
  1244. * from the event itself. It also tests the event against filters and
  1245. * if the event is soft disabled and should be discarded.
  1246. *
  1247. * Same as event_trigger_unlock_commit() but calls
  1248. * trace_buffer_unlock_commit_regs() instead of trace_buffer_unlock_commit().
  1249. */
  1250. static inline void
  1251. event_trigger_unlock_commit_regs(struct trace_event_file *file,
  1252. struct ring_buffer *buffer,
  1253. struct ring_buffer_event *event,
  1254. void *entry, unsigned long irq_flags, int pc,
  1255. struct pt_regs *regs)
  1256. {
  1257. enum event_trigger_type tt = ETT_NONE;
  1258. if (!__event_trigger_test_discard(file, buffer, event, entry, &tt))
  1259. trace_buffer_unlock_commit_regs(file->tr, buffer, event,
  1260. irq_flags, pc, regs);
  1261. if (tt)
  1262. event_triggers_post_call(file, tt);
  1263. }
  1264. #define FILTER_PRED_INVALID ((unsigned short)-1)
  1265. #define FILTER_PRED_IS_RIGHT (1 << 15)
  1266. #define FILTER_PRED_FOLD (1 << 15)
  1267. /*
  1268. * The max preds is the size of unsigned short with
  1269. * two flags at the MSBs. One bit is used for both the IS_RIGHT
  1270. * and FOLD flags. The other is reserved.
  1271. *
  1272. * 2^14 preds is way more than enough.
  1273. */
  1274. #define MAX_FILTER_PRED 16384
  1275. struct filter_pred;
  1276. struct regex;
  1277. typedef int (*filter_pred_fn_t) (struct filter_pred *pred, void *event);
  1278. typedef int (*regex_match_func)(char *str, struct regex *r, int len);
  1279. enum regex_type {
  1280. MATCH_FULL = 0,
  1281. MATCH_FRONT_ONLY,
  1282. MATCH_MIDDLE_ONLY,
  1283. MATCH_END_ONLY,
  1284. MATCH_GLOB,
  1285. };
  1286. struct regex {
  1287. char pattern[MAX_FILTER_STR_VAL];
  1288. int len;
  1289. int field_len;
  1290. regex_match_func match;
  1291. };
  1292. struct filter_pred {
  1293. filter_pred_fn_t fn;
  1294. u64 val;
  1295. struct regex regex;
  1296. unsigned short *ops;
  1297. struct ftrace_event_field *field;
  1298. int offset;
  1299. int not;
  1300. int op;
  1301. };
  1302. static inline bool is_string_field(struct ftrace_event_field *field)
  1303. {
  1304. return field->filter_type == FILTER_DYN_STRING ||
  1305. field->filter_type == FILTER_STATIC_STRING ||
  1306. field->filter_type == FILTER_PTR_STRING ||
  1307. field->filter_type == FILTER_COMM;
  1308. }
  1309. static inline bool is_function_field(struct ftrace_event_field *field)
  1310. {
  1311. return field->filter_type == FILTER_TRACE_FN;
  1312. }
  1313. extern enum regex_type
  1314. filter_parse_regex(char *buff, int len, char **search, int *not);
  1315. extern void print_event_filter(struct trace_event_file *file,
  1316. struct trace_seq *s);
  1317. extern int apply_event_filter(struct trace_event_file *file,
  1318. char *filter_string);
  1319. extern int apply_subsystem_event_filter(struct trace_subsystem_dir *dir,
  1320. char *filter_string);
  1321. extern void print_subsystem_event_filter(struct event_subsystem *system,
  1322. struct trace_seq *s);
  1323. extern int filter_assign_type(const char *type);
  1324. extern int create_event_filter(struct trace_event_call *call,
  1325. char *filter_str, bool set_str,
  1326. struct event_filter **filterp);
  1327. extern void free_event_filter(struct event_filter *filter);
  1328. struct ftrace_event_field *
  1329. trace_find_event_field(struct trace_event_call *call, char *name);
  1330. extern void trace_event_enable_cmd_record(bool enable);
  1331. extern void trace_event_enable_tgid_record(bool enable);
  1332. extern int event_trace_init(void);
  1333. extern int event_trace_add_tracer(struct dentry *parent, struct trace_array *tr);
  1334. extern int event_trace_del_tracer(struct trace_array *tr);
  1335. extern struct trace_event_file *__find_event_file(struct trace_array *tr,
  1336. const char *system,
  1337. const char *event);
  1338. extern struct trace_event_file *find_event_file(struct trace_array *tr,
  1339. const char *system,
  1340. const char *event);
  1341. static inline void *event_file_data(struct file *filp)
  1342. {
  1343. return READ_ONCE(file_inode(filp)->i_private);
  1344. }
  1345. extern struct mutex event_mutex;
  1346. extern struct list_head ftrace_events;
  1347. extern const struct file_operations event_trigger_fops;
  1348. extern const struct file_operations event_hist_fops;
  1349. #ifdef CONFIG_HIST_TRIGGERS
  1350. extern int register_trigger_hist_cmd(void);
  1351. extern int register_trigger_hist_enable_disable_cmds(void);
  1352. #else
  1353. static inline int register_trigger_hist_cmd(void) { return 0; }
  1354. static inline int register_trigger_hist_enable_disable_cmds(void) { return 0; }
  1355. #endif
  1356. extern int register_trigger_cmds(void);
  1357. extern void clear_event_triggers(struct trace_array *tr);
  1358. struct event_trigger_data {
  1359. unsigned long count;
  1360. int ref;
  1361. struct event_trigger_ops *ops;
  1362. struct event_command *cmd_ops;
  1363. struct event_filter __rcu *filter;
  1364. char *filter_str;
  1365. void *private_data;
  1366. bool paused;
  1367. bool paused_tmp;
  1368. struct list_head list;
  1369. char *name;
  1370. struct list_head named_list;
  1371. struct event_trigger_data *named_data;
  1372. };
  1373. /* Avoid typos */
  1374. #define ENABLE_EVENT_STR "enable_event"
  1375. #define DISABLE_EVENT_STR "disable_event"
  1376. #define ENABLE_HIST_STR "enable_hist"
  1377. #define DISABLE_HIST_STR "disable_hist"
  1378. struct enable_trigger_data {
  1379. struct trace_event_file *file;
  1380. bool enable;
  1381. bool hist;
  1382. };
  1383. extern int event_enable_trigger_print(struct seq_file *m,
  1384. struct event_trigger_ops *ops,
  1385. struct event_trigger_data *data);
  1386. extern void event_enable_trigger_free(struct event_trigger_ops *ops,
  1387. struct event_trigger_data *data);
  1388. extern int event_enable_trigger_func(struct event_command *cmd_ops,
  1389. struct trace_event_file *file,
  1390. char *glob, char *cmd, char *param);
  1391. extern int event_enable_register_trigger(char *glob,
  1392. struct event_trigger_ops *ops,
  1393. struct event_trigger_data *data,
  1394. struct trace_event_file *file);
  1395. extern void event_enable_unregister_trigger(char *glob,
  1396. struct event_trigger_ops *ops,
  1397. struct event_trigger_data *test,
  1398. struct trace_event_file *file);
  1399. extern void trigger_data_free(struct event_trigger_data *data);
  1400. extern int event_trigger_init(struct event_trigger_ops *ops,
  1401. struct event_trigger_data *data);
  1402. extern int trace_event_trigger_enable_disable(struct trace_event_file *file,
  1403. int trigger_enable);
  1404. extern void update_cond_flag(struct trace_event_file *file);
  1405. extern int set_trigger_filter(char *filter_str,
  1406. struct event_trigger_data *trigger_data,
  1407. struct trace_event_file *file);
  1408. extern struct event_trigger_data *find_named_trigger(const char *name);
  1409. extern bool is_named_trigger(struct event_trigger_data *test);
  1410. extern int save_named_trigger(const char *name,
  1411. struct event_trigger_data *data);
  1412. extern void del_named_trigger(struct event_trigger_data *data);
  1413. extern void pause_named_trigger(struct event_trigger_data *data);
  1414. extern void unpause_named_trigger(struct event_trigger_data *data);
  1415. extern void set_named_trigger_data(struct event_trigger_data *data,
  1416. struct event_trigger_data *named_data);
  1417. extern struct event_trigger_data *
  1418. get_named_trigger_data(struct event_trigger_data *data);
  1419. extern int register_event_command(struct event_command *cmd);
  1420. extern int unregister_event_command(struct event_command *cmd);
  1421. extern int register_trigger_hist_enable_disable_cmds(void);
  1422. /**
  1423. * struct event_trigger_ops - callbacks for trace event triggers
  1424. *
  1425. * The methods in this structure provide per-event trigger hooks for
  1426. * various trigger operations.
  1427. *
  1428. * All the methods below, except for @init() and @free(), must be
  1429. * implemented.
  1430. *
  1431. * @func: The trigger 'probe' function called when the triggering
  1432. * event occurs. The data passed into this callback is the data
  1433. * that was supplied to the event_command @reg() function that
  1434. * registered the trigger (see struct event_command) along with
  1435. * the trace record, rec.
  1436. *
  1437. * @init: An optional initialization function called for the trigger
  1438. * when the trigger is registered (via the event_command reg()
  1439. * function). This can be used to perform per-trigger
  1440. * initialization such as incrementing a per-trigger reference
  1441. * count, for instance. This is usually implemented by the
  1442. * generic utility function @event_trigger_init() (see
  1443. * trace_event_triggers.c).
  1444. *
  1445. * @free: An optional de-initialization function called for the
  1446. * trigger when the trigger is unregistered (via the
  1447. * event_command @reg() function). This can be used to perform
  1448. * per-trigger de-initialization such as decrementing a
  1449. * per-trigger reference count and freeing corresponding trigger
  1450. * data, for instance. This is usually implemented by the
  1451. * generic utility function @event_trigger_free() (see
  1452. * trace_event_triggers.c).
  1453. *
  1454. * @print: The callback function invoked to have the trigger print
  1455. * itself. This is usually implemented by a wrapper function
  1456. * that calls the generic utility function @event_trigger_print()
  1457. * (see trace_event_triggers.c).
  1458. */
  1459. struct event_trigger_ops {
  1460. void (*func)(struct event_trigger_data *data,
  1461. void *rec,
  1462. struct ring_buffer_event *rbe);
  1463. int (*init)(struct event_trigger_ops *ops,
  1464. struct event_trigger_data *data);
  1465. void (*free)(struct event_trigger_ops *ops,
  1466. struct event_trigger_data *data);
  1467. int (*print)(struct seq_file *m,
  1468. struct event_trigger_ops *ops,
  1469. struct event_trigger_data *data);
  1470. };
  1471. /**
  1472. * struct event_command - callbacks and data members for event commands
  1473. *
  1474. * Event commands are invoked by users by writing the command name
  1475. * into the 'trigger' file associated with a trace event. The
  1476. * parameters associated with a specific invocation of an event
  1477. * command are used to create an event trigger instance, which is
  1478. * added to the list of trigger instances associated with that trace
  1479. * event. When the event is hit, the set of triggers associated with
  1480. * that event is invoked.
  1481. *
  1482. * The data members in this structure provide per-event command data
  1483. * for various event commands.
  1484. *
  1485. * All the data members below, except for @post_trigger, must be set
  1486. * for each event command.
  1487. *
  1488. * @name: The unique name that identifies the event command. This is
  1489. * the name used when setting triggers via trigger files.
  1490. *
  1491. * @trigger_type: A unique id that identifies the event command
  1492. * 'type'. This value has two purposes, the first to ensure that
  1493. * only one trigger of the same type can be set at a given time
  1494. * for a particular event e.g. it doesn't make sense to have both
  1495. * a traceon and traceoff trigger attached to a single event at
  1496. * the same time, so traceon and traceoff have the same type
  1497. * though they have different names. The @trigger_type value is
  1498. * also used as a bit value for deferring the actual trigger
  1499. * action until after the current event is finished. Some
  1500. * commands need to do this if they themselves log to the trace
  1501. * buffer (see the @post_trigger() member below). @trigger_type
  1502. * values are defined by adding new values to the trigger_type
  1503. * enum in include/linux/trace_events.h.
  1504. *
  1505. * @flags: See the enum event_command_flags below.
  1506. *
  1507. * All the methods below, except for @set_filter() and @unreg_all(),
  1508. * must be implemented.
  1509. *
  1510. * @func: The callback function responsible for parsing and
  1511. * registering the trigger written to the 'trigger' file by the
  1512. * user. It allocates the trigger instance and registers it with
  1513. * the appropriate trace event. It makes use of the other
  1514. * event_command callback functions to orchestrate this, and is
  1515. * usually implemented by the generic utility function
  1516. * @event_trigger_callback() (see trace_event_triggers.c).
  1517. *
  1518. * @reg: Adds the trigger to the list of triggers associated with the
  1519. * event, and enables the event trigger itself, after
  1520. * initializing it (via the event_trigger_ops @init() function).
  1521. * This is also where commands can use the @trigger_type value to
  1522. * make the decision as to whether or not multiple instances of
  1523. * the trigger should be allowed. This is usually implemented by
  1524. * the generic utility function @register_trigger() (see
  1525. * trace_event_triggers.c).
  1526. *
  1527. * @unreg: Removes the trigger from the list of triggers associated
  1528. * with the event, and disables the event trigger itself, after
  1529. * initializing it (via the event_trigger_ops @free() function).
  1530. * This is usually implemented by the generic utility function
  1531. * @unregister_trigger() (see trace_event_triggers.c).
  1532. *
  1533. * @unreg_all: An optional function called to remove all the triggers
  1534. * from the list of triggers associated with the event. Called
  1535. * when a trigger file is opened in truncate mode.
  1536. *
  1537. * @set_filter: An optional function called to parse and set a filter
  1538. * for the trigger. If no @set_filter() method is set for the
  1539. * event command, filters set by the user for the command will be
  1540. * ignored. This is usually implemented by the generic utility
  1541. * function @set_trigger_filter() (see trace_event_triggers.c).
  1542. *
  1543. * @get_trigger_ops: The callback function invoked to retrieve the
  1544. * event_trigger_ops implementation associated with the command.
  1545. */
  1546. struct event_command {
  1547. struct list_head list;
  1548. char *name;
  1549. enum event_trigger_type trigger_type;
  1550. int flags;
  1551. int (*func)(struct event_command *cmd_ops,
  1552. struct trace_event_file *file,
  1553. char *glob, char *cmd, char *params);
  1554. int (*reg)(char *glob,
  1555. struct event_trigger_ops *ops,
  1556. struct event_trigger_data *data,
  1557. struct trace_event_file *file);
  1558. void (*unreg)(char *glob,
  1559. struct event_trigger_ops *ops,
  1560. struct event_trigger_data *data,
  1561. struct trace_event_file *file);
  1562. void (*unreg_all)(struct trace_event_file *file);
  1563. int (*set_filter)(char *filter_str,
  1564. struct event_trigger_data *data,
  1565. struct trace_event_file *file);
  1566. struct event_trigger_ops *(*get_trigger_ops)(char *cmd, char *param);
  1567. };
  1568. /**
  1569. * enum event_command_flags - flags for struct event_command
  1570. *
  1571. * @POST_TRIGGER: A flag that says whether or not this command needs
  1572. * to have its action delayed until after the current event has
  1573. * been closed. Some triggers need to avoid being invoked while
  1574. * an event is currently in the process of being logged, since
  1575. * the trigger may itself log data into the trace buffer. Thus
  1576. * we make sure the current event is committed before invoking
  1577. * those triggers. To do that, the trigger invocation is split
  1578. * in two - the first part checks the filter using the current
  1579. * trace record; if a command has the @post_trigger flag set, it
  1580. * sets a bit for itself in the return value, otherwise it
  1581. * directly invokes the trigger. Once all commands have been
  1582. * either invoked or set their return flag, the current record is
  1583. * either committed or discarded. At that point, if any commands
  1584. * have deferred their triggers, those commands are finally
  1585. * invoked following the close of the current event. In other
  1586. * words, if the event_trigger_ops @func() probe implementation
  1587. * itself logs to the trace buffer, this flag should be set,
  1588. * otherwise it can be left unspecified.
  1589. *
  1590. * @NEEDS_REC: A flag that says whether or not this command needs
  1591. * access to the trace record in order to perform its function,
  1592. * regardless of whether or not it has a filter associated with
  1593. * it (filters make a trigger require access to the trace record
  1594. * but are not always present).
  1595. */
  1596. enum event_command_flags {
  1597. EVENT_CMD_FL_POST_TRIGGER = 1,
  1598. EVENT_CMD_FL_NEEDS_REC = 2,
  1599. };
  1600. static inline bool event_command_post_trigger(struct event_command *cmd_ops)
  1601. {
  1602. return cmd_ops->flags & EVENT_CMD_FL_POST_TRIGGER;
  1603. }
  1604. static inline bool event_command_needs_rec(struct event_command *cmd_ops)
  1605. {
  1606. return cmd_ops->flags & EVENT_CMD_FL_NEEDS_REC;
  1607. }
  1608. extern int trace_event_enable_disable(struct trace_event_file *file,
  1609. int enable, int soft_disable);
  1610. extern int tracing_alloc_snapshot(void);
  1611. extern const char *__start___trace_bprintk_fmt[];
  1612. extern const char *__stop___trace_bprintk_fmt[];
  1613. extern const char *__start___tracepoint_str[];
  1614. extern const char *__stop___tracepoint_str[];
  1615. void trace_printk_control(bool enabled);
  1616. void trace_printk_init_buffers(void);
  1617. void trace_printk_start_comm(void);
  1618. int trace_keep_overwrite(struct tracer *tracer, u32 mask, int set);
  1619. int set_tracer_flag(struct trace_array *tr, unsigned int mask, int enabled);
  1620. #define MAX_EVENT_NAME_LEN 64
  1621. extern int trace_run_command(const char *buf, int (*createfn)(int, char**));
  1622. extern ssize_t trace_parse_run_command(struct file *file,
  1623. const char __user *buffer, size_t count, loff_t *ppos,
  1624. int (*createfn)(int, char**));
  1625. /*
  1626. * Normal trace_printk() and friends allocates special buffers
  1627. * to do the manipulation, as well as saves the print formats
  1628. * into sections to display. But the trace infrastructure wants
  1629. * to use these without the added overhead at the price of being
  1630. * a bit slower (used mainly for warnings, where we don't care
  1631. * about performance). The internal_trace_puts() is for such
  1632. * a purpose.
  1633. */
  1634. #define internal_trace_puts(str) __trace_puts(_THIS_IP_, str, strlen(str))
  1635. #undef FTRACE_ENTRY
  1636. #define FTRACE_ENTRY(call, struct_name, id, tstruct, print, filter) \
  1637. extern struct trace_event_call \
  1638. __aligned(4) event_##call;
  1639. #undef FTRACE_ENTRY_DUP
  1640. #define FTRACE_ENTRY_DUP(call, struct_name, id, tstruct, print, filter) \
  1641. FTRACE_ENTRY(call, struct_name, id, PARAMS(tstruct), PARAMS(print), \
  1642. filter)
  1643. #undef FTRACE_ENTRY_PACKED
  1644. #define FTRACE_ENTRY_PACKED(call, struct_name, id, tstruct, print, filter) \
  1645. FTRACE_ENTRY(call, struct_name, id, PARAMS(tstruct), PARAMS(print), \
  1646. filter)
  1647. #include "trace_entries.h"
  1648. #if defined(CONFIG_PERF_EVENTS) && defined(CONFIG_FUNCTION_TRACER)
  1649. int perf_ftrace_event_register(struct trace_event_call *call,
  1650. enum trace_reg type, void *data);
  1651. #else
  1652. #define perf_ftrace_event_register NULL
  1653. #endif
  1654. #ifdef CONFIG_FTRACE_SYSCALLS
  1655. void init_ftrace_syscalls(void);
  1656. const char *get_syscall_name(int syscall);
  1657. #else
  1658. static inline void init_ftrace_syscalls(void) { }
  1659. static inline const char *get_syscall_name(int syscall)
  1660. {
  1661. return NULL;
  1662. }
  1663. #endif
  1664. #ifdef CONFIG_EVENT_TRACING
  1665. void trace_event_init(void);
  1666. void trace_event_eval_update(struct trace_eval_map **map, int len);
  1667. #else
  1668. static inline void __init trace_event_init(void) { }
  1669. static inline void trace_event_eval_update(struct trace_eval_map **map, int len) { }
  1670. #endif
  1671. #ifdef CONFIG_TRACER_SNAPSHOT
  1672. void tracing_snapshot_instance(struct trace_array *tr);
  1673. int tracing_alloc_snapshot_instance(struct trace_array *tr);
  1674. #else
  1675. static inline void tracing_snapshot_instance(struct trace_array *tr) { }
  1676. static inline int tracing_alloc_snapshot_instance(struct trace_array *tr)
  1677. {
  1678. return 0;
  1679. }
  1680. #endif
  1681. #ifdef CONFIG_PREEMPT_TRACER
  1682. void tracer_preempt_on(unsigned long a0, unsigned long a1);
  1683. void tracer_preempt_off(unsigned long a0, unsigned long a1);
  1684. #else
  1685. static inline void tracer_preempt_on(unsigned long a0, unsigned long a1) { }
  1686. static inline void tracer_preempt_off(unsigned long a0, unsigned long a1) { }
  1687. #endif
  1688. #ifdef CONFIG_IRQSOFF_TRACER
  1689. void tracer_hardirqs_on(unsigned long a0, unsigned long a1);
  1690. void tracer_hardirqs_off(unsigned long a0, unsigned long a1);
  1691. #else
  1692. static inline void tracer_hardirqs_on(unsigned long a0, unsigned long a1) { }
  1693. static inline void tracer_hardirqs_off(unsigned long a0, unsigned long a1) { }
  1694. #endif
  1695. extern struct trace_iterator *tracepoint_print_iter;
  1696. /*
  1697. * Reset the state of the trace_iterator so that it can read consumed data.
  1698. * Normally, the trace_iterator is used for reading the data when it is not
  1699. * consumed, and must retain state.
  1700. */
  1701. static __always_inline void trace_iterator_reset(struct trace_iterator *iter)
  1702. {
  1703. const size_t offset = offsetof(struct trace_iterator, seq);
  1704. /*
  1705. * Keep gcc from complaining about overwriting more than just one
  1706. * member in the structure.
  1707. */
  1708. memset((char *)iter + offset, 0, sizeof(struct trace_iterator) - offset);
  1709. iter->pos = -1;
  1710. }
  1711. #endif /* _LINUX_KERNEL_TRACE_H */