trace_event_perf.c 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * trace event based perf event profiling/tracing
  4. *
  5. * Copyright (C) 2009 Red Hat Inc, Peter Zijlstra
  6. * Copyright (C) 2009-2010 Frederic Weisbecker <fweisbec@gmail.com>
  7. */
  8. #include <linux/module.h>
  9. #include <linux/kprobes.h>
  10. #include "trace.h"
  11. #include "trace_probe.h"
  12. static char __percpu *perf_trace_buf[PERF_NR_CONTEXTS];
  13. /*
  14. * Force it to be aligned to unsigned long to avoid misaligned accesses
  15. * suprises
  16. */
  17. typedef typeof(unsigned long [PERF_MAX_TRACE_SIZE / sizeof(unsigned long)])
  18. perf_trace_t;
  19. /* Count the events in use (per event id, not per instance) */
  20. static int total_ref_count;
  21. static int perf_trace_event_perm(struct trace_event_call *tp_event,
  22. struct perf_event *p_event)
  23. {
  24. if (tp_event->perf_perm) {
  25. int ret = tp_event->perf_perm(tp_event, p_event);
  26. if (ret)
  27. return ret;
  28. }
  29. /*
  30. * We checked and allowed to create parent,
  31. * allow children without checking.
  32. */
  33. if (p_event->parent)
  34. return 0;
  35. /*
  36. * It's ok to check current process (owner) permissions in here,
  37. * because code below is called only via perf_event_open syscall.
  38. */
  39. /* The ftrace function trace is allowed only for root. */
  40. if (ftrace_event_is_function(tp_event)) {
  41. if (perf_paranoid_tracepoint_raw() && !capable(CAP_SYS_ADMIN))
  42. return -EPERM;
  43. if (!is_sampling_event(p_event))
  44. return 0;
  45. /*
  46. * We don't allow user space callchains for function trace
  47. * event, due to issues with page faults while tracing page
  48. * fault handler and its overall trickiness nature.
  49. */
  50. if (!p_event->attr.exclude_callchain_user)
  51. return -EINVAL;
  52. /*
  53. * Same reason to disable user stack dump as for user space
  54. * callchains above.
  55. */
  56. if (p_event->attr.sample_type & PERF_SAMPLE_STACK_USER)
  57. return -EINVAL;
  58. }
  59. /* No tracing, just counting, so no obvious leak */
  60. if (!(p_event->attr.sample_type & PERF_SAMPLE_RAW))
  61. return 0;
  62. /* Some events are ok to be traced by non-root users... */
  63. if (p_event->attach_state == PERF_ATTACH_TASK) {
  64. if (tp_event->flags & TRACE_EVENT_FL_CAP_ANY)
  65. return 0;
  66. }
  67. /*
  68. * ...otherwise raw tracepoint data can be a severe data leak,
  69. * only allow root to have these.
  70. */
  71. if (perf_paranoid_tracepoint_raw() && !capable(CAP_SYS_ADMIN))
  72. return -EPERM;
  73. return 0;
  74. }
  75. static int perf_trace_event_reg(struct trace_event_call *tp_event,
  76. struct perf_event *p_event)
  77. {
  78. struct hlist_head __percpu *list;
  79. int ret = -ENOMEM;
  80. int cpu;
  81. p_event->tp_event = tp_event;
  82. if (tp_event->perf_refcount++ > 0)
  83. return 0;
  84. list = alloc_percpu(struct hlist_head);
  85. if (!list)
  86. goto fail;
  87. for_each_possible_cpu(cpu)
  88. INIT_HLIST_HEAD(per_cpu_ptr(list, cpu));
  89. tp_event->perf_events = list;
  90. if (!total_ref_count) {
  91. char __percpu *buf;
  92. int i;
  93. for (i = 0; i < PERF_NR_CONTEXTS; i++) {
  94. buf = (char __percpu *)alloc_percpu(perf_trace_t);
  95. if (!buf)
  96. goto fail;
  97. perf_trace_buf[i] = buf;
  98. }
  99. }
  100. ret = tp_event->class->reg(tp_event, TRACE_REG_PERF_REGISTER, NULL);
  101. if (ret)
  102. goto fail;
  103. total_ref_count++;
  104. return 0;
  105. fail:
  106. if (!total_ref_count) {
  107. int i;
  108. for (i = 0; i < PERF_NR_CONTEXTS; i++) {
  109. free_percpu(perf_trace_buf[i]);
  110. perf_trace_buf[i] = NULL;
  111. }
  112. }
  113. if (!--tp_event->perf_refcount) {
  114. free_percpu(tp_event->perf_events);
  115. tp_event->perf_events = NULL;
  116. }
  117. return ret;
  118. }
  119. static void perf_trace_event_unreg(struct perf_event *p_event)
  120. {
  121. struct trace_event_call *tp_event = p_event->tp_event;
  122. int i;
  123. if (--tp_event->perf_refcount > 0)
  124. goto out;
  125. tp_event->class->reg(tp_event, TRACE_REG_PERF_UNREGISTER, NULL);
  126. /*
  127. * Ensure our callback won't be called anymore. The buffers
  128. * will be freed after that.
  129. */
  130. tracepoint_synchronize_unregister();
  131. free_percpu(tp_event->perf_events);
  132. tp_event->perf_events = NULL;
  133. if (!--total_ref_count) {
  134. for (i = 0; i < PERF_NR_CONTEXTS; i++) {
  135. free_percpu(perf_trace_buf[i]);
  136. perf_trace_buf[i] = NULL;
  137. }
  138. }
  139. out:
  140. module_put(tp_event->mod);
  141. }
  142. static int perf_trace_event_open(struct perf_event *p_event)
  143. {
  144. struct trace_event_call *tp_event = p_event->tp_event;
  145. return tp_event->class->reg(tp_event, TRACE_REG_PERF_OPEN, p_event);
  146. }
  147. static void perf_trace_event_close(struct perf_event *p_event)
  148. {
  149. struct trace_event_call *tp_event = p_event->tp_event;
  150. tp_event->class->reg(tp_event, TRACE_REG_PERF_CLOSE, p_event);
  151. }
  152. static int perf_trace_event_init(struct trace_event_call *tp_event,
  153. struct perf_event *p_event)
  154. {
  155. int ret;
  156. ret = perf_trace_event_perm(tp_event, p_event);
  157. if (ret)
  158. return ret;
  159. ret = perf_trace_event_reg(tp_event, p_event);
  160. if (ret)
  161. return ret;
  162. ret = perf_trace_event_open(p_event);
  163. if (ret) {
  164. perf_trace_event_unreg(p_event);
  165. return ret;
  166. }
  167. return 0;
  168. }
  169. int perf_trace_init(struct perf_event *p_event)
  170. {
  171. struct trace_event_call *tp_event;
  172. u64 event_id = p_event->attr.config;
  173. int ret = -EINVAL;
  174. mutex_lock(&event_mutex);
  175. list_for_each_entry(tp_event, &ftrace_events, list) {
  176. if (tp_event->event.type == event_id &&
  177. tp_event->class && tp_event->class->reg &&
  178. try_module_get(tp_event->mod)) {
  179. ret = perf_trace_event_init(tp_event, p_event);
  180. if (ret)
  181. module_put(tp_event->mod);
  182. break;
  183. }
  184. }
  185. mutex_unlock(&event_mutex);
  186. return ret;
  187. }
  188. void perf_trace_destroy(struct perf_event *p_event)
  189. {
  190. mutex_lock(&event_mutex);
  191. perf_trace_event_close(p_event);
  192. perf_trace_event_unreg(p_event);
  193. mutex_unlock(&event_mutex);
  194. }
  195. #ifdef CONFIG_KPROBE_EVENTS
  196. int perf_kprobe_init(struct perf_event *p_event, bool is_retprobe)
  197. {
  198. int ret;
  199. char *func = NULL;
  200. struct trace_event_call *tp_event;
  201. if (p_event->attr.kprobe_func) {
  202. func = kzalloc(KSYM_NAME_LEN, GFP_KERNEL);
  203. if (!func)
  204. return -ENOMEM;
  205. ret = strncpy_from_user(
  206. func, u64_to_user_ptr(p_event->attr.kprobe_func),
  207. KSYM_NAME_LEN);
  208. if (ret == KSYM_NAME_LEN)
  209. ret = -E2BIG;
  210. if (ret < 0)
  211. goto out;
  212. if (func[0] == '\0') {
  213. kfree(func);
  214. func = NULL;
  215. }
  216. }
  217. tp_event = create_local_trace_kprobe(
  218. func, (void *)(unsigned long)(p_event->attr.kprobe_addr),
  219. p_event->attr.probe_offset, is_retprobe);
  220. if (IS_ERR(tp_event)) {
  221. ret = PTR_ERR(tp_event);
  222. goto out;
  223. }
  224. mutex_lock(&event_mutex);
  225. ret = perf_trace_event_init(tp_event, p_event);
  226. if (ret)
  227. destroy_local_trace_kprobe(tp_event);
  228. mutex_unlock(&event_mutex);
  229. out:
  230. kfree(func);
  231. return ret;
  232. }
  233. void perf_kprobe_destroy(struct perf_event *p_event)
  234. {
  235. mutex_lock(&event_mutex);
  236. perf_trace_event_close(p_event);
  237. perf_trace_event_unreg(p_event);
  238. mutex_unlock(&event_mutex);
  239. destroy_local_trace_kprobe(p_event->tp_event);
  240. }
  241. #endif /* CONFIG_KPROBE_EVENTS */
  242. #ifdef CONFIG_UPROBE_EVENTS
  243. int perf_uprobe_init(struct perf_event *p_event, bool is_retprobe)
  244. {
  245. int ret;
  246. char *path = NULL;
  247. struct trace_event_call *tp_event;
  248. if (!p_event->attr.uprobe_path)
  249. return -EINVAL;
  250. path = strndup_user(u64_to_user_ptr(p_event->attr.uprobe_path),
  251. PATH_MAX);
  252. if (IS_ERR(path)) {
  253. ret = PTR_ERR(path);
  254. return (ret == -EINVAL) ? -E2BIG : ret;
  255. }
  256. if (path[0] == '\0') {
  257. ret = -EINVAL;
  258. goto out;
  259. }
  260. tp_event = create_local_trace_uprobe(
  261. path, p_event->attr.probe_offset, is_retprobe);
  262. if (IS_ERR(tp_event)) {
  263. ret = PTR_ERR(tp_event);
  264. goto out;
  265. }
  266. /*
  267. * local trace_uprobe need to hold event_mutex to call
  268. * uprobe_buffer_enable() and uprobe_buffer_disable().
  269. * event_mutex is not required for local trace_kprobes.
  270. */
  271. mutex_lock(&event_mutex);
  272. ret = perf_trace_event_init(tp_event, p_event);
  273. if (ret)
  274. destroy_local_trace_uprobe(tp_event);
  275. mutex_unlock(&event_mutex);
  276. out:
  277. kfree(path);
  278. return ret;
  279. }
  280. void perf_uprobe_destroy(struct perf_event *p_event)
  281. {
  282. mutex_lock(&event_mutex);
  283. perf_trace_event_close(p_event);
  284. perf_trace_event_unreg(p_event);
  285. mutex_unlock(&event_mutex);
  286. destroy_local_trace_uprobe(p_event->tp_event);
  287. }
  288. #endif /* CONFIG_UPROBE_EVENTS */
  289. int perf_trace_add(struct perf_event *p_event, int flags)
  290. {
  291. struct trace_event_call *tp_event = p_event->tp_event;
  292. if (!(flags & PERF_EF_START))
  293. p_event->hw.state = PERF_HES_STOPPED;
  294. /*
  295. * If TRACE_REG_PERF_ADD returns false; no custom action was performed
  296. * and we need to take the default action of enqueueing our event on
  297. * the right per-cpu hlist.
  298. */
  299. if (!tp_event->class->reg(tp_event, TRACE_REG_PERF_ADD, p_event)) {
  300. struct hlist_head __percpu *pcpu_list;
  301. struct hlist_head *list;
  302. pcpu_list = tp_event->perf_events;
  303. if (WARN_ON_ONCE(!pcpu_list))
  304. return -EINVAL;
  305. list = this_cpu_ptr(pcpu_list);
  306. hlist_add_head_rcu(&p_event->hlist_entry, list);
  307. }
  308. return 0;
  309. }
  310. void perf_trace_del(struct perf_event *p_event, int flags)
  311. {
  312. struct trace_event_call *tp_event = p_event->tp_event;
  313. /*
  314. * If TRACE_REG_PERF_DEL returns false; no custom action was performed
  315. * and we need to take the default action of dequeueing our event from
  316. * the right per-cpu hlist.
  317. */
  318. if (!tp_event->class->reg(tp_event, TRACE_REG_PERF_DEL, p_event))
  319. hlist_del_rcu(&p_event->hlist_entry);
  320. }
  321. void *perf_trace_buf_alloc(int size, struct pt_regs **regs, int *rctxp)
  322. {
  323. char *raw_data;
  324. int rctx;
  325. BUILD_BUG_ON(PERF_MAX_TRACE_SIZE % sizeof(unsigned long));
  326. if (WARN_ONCE(size > PERF_MAX_TRACE_SIZE,
  327. "perf buffer not large enough"))
  328. return NULL;
  329. *rctxp = rctx = perf_swevent_get_recursion_context();
  330. if (rctx < 0)
  331. return NULL;
  332. if (regs)
  333. *regs = this_cpu_ptr(&__perf_regs[rctx]);
  334. raw_data = this_cpu_ptr(perf_trace_buf[rctx]);
  335. /* zero the dead bytes from align to not leak stack to user */
  336. memset(&raw_data[size - sizeof(u64)], 0, sizeof(u64));
  337. return raw_data;
  338. }
  339. EXPORT_SYMBOL_GPL(perf_trace_buf_alloc);
  340. NOKPROBE_SYMBOL(perf_trace_buf_alloc);
  341. void perf_trace_buf_update(void *record, u16 type)
  342. {
  343. struct trace_entry *entry = record;
  344. int pc = preempt_count();
  345. unsigned long flags;
  346. local_save_flags(flags);
  347. tracing_generic_entry_update(entry, flags, pc);
  348. entry->type = type;
  349. }
  350. NOKPROBE_SYMBOL(perf_trace_buf_update);
  351. #ifdef CONFIG_FUNCTION_TRACER
  352. static void
  353. perf_ftrace_function_call(unsigned long ip, unsigned long parent_ip,
  354. struct ftrace_ops *ops, struct pt_regs *pt_regs)
  355. {
  356. struct ftrace_entry *entry;
  357. struct perf_event *event;
  358. struct hlist_head head;
  359. struct pt_regs regs;
  360. int rctx;
  361. if ((unsigned long)ops->private != smp_processor_id())
  362. return;
  363. event = container_of(ops, struct perf_event, ftrace_ops);
  364. /*
  365. * @event->hlist entry is NULL (per INIT_HLIST_NODE), and all
  366. * the perf code does is hlist_for_each_entry_rcu(), so we can
  367. * get away with simply setting the @head.first pointer in order
  368. * to create a singular list.
  369. */
  370. head.first = &event->hlist_entry;
  371. #define ENTRY_SIZE (ALIGN(sizeof(struct ftrace_entry) + sizeof(u32), \
  372. sizeof(u64)) - sizeof(u32))
  373. BUILD_BUG_ON(ENTRY_SIZE > PERF_MAX_TRACE_SIZE);
  374. memset(&regs, 0, sizeof(regs));
  375. perf_fetch_caller_regs(&regs);
  376. entry = perf_trace_buf_alloc(ENTRY_SIZE, NULL, &rctx);
  377. if (!entry)
  378. return;
  379. entry->ip = ip;
  380. entry->parent_ip = parent_ip;
  381. perf_trace_buf_submit(entry, ENTRY_SIZE, rctx, TRACE_FN,
  382. 1, &regs, &head, NULL);
  383. #undef ENTRY_SIZE
  384. }
  385. static int perf_ftrace_function_register(struct perf_event *event)
  386. {
  387. struct ftrace_ops *ops = &event->ftrace_ops;
  388. ops->flags = FTRACE_OPS_FL_RCU;
  389. ops->func = perf_ftrace_function_call;
  390. ops->private = (void *)(unsigned long)nr_cpu_ids;
  391. return register_ftrace_function(ops);
  392. }
  393. static int perf_ftrace_function_unregister(struct perf_event *event)
  394. {
  395. struct ftrace_ops *ops = &event->ftrace_ops;
  396. int ret = unregister_ftrace_function(ops);
  397. ftrace_free_filter(ops);
  398. return ret;
  399. }
  400. int perf_ftrace_event_register(struct trace_event_call *call,
  401. enum trace_reg type, void *data)
  402. {
  403. struct perf_event *event = data;
  404. switch (type) {
  405. case TRACE_REG_REGISTER:
  406. case TRACE_REG_UNREGISTER:
  407. break;
  408. case TRACE_REG_PERF_REGISTER:
  409. case TRACE_REG_PERF_UNREGISTER:
  410. return 0;
  411. case TRACE_REG_PERF_OPEN:
  412. return perf_ftrace_function_register(data);
  413. case TRACE_REG_PERF_CLOSE:
  414. return perf_ftrace_function_unregister(data);
  415. case TRACE_REG_PERF_ADD:
  416. event->ftrace_ops.private = (void *)(unsigned long)smp_processor_id();
  417. return 1;
  418. case TRACE_REG_PERF_DEL:
  419. event->ftrace_ops.private = (void *)(unsigned long)nr_cpu_ids;
  420. return 1;
  421. }
  422. return -EINVAL;
  423. }
  424. #endif /* CONFIG_FUNCTION_TRACER */