trace_syscalls.c 21 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828
  1. // SPDX-License-Identifier: GPL-2.0
  2. #include <trace/syscall.h>
  3. #include <trace/events/syscalls.h>
  4. #include <linux/syscalls.h>
  5. #include <linux/slab.h>
  6. #include <linux/kernel.h>
  7. #include <linux/module.h> /* for MODULE_NAME_LEN via KSYM_SYMBOL_LEN */
  8. #include <linux/ftrace.h>
  9. #include <linux/perf_event.h>
  10. #include <linux/xarray.h>
  11. #include <asm/syscall.h>
  12. #include "trace_output.h"
  13. #include "trace.h"
  14. static DEFINE_MUTEX(syscall_trace_lock);
  15. static int syscall_enter_register(struct trace_event_call *event,
  16. enum trace_reg type, void *data);
  17. static int syscall_exit_register(struct trace_event_call *event,
  18. enum trace_reg type, void *data);
  19. static struct list_head *
  20. syscall_get_enter_fields(struct trace_event_call *call)
  21. {
  22. struct syscall_metadata *entry = call->data;
  23. return &entry->enter_fields;
  24. }
  25. extern struct syscall_metadata *__start_syscalls_metadata[];
  26. extern struct syscall_metadata *__stop_syscalls_metadata[];
  27. static DEFINE_XARRAY(syscalls_metadata_sparse);
  28. static struct syscall_metadata **syscalls_metadata;
  29. #ifndef ARCH_HAS_SYSCALL_MATCH_SYM_NAME
  30. static inline bool arch_syscall_match_sym_name(const char *sym, const char *name)
  31. {
  32. /*
  33. * Only compare after the "sys" prefix. Archs that use
  34. * syscall wrappers may have syscalls symbols aliases prefixed
  35. * with ".SyS" or ".sys" instead of "sys", leading to an unwanted
  36. * mismatch.
  37. */
  38. return !strcmp(sym + 3, name + 3);
  39. }
  40. #endif
  41. #ifdef ARCH_TRACE_IGNORE_COMPAT_SYSCALLS
  42. /*
  43. * Some architectures that allow for 32bit applications
  44. * to run on a 64bit kernel, do not map the syscalls for
  45. * the 32bit tasks the same as they do for 64bit tasks.
  46. *
  47. * *cough*x86*cough*
  48. *
  49. * In such a case, instead of reporting the wrong syscalls,
  50. * simply ignore them.
  51. *
  52. * For an arch to ignore the compat syscalls it needs to
  53. * define ARCH_TRACE_IGNORE_COMPAT_SYSCALLS as well as
  54. * define the function arch_trace_is_compat_syscall() to let
  55. * the tracing system know that it should ignore it.
  56. */
  57. static int
  58. trace_get_syscall_nr(struct task_struct *task, struct pt_regs *regs)
  59. {
  60. if (unlikely(arch_trace_is_compat_syscall(regs)))
  61. return -1;
  62. return syscall_get_nr(task, regs);
  63. }
  64. #else
  65. static inline int
  66. trace_get_syscall_nr(struct task_struct *task, struct pt_regs *regs)
  67. {
  68. return syscall_get_nr(task, regs);
  69. }
  70. #endif /* ARCH_TRACE_IGNORE_COMPAT_SYSCALLS */
  71. static __init struct syscall_metadata *
  72. find_syscall_meta(unsigned long syscall)
  73. {
  74. struct syscall_metadata **start;
  75. struct syscall_metadata **stop;
  76. char str[KSYM_SYMBOL_LEN];
  77. start = __start_syscalls_metadata;
  78. stop = __stop_syscalls_metadata;
  79. kallsyms_lookup(syscall, NULL, NULL, NULL, str);
  80. if (arch_syscall_match_sym_name(str, "sys_ni_syscall"))
  81. return NULL;
  82. for ( ; start < stop; start++) {
  83. if ((*start)->name && arch_syscall_match_sym_name(str, (*start)->name))
  84. return *start;
  85. }
  86. return NULL;
  87. }
  88. static struct syscall_metadata *syscall_nr_to_meta(int nr)
  89. {
  90. if (IS_ENABLED(CONFIG_HAVE_SPARSE_SYSCALL_NR))
  91. return xa_load(&syscalls_metadata_sparse, (unsigned long)nr);
  92. if (!syscalls_metadata || nr >= NR_syscalls || nr < 0)
  93. return NULL;
  94. return syscalls_metadata[nr];
  95. }
  96. const char *get_syscall_name(int syscall)
  97. {
  98. struct syscall_metadata *entry;
  99. entry = syscall_nr_to_meta(syscall);
  100. if (!entry)
  101. return NULL;
  102. return entry->name;
  103. }
  104. static enum print_line_t
  105. print_syscall_enter(struct trace_iterator *iter, int flags,
  106. struct trace_event *event)
  107. {
  108. struct trace_array *tr = iter->tr;
  109. struct trace_seq *s = &iter->seq;
  110. struct trace_entry *ent = iter->ent;
  111. struct syscall_trace_enter *trace;
  112. struct syscall_metadata *entry;
  113. int i, syscall;
  114. trace = (typeof(trace))ent;
  115. syscall = trace->nr;
  116. entry = syscall_nr_to_meta(syscall);
  117. if (!entry)
  118. goto end;
  119. if (entry->enter_event->event.type != ent->type) {
  120. WARN_ON_ONCE(1);
  121. goto end;
  122. }
  123. trace_seq_printf(s, "%s(", entry->name);
  124. for (i = 0; i < entry->nb_args; i++) {
  125. if (trace_seq_has_overflowed(s))
  126. goto end;
  127. /* parameter types */
  128. if (tr && tr->trace_flags & TRACE_ITER_VERBOSE)
  129. trace_seq_printf(s, "%s ", entry->types[i]);
  130. /* parameter values */
  131. trace_seq_printf(s, "%s: %lx%s", entry->args[i],
  132. trace->args[i],
  133. i == entry->nb_args - 1 ? "" : ", ");
  134. }
  135. trace_seq_putc(s, ')');
  136. end:
  137. trace_seq_putc(s, '\n');
  138. return trace_handle_return(s);
  139. }
  140. static enum print_line_t
  141. print_syscall_exit(struct trace_iterator *iter, int flags,
  142. struct trace_event *event)
  143. {
  144. struct trace_seq *s = &iter->seq;
  145. struct trace_entry *ent = iter->ent;
  146. struct syscall_trace_exit *trace;
  147. int syscall;
  148. struct syscall_metadata *entry;
  149. trace = (typeof(trace))ent;
  150. syscall = trace->nr;
  151. entry = syscall_nr_to_meta(syscall);
  152. if (!entry) {
  153. trace_seq_putc(s, '\n');
  154. goto out;
  155. }
  156. if (entry->exit_event->event.type != ent->type) {
  157. WARN_ON_ONCE(1);
  158. return TRACE_TYPE_UNHANDLED;
  159. }
  160. trace_seq_printf(s, "%s -> 0x%lx\n", entry->name,
  161. trace->ret);
  162. out:
  163. return trace_handle_return(s);
  164. }
  165. #define SYSCALL_FIELD(_type, _name) { \
  166. .type = #_type, .name = #_name, \
  167. .size = sizeof(_type), .align = __alignof__(_type), \
  168. .is_signed = is_signed_type(_type), .filter_type = FILTER_OTHER }
  169. static int __init
  170. __set_enter_print_fmt(struct syscall_metadata *entry, char *buf, int len)
  171. {
  172. int i;
  173. int pos = 0;
  174. /* When len=0, we just calculate the needed length */
  175. #define LEN_OR_ZERO (len ? len - pos : 0)
  176. pos += snprintf(buf + pos, LEN_OR_ZERO, "\"");
  177. for (i = 0; i < entry->nb_args; i++) {
  178. pos += snprintf(buf + pos, LEN_OR_ZERO, "%s: 0x%%0%zulx%s",
  179. entry->args[i], sizeof(unsigned long),
  180. i == entry->nb_args - 1 ? "" : ", ");
  181. }
  182. pos += snprintf(buf + pos, LEN_OR_ZERO, "\"");
  183. for (i = 0; i < entry->nb_args; i++) {
  184. pos += snprintf(buf + pos, LEN_OR_ZERO,
  185. ", ((unsigned long)(REC->%s))", entry->args[i]);
  186. }
  187. #undef LEN_OR_ZERO
  188. /* return the length of print_fmt */
  189. return pos;
  190. }
  191. static int __init set_syscall_print_fmt(struct trace_event_call *call)
  192. {
  193. char *print_fmt;
  194. int len;
  195. struct syscall_metadata *entry = call->data;
  196. if (entry->enter_event != call) {
  197. call->print_fmt = "\"0x%lx\", REC->ret";
  198. return 0;
  199. }
  200. /* First: called with 0 length to calculate the needed length */
  201. len = __set_enter_print_fmt(entry, NULL, 0);
  202. print_fmt = kmalloc(len + 1, GFP_KERNEL);
  203. if (!print_fmt)
  204. return -ENOMEM;
  205. /* Second: actually write the @print_fmt */
  206. __set_enter_print_fmt(entry, print_fmt, len + 1);
  207. call->print_fmt = print_fmt;
  208. return 0;
  209. }
  210. static void __init free_syscall_print_fmt(struct trace_event_call *call)
  211. {
  212. struct syscall_metadata *entry = call->data;
  213. if (entry->enter_event == call)
  214. kfree(call->print_fmt);
  215. }
  216. static int __init syscall_enter_define_fields(struct trace_event_call *call)
  217. {
  218. struct syscall_trace_enter trace;
  219. struct syscall_metadata *meta = call->data;
  220. int offset = offsetof(typeof(trace), args);
  221. int ret = 0;
  222. int i;
  223. for (i = 0; i < meta->nb_args; i++) {
  224. ret = trace_define_field(call, meta->types[i],
  225. meta->args[i], offset,
  226. sizeof(unsigned long), 0,
  227. FILTER_OTHER);
  228. if (ret)
  229. break;
  230. offset += sizeof(unsigned long);
  231. }
  232. return ret;
  233. }
  234. static void ftrace_syscall_enter(void *data, struct pt_regs *regs, long id)
  235. {
  236. struct trace_array *tr = data;
  237. struct trace_event_file *trace_file;
  238. struct syscall_trace_enter *entry;
  239. struct syscall_metadata *sys_data;
  240. struct trace_event_buffer fbuffer;
  241. unsigned long args[6];
  242. int syscall_nr;
  243. int size;
  244. /*
  245. * Syscall probe called with preemption enabled, but the ring
  246. * buffer and per-cpu data require preemption to be disabled.
  247. */
  248. guard(preempt_notrace)();
  249. syscall_nr = trace_get_syscall_nr(current, regs);
  250. if (syscall_nr < 0 || syscall_nr >= NR_syscalls)
  251. return;
  252. /* Here we're inside tp handler's rcu_read_lock_sched (__DO_TRACE) */
  253. trace_file = rcu_dereference_sched(tr->enter_syscall_files[syscall_nr]);
  254. if (!trace_file)
  255. return;
  256. if (trace_trigger_soft_disabled(trace_file))
  257. return;
  258. sys_data = syscall_nr_to_meta(syscall_nr);
  259. if (!sys_data)
  260. return;
  261. size = sizeof(*entry) + sizeof(unsigned long) * sys_data->nb_args;
  262. entry = trace_event_buffer_reserve(&fbuffer, trace_file, size);
  263. if (!entry)
  264. return;
  265. entry = ring_buffer_event_data(fbuffer.event);
  266. entry->nr = syscall_nr;
  267. syscall_get_arguments(current, regs, args);
  268. memcpy(entry->args, args, sizeof(unsigned long) * sys_data->nb_args);
  269. trace_event_buffer_commit(&fbuffer);
  270. }
  271. static void ftrace_syscall_exit(void *data, struct pt_regs *regs, long ret)
  272. {
  273. struct trace_array *tr = data;
  274. struct trace_event_file *trace_file;
  275. struct syscall_trace_exit *entry;
  276. struct syscall_metadata *sys_data;
  277. struct trace_event_buffer fbuffer;
  278. int syscall_nr;
  279. /*
  280. * Syscall probe called with preemption enabled, but the ring
  281. * buffer and per-cpu data require preemption to be disabled.
  282. */
  283. guard(preempt_notrace)();
  284. syscall_nr = trace_get_syscall_nr(current, regs);
  285. if (syscall_nr < 0 || syscall_nr >= NR_syscalls)
  286. return;
  287. /* Here we're inside tp handler's rcu_read_lock_sched (__DO_TRACE()) */
  288. trace_file = rcu_dereference_sched(tr->exit_syscall_files[syscall_nr]);
  289. if (!trace_file)
  290. return;
  291. if (trace_trigger_soft_disabled(trace_file))
  292. return;
  293. sys_data = syscall_nr_to_meta(syscall_nr);
  294. if (!sys_data)
  295. return;
  296. entry = trace_event_buffer_reserve(&fbuffer, trace_file, sizeof(*entry));
  297. if (!entry)
  298. return;
  299. entry = ring_buffer_event_data(fbuffer.event);
  300. entry->nr = syscall_nr;
  301. entry->ret = syscall_get_return_value(current, regs);
  302. trace_event_buffer_commit(&fbuffer);
  303. }
  304. static int reg_event_syscall_enter(struct trace_event_file *file,
  305. struct trace_event_call *call)
  306. {
  307. struct trace_array *tr = file->tr;
  308. int ret = 0;
  309. int num;
  310. num = ((struct syscall_metadata *)call->data)->syscall_nr;
  311. if (WARN_ON_ONCE(num < 0 || num >= NR_syscalls))
  312. return -ENOSYS;
  313. mutex_lock(&syscall_trace_lock);
  314. if (!tr->sys_refcount_enter)
  315. ret = register_trace_sys_enter(ftrace_syscall_enter, tr);
  316. if (!ret) {
  317. rcu_assign_pointer(tr->enter_syscall_files[num], file);
  318. tr->sys_refcount_enter++;
  319. }
  320. mutex_unlock(&syscall_trace_lock);
  321. return ret;
  322. }
  323. static void unreg_event_syscall_enter(struct trace_event_file *file,
  324. struct trace_event_call *call)
  325. {
  326. struct trace_array *tr = file->tr;
  327. int num;
  328. num = ((struct syscall_metadata *)call->data)->syscall_nr;
  329. if (WARN_ON_ONCE(num < 0 || num >= NR_syscalls))
  330. return;
  331. mutex_lock(&syscall_trace_lock);
  332. tr->sys_refcount_enter--;
  333. RCU_INIT_POINTER(tr->enter_syscall_files[num], NULL);
  334. if (!tr->sys_refcount_enter)
  335. unregister_trace_sys_enter(ftrace_syscall_enter, tr);
  336. mutex_unlock(&syscall_trace_lock);
  337. }
  338. static int reg_event_syscall_exit(struct trace_event_file *file,
  339. struct trace_event_call *call)
  340. {
  341. struct trace_array *tr = file->tr;
  342. int ret = 0;
  343. int num;
  344. num = ((struct syscall_metadata *)call->data)->syscall_nr;
  345. if (WARN_ON_ONCE(num < 0 || num >= NR_syscalls))
  346. return -ENOSYS;
  347. mutex_lock(&syscall_trace_lock);
  348. if (!tr->sys_refcount_exit)
  349. ret = register_trace_sys_exit(ftrace_syscall_exit, tr);
  350. if (!ret) {
  351. rcu_assign_pointer(tr->exit_syscall_files[num], file);
  352. tr->sys_refcount_exit++;
  353. }
  354. mutex_unlock(&syscall_trace_lock);
  355. return ret;
  356. }
  357. static void unreg_event_syscall_exit(struct trace_event_file *file,
  358. struct trace_event_call *call)
  359. {
  360. struct trace_array *tr = file->tr;
  361. int num;
  362. num = ((struct syscall_metadata *)call->data)->syscall_nr;
  363. if (WARN_ON_ONCE(num < 0 || num >= NR_syscalls))
  364. return;
  365. mutex_lock(&syscall_trace_lock);
  366. tr->sys_refcount_exit--;
  367. RCU_INIT_POINTER(tr->exit_syscall_files[num], NULL);
  368. if (!tr->sys_refcount_exit)
  369. unregister_trace_sys_exit(ftrace_syscall_exit, tr);
  370. mutex_unlock(&syscall_trace_lock);
  371. }
  372. static int __init init_syscall_trace(struct trace_event_call *call)
  373. {
  374. int id;
  375. int num;
  376. num = ((struct syscall_metadata *)call->data)->syscall_nr;
  377. if (num < 0 || num >= NR_syscalls) {
  378. pr_debug("syscall %s metadata not mapped, disabling ftrace event\n",
  379. ((struct syscall_metadata *)call->data)->name);
  380. return -ENOSYS;
  381. }
  382. if (set_syscall_print_fmt(call) < 0)
  383. return -ENOMEM;
  384. id = trace_event_raw_init(call);
  385. if (id < 0) {
  386. free_syscall_print_fmt(call);
  387. return id;
  388. }
  389. return id;
  390. }
  391. static struct trace_event_fields __refdata syscall_enter_fields_array[] = {
  392. SYSCALL_FIELD(int, __syscall_nr),
  393. { .type = TRACE_FUNCTION_TYPE,
  394. .define_fields = syscall_enter_define_fields },
  395. {}
  396. };
  397. struct trace_event_functions enter_syscall_print_funcs = {
  398. .trace = print_syscall_enter,
  399. };
  400. struct trace_event_functions exit_syscall_print_funcs = {
  401. .trace = print_syscall_exit,
  402. };
  403. struct trace_event_class __refdata event_class_syscall_enter = {
  404. .system = "syscalls",
  405. .reg = syscall_enter_register,
  406. .fields_array = syscall_enter_fields_array,
  407. .get_fields = syscall_get_enter_fields,
  408. .raw_init = init_syscall_trace,
  409. };
  410. struct trace_event_class __refdata event_class_syscall_exit = {
  411. .system = "syscalls",
  412. .reg = syscall_exit_register,
  413. .fields_array = (struct trace_event_fields[]){
  414. SYSCALL_FIELD(int, __syscall_nr),
  415. SYSCALL_FIELD(long, ret),
  416. {}
  417. },
  418. .fields = LIST_HEAD_INIT(event_class_syscall_exit.fields),
  419. .raw_init = init_syscall_trace,
  420. };
  421. unsigned long __init __weak arch_syscall_addr(int nr)
  422. {
  423. return (unsigned long)sys_call_table[nr];
  424. }
  425. void __init init_ftrace_syscalls(void)
  426. {
  427. struct syscall_metadata *meta;
  428. unsigned long addr;
  429. int i;
  430. void *ret;
  431. if (!IS_ENABLED(CONFIG_HAVE_SPARSE_SYSCALL_NR)) {
  432. syscalls_metadata = kcalloc(NR_syscalls,
  433. sizeof(*syscalls_metadata),
  434. GFP_KERNEL);
  435. if (!syscalls_metadata) {
  436. WARN_ON(1);
  437. return;
  438. }
  439. }
  440. for (i = 0; i < NR_syscalls; i++) {
  441. addr = arch_syscall_addr(i);
  442. meta = find_syscall_meta(addr);
  443. if (!meta)
  444. continue;
  445. meta->syscall_nr = i;
  446. if (!IS_ENABLED(CONFIG_HAVE_SPARSE_SYSCALL_NR)) {
  447. syscalls_metadata[i] = meta;
  448. } else {
  449. ret = xa_store(&syscalls_metadata_sparse, i, meta,
  450. GFP_KERNEL);
  451. WARN(xa_is_err(ret),
  452. "Syscall memory allocation failed\n");
  453. }
  454. }
  455. }
  456. #ifdef CONFIG_PERF_EVENTS
  457. static DECLARE_BITMAP(enabled_perf_enter_syscalls, NR_syscalls);
  458. static DECLARE_BITMAP(enabled_perf_exit_syscalls, NR_syscalls);
  459. static int sys_perf_refcount_enter;
  460. static int sys_perf_refcount_exit;
  461. static int perf_call_bpf_enter(struct trace_event_call *call, struct pt_regs *regs,
  462. struct syscall_metadata *sys_data,
  463. struct syscall_trace_enter *rec)
  464. {
  465. struct syscall_tp_t {
  466. struct trace_entry ent;
  467. int syscall_nr;
  468. unsigned long args[SYSCALL_DEFINE_MAXARGS];
  469. } __aligned(8) param;
  470. int i;
  471. BUILD_BUG_ON(sizeof(param.ent) < sizeof(void *));
  472. /* bpf prog requires 'regs' to be the first member in the ctx (a.k.a. &param) */
  473. perf_fetch_caller_regs(regs);
  474. *(struct pt_regs **)&param = regs;
  475. param.syscall_nr = rec->nr;
  476. for (i = 0; i < sys_data->nb_args; i++)
  477. param.args[i] = rec->args[i];
  478. return trace_call_bpf(call, &param);
  479. }
  480. static void perf_syscall_enter(void *ignore, struct pt_regs *regs, long id)
  481. {
  482. struct syscall_metadata *sys_data;
  483. struct syscall_trace_enter *rec;
  484. struct pt_regs *fake_regs;
  485. struct hlist_head *head;
  486. unsigned long args[6];
  487. bool valid_prog_array;
  488. int syscall_nr;
  489. int rctx;
  490. int size;
  491. syscall_nr = trace_get_syscall_nr(current, regs);
  492. if (syscall_nr < 0 || syscall_nr >= NR_syscalls)
  493. return;
  494. if (!test_bit(syscall_nr, enabled_perf_enter_syscalls))
  495. return;
  496. sys_data = syscall_nr_to_meta(syscall_nr);
  497. if (!sys_data)
  498. return;
  499. head = this_cpu_ptr(sys_data->enter_event->perf_events);
  500. valid_prog_array = bpf_prog_array_valid(sys_data->enter_event);
  501. if (!valid_prog_array && hlist_empty(head))
  502. return;
  503. /* get the size after alignment with the u32 buffer size field */
  504. size = sizeof(unsigned long) * sys_data->nb_args + sizeof(*rec);
  505. size = ALIGN(size + sizeof(u32), sizeof(u64));
  506. size -= sizeof(u32);
  507. rec = perf_trace_buf_alloc(size, &fake_regs, &rctx);
  508. if (!rec)
  509. return;
  510. rec->nr = syscall_nr;
  511. syscall_get_arguments(current, regs, args);
  512. memcpy(&rec->args, args, sizeof(unsigned long) * sys_data->nb_args);
  513. if ((valid_prog_array &&
  514. !perf_call_bpf_enter(sys_data->enter_event, fake_regs, sys_data, rec)) ||
  515. hlist_empty(head)) {
  516. perf_swevent_put_recursion_context(rctx);
  517. return;
  518. }
  519. perf_trace_buf_submit(rec, size, rctx,
  520. sys_data->enter_event->event.type, 1, regs,
  521. head, NULL);
  522. }
  523. static int perf_sysenter_enable(struct trace_event_call *call)
  524. {
  525. int ret = 0;
  526. int num;
  527. num = ((struct syscall_metadata *)call->data)->syscall_nr;
  528. mutex_lock(&syscall_trace_lock);
  529. if (!sys_perf_refcount_enter)
  530. ret = register_trace_sys_enter(perf_syscall_enter, NULL);
  531. if (ret) {
  532. pr_info("event trace: Could not activate syscall entry trace point");
  533. } else {
  534. set_bit(num, enabled_perf_enter_syscalls);
  535. sys_perf_refcount_enter++;
  536. }
  537. mutex_unlock(&syscall_trace_lock);
  538. return ret;
  539. }
  540. static void perf_sysenter_disable(struct trace_event_call *call)
  541. {
  542. int num;
  543. num = ((struct syscall_metadata *)call->data)->syscall_nr;
  544. mutex_lock(&syscall_trace_lock);
  545. sys_perf_refcount_enter--;
  546. clear_bit(num, enabled_perf_enter_syscalls);
  547. if (!sys_perf_refcount_enter)
  548. unregister_trace_sys_enter(perf_syscall_enter, NULL);
  549. mutex_unlock(&syscall_trace_lock);
  550. }
  551. static int perf_call_bpf_exit(struct trace_event_call *call, struct pt_regs *regs,
  552. struct syscall_trace_exit *rec)
  553. {
  554. struct syscall_tp_t {
  555. struct trace_entry ent;
  556. int syscall_nr;
  557. unsigned long ret;
  558. } __aligned(8) param;
  559. /* bpf prog requires 'regs' to be the first member in the ctx (a.k.a. &param) */
  560. perf_fetch_caller_regs(regs);
  561. *(struct pt_regs **)&param = regs;
  562. param.syscall_nr = rec->nr;
  563. param.ret = rec->ret;
  564. return trace_call_bpf(call, &param);
  565. }
  566. static void perf_syscall_exit(void *ignore, struct pt_regs *regs, long ret)
  567. {
  568. struct syscall_metadata *sys_data;
  569. struct syscall_trace_exit *rec;
  570. struct pt_regs *fake_regs;
  571. struct hlist_head *head;
  572. bool valid_prog_array;
  573. int syscall_nr;
  574. int rctx;
  575. int size;
  576. syscall_nr = trace_get_syscall_nr(current, regs);
  577. if (syscall_nr < 0 || syscall_nr >= NR_syscalls)
  578. return;
  579. if (!test_bit(syscall_nr, enabled_perf_exit_syscalls))
  580. return;
  581. sys_data = syscall_nr_to_meta(syscall_nr);
  582. if (!sys_data)
  583. return;
  584. head = this_cpu_ptr(sys_data->exit_event->perf_events);
  585. valid_prog_array = bpf_prog_array_valid(sys_data->exit_event);
  586. if (!valid_prog_array && hlist_empty(head))
  587. return;
  588. /* We can probably do that at build time */
  589. size = ALIGN(sizeof(*rec) + sizeof(u32), sizeof(u64));
  590. size -= sizeof(u32);
  591. rec = perf_trace_buf_alloc(size, &fake_regs, &rctx);
  592. if (!rec)
  593. return;
  594. rec->nr = syscall_nr;
  595. rec->ret = syscall_get_return_value(current, regs);
  596. if ((valid_prog_array &&
  597. !perf_call_bpf_exit(sys_data->exit_event, fake_regs, rec)) ||
  598. hlist_empty(head)) {
  599. perf_swevent_put_recursion_context(rctx);
  600. return;
  601. }
  602. perf_trace_buf_submit(rec, size, rctx, sys_data->exit_event->event.type,
  603. 1, regs, head, NULL);
  604. }
  605. static int perf_sysexit_enable(struct trace_event_call *call)
  606. {
  607. int ret = 0;
  608. int num;
  609. num = ((struct syscall_metadata *)call->data)->syscall_nr;
  610. mutex_lock(&syscall_trace_lock);
  611. if (!sys_perf_refcount_exit)
  612. ret = register_trace_sys_exit(perf_syscall_exit, NULL);
  613. if (ret) {
  614. pr_info("event trace: Could not activate syscall exit trace point");
  615. } else {
  616. set_bit(num, enabled_perf_exit_syscalls);
  617. sys_perf_refcount_exit++;
  618. }
  619. mutex_unlock(&syscall_trace_lock);
  620. return ret;
  621. }
  622. static void perf_sysexit_disable(struct trace_event_call *call)
  623. {
  624. int num;
  625. num = ((struct syscall_metadata *)call->data)->syscall_nr;
  626. mutex_lock(&syscall_trace_lock);
  627. sys_perf_refcount_exit--;
  628. clear_bit(num, enabled_perf_exit_syscalls);
  629. if (!sys_perf_refcount_exit)
  630. unregister_trace_sys_exit(perf_syscall_exit, NULL);
  631. mutex_unlock(&syscall_trace_lock);
  632. }
  633. #endif /* CONFIG_PERF_EVENTS */
  634. static int syscall_enter_register(struct trace_event_call *event,
  635. enum trace_reg type, void *data)
  636. {
  637. struct trace_event_file *file = data;
  638. switch (type) {
  639. case TRACE_REG_REGISTER:
  640. return reg_event_syscall_enter(file, event);
  641. case TRACE_REG_UNREGISTER:
  642. unreg_event_syscall_enter(file, event);
  643. return 0;
  644. #ifdef CONFIG_PERF_EVENTS
  645. case TRACE_REG_PERF_REGISTER:
  646. return perf_sysenter_enable(event);
  647. case TRACE_REG_PERF_UNREGISTER:
  648. perf_sysenter_disable(event);
  649. return 0;
  650. case TRACE_REG_PERF_OPEN:
  651. case TRACE_REG_PERF_CLOSE:
  652. case TRACE_REG_PERF_ADD:
  653. case TRACE_REG_PERF_DEL:
  654. return 0;
  655. #endif
  656. }
  657. return 0;
  658. }
  659. static int syscall_exit_register(struct trace_event_call *event,
  660. enum trace_reg type, void *data)
  661. {
  662. struct trace_event_file *file = data;
  663. switch (type) {
  664. case TRACE_REG_REGISTER:
  665. return reg_event_syscall_exit(file, event);
  666. case TRACE_REG_UNREGISTER:
  667. unreg_event_syscall_exit(file, event);
  668. return 0;
  669. #ifdef CONFIG_PERF_EVENTS
  670. case TRACE_REG_PERF_REGISTER:
  671. return perf_sysexit_enable(event);
  672. case TRACE_REG_PERF_UNREGISTER:
  673. perf_sysexit_disable(event);
  674. return 0;
  675. case TRACE_REG_PERF_OPEN:
  676. case TRACE_REG_PERF_CLOSE:
  677. case TRACE_REG_PERF_ADD:
  678. case TRACE_REG_PERF_DEL:
  679. return 0;
  680. #endif
  681. }
  682. return 0;
  683. }