builtin-test.c 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * builtin-test.c
  4. *
  5. * Builtin regression testing command: ever growing number of sanity tests
  6. */
  7. #include <fcntl.h>
  8. #include <errno.h>
  9. #include <poll.h>
  10. #include <unistd.h>
  11. #include <string.h>
  12. #include <stdlib.h>
  13. #include <sys/types.h>
  14. #include <dirent.h>
  15. #include <sys/wait.h>
  16. #include <sys/stat.h>
  17. #include "builtin.h"
  18. #include "config.h"
  19. #include "hist.h"
  20. #include "intlist.h"
  21. #include "tests.h"
  22. #include "debug.h"
  23. #include "color.h"
  24. #include <subcmd/parse-options.h>
  25. #include <subcmd/run-command.h>
  26. #include "string2.h"
  27. #include "symbol.h"
  28. #include "util/rlimit.h"
  29. #include "util/strbuf.h"
  30. #include <linux/kernel.h>
  31. #include <linux/string.h>
  32. #include <subcmd/exec-cmd.h>
  33. #include <linux/zalloc.h>
  34. #include "tests-scripts.h"
  35. /*
  36. * Command line option to not fork the test running in the same process and
  37. * making them easier to debug.
  38. */
  39. static bool dont_fork;
  40. /* Don't fork the tests in parallel and wait for their completion. */
  41. static bool sequential = true;
  42. /* Do it in parallel, lacks infrastructure to avoid running tests that clash for resources,
  43. * So leave it as the developers choice to enable while working on the needed infra */
  44. static bool parallel;
  45. const char *dso_to_test;
  46. const char *test_objdump_path = "objdump";
  47. /*
  48. * List of architecture specific tests. Not a weak symbol as the array length is
  49. * dependent on the initialization, as such GCC with LTO complains of
  50. * conflicting definitions with a weak symbol.
  51. */
  52. #if defined(__i386__) || defined(__x86_64__) || defined(__aarch64__) || defined(__powerpc64__)
  53. extern struct test_suite *arch_tests[];
  54. #else
  55. static struct test_suite *arch_tests[] = {
  56. NULL,
  57. };
  58. #endif
  59. static struct test_suite *generic_tests[] = {
  60. &suite__vmlinux_matches_kallsyms,
  61. #ifdef HAVE_LIBTRACEEVENT
  62. &suite__openat_syscall_event,
  63. &suite__openat_syscall_event_on_all_cpus,
  64. &suite__basic_mmap,
  65. #endif
  66. &suite__mem,
  67. &suite__parse_events,
  68. &suite__expr,
  69. &suite__PERF_RECORD,
  70. &suite__pmu,
  71. &suite__pmu_events,
  72. &suite__dso_data,
  73. &suite__perf_evsel__roundtrip_name_test,
  74. #ifdef HAVE_LIBTRACEEVENT
  75. &suite__perf_evsel__tp_sched_test,
  76. &suite__syscall_openat_tp_fields,
  77. #endif
  78. &suite__attr,
  79. &suite__hists_link,
  80. &suite__python_use,
  81. &suite__bp_signal,
  82. &suite__bp_signal_overflow,
  83. &suite__bp_accounting,
  84. &suite__wp,
  85. &suite__task_exit,
  86. &suite__sw_clock_freq,
  87. &suite__code_reading,
  88. &suite__sample_parsing,
  89. &suite__keep_tracking,
  90. &suite__parse_no_sample_id_all,
  91. &suite__hists_filter,
  92. &suite__mmap_thread_lookup,
  93. &suite__thread_maps_share,
  94. &suite__hists_output,
  95. &suite__hists_cumulate,
  96. #ifdef HAVE_LIBTRACEEVENT
  97. &suite__switch_tracking,
  98. #endif
  99. &suite__fdarray__filter,
  100. &suite__fdarray__add,
  101. &suite__kmod_path__parse,
  102. &suite__thread_map,
  103. &suite__session_topology,
  104. &suite__thread_map_synthesize,
  105. &suite__thread_map_remove,
  106. &suite__cpu_map,
  107. &suite__synthesize_stat_config,
  108. &suite__synthesize_stat,
  109. &suite__synthesize_stat_round,
  110. &suite__event_update,
  111. &suite__event_times,
  112. &suite__backward_ring_buffer,
  113. &suite__sdt_event,
  114. &suite__is_printable_array,
  115. &suite__bitmap_print,
  116. &suite__perf_hooks,
  117. &suite__unit_number__scnprint,
  118. &suite__mem2node,
  119. &suite__time_utils,
  120. &suite__jit_write_elf,
  121. &suite__pfm,
  122. &suite__api_io,
  123. &suite__maps__merge_in,
  124. &suite__demangle_java,
  125. &suite__demangle_ocaml,
  126. &suite__parse_metric,
  127. &suite__pe_file_parsing,
  128. &suite__expand_cgroup_events,
  129. &suite__perf_time_to_tsc,
  130. &suite__dlfilter,
  131. &suite__sigtrap,
  132. &suite__event_groups,
  133. &suite__symbols,
  134. &suite__util,
  135. NULL,
  136. };
  137. static struct test_suite **tests[] = {
  138. generic_tests,
  139. arch_tests,
  140. NULL, /* shell tests created at runtime. */
  141. };
  142. static struct test_workload *workloads[] = {
  143. &workload__noploop,
  144. &workload__thloop,
  145. &workload__leafloop,
  146. &workload__sqrtloop,
  147. &workload__brstack,
  148. &workload__datasym,
  149. &workload__landlock,
  150. };
  151. static int num_subtests(const struct test_suite *t)
  152. {
  153. int num;
  154. if (!t->test_cases)
  155. return 0;
  156. num = 0;
  157. while (t->test_cases[num].name)
  158. num++;
  159. return num;
  160. }
  161. static bool has_subtests(const struct test_suite *t)
  162. {
  163. return num_subtests(t) > 1;
  164. }
  165. static const char *skip_reason(const struct test_suite *t, int subtest)
  166. {
  167. if (!t->test_cases)
  168. return NULL;
  169. return t->test_cases[subtest >= 0 ? subtest : 0].skip_reason;
  170. }
  171. static const char *test_description(const struct test_suite *t, int subtest)
  172. {
  173. if (t->test_cases && subtest >= 0)
  174. return t->test_cases[subtest].desc;
  175. return t->desc;
  176. }
  177. static test_fnptr test_function(const struct test_suite *t, int subtest)
  178. {
  179. if (subtest <= 0)
  180. return t->test_cases[0].run_case;
  181. return t->test_cases[subtest].run_case;
  182. }
  183. static bool perf_test__matches(const char *desc, int curr, int argc, const char *argv[])
  184. {
  185. int i;
  186. if (argc == 0)
  187. return true;
  188. for (i = 0; i < argc; ++i) {
  189. char *end;
  190. long nr = strtoul(argv[i], &end, 10);
  191. if (*end == '\0') {
  192. if (nr == curr + 1)
  193. return true;
  194. continue;
  195. }
  196. if (strcasestr(desc, argv[i]))
  197. return true;
  198. }
  199. return false;
  200. }
  201. struct child_test {
  202. struct child_process process;
  203. struct test_suite *test;
  204. int test_num;
  205. int subtest;
  206. };
  207. static int run_test_child(struct child_process *process)
  208. {
  209. struct child_test *child = container_of(process, struct child_test, process);
  210. int err;
  211. pr_debug("--- start ---\n");
  212. pr_debug("test child forked, pid %d\n", getpid());
  213. err = test_function(child->test, child->subtest)(child->test, child->subtest);
  214. pr_debug("---- end(%d) ----\n", err);
  215. fflush(NULL);
  216. return -err;
  217. }
  218. static int print_test_result(struct test_suite *t, int i, int subtest, int result, int width)
  219. {
  220. if (has_subtests(t)) {
  221. int subw = width > 2 ? width - 2 : width;
  222. pr_info("%3d.%1d: %-*s:", i + 1, subtest + 1, subw, test_description(t, subtest));
  223. } else
  224. pr_info("%3d: %-*s:", i + 1, width, test_description(t, subtest));
  225. switch (result) {
  226. case TEST_OK:
  227. pr_info(" Ok\n");
  228. break;
  229. case TEST_SKIP: {
  230. const char *reason = skip_reason(t, subtest);
  231. if (reason)
  232. color_fprintf(stderr, PERF_COLOR_YELLOW, " Skip (%s)\n", reason);
  233. else
  234. color_fprintf(stderr, PERF_COLOR_YELLOW, " Skip\n");
  235. }
  236. break;
  237. case TEST_FAIL:
  238. default:
  239. color_fprintf(stderr, PERF_COLOR_RED, " FAILED!\n");
  240. break;
  241. }
  242. return 0;
  243. }
  244. static int finish_test(struct child_test *child_test, int width)
  245. {
  246. struct test_suite *t = child_test->test;
  247. int i = child_test->test_num;
  248. int subi = child_test->subtest;
  249. int err = child_test->process.err;
  250. bool err_done = err <= 0;
  251. struct strbuf err_output = STRBUF_INIT;
  252. int ret;
  253. /*
  254. * For test suites with subtests, display the suite name ahead of the
  255. * sub test names.
  256. */
  257. if (has_subtests(t) && subi == 0)
  258. pr_info("%3d: %-*s:\n", i + 1, width, test_description(t, -1));
  259. /*
  260. * Busy loop reading from the child's stdout/stderr that are set to be
  261. * non-blocking until EOF.
  262. */
  263. if (!err_done)
  264. fcntl(err, F_SETFL, O_NONBLOCK);
  265. if (verbose > 1) {
  266. if (has_subtests(t))
  267. pr_info("%3d.%1d: %s:\n", i + 1, subi + 1, test_description(t, subi));
  268. else
  269. pr_info("%3d: %s:\n", i + 1, test_description(t, -1));
  270. }
  271. while (!err_done) {
  272. struct pollfd pfds[1] = {
  273. { .fd = err,
  274. .events = POLLIN | POLLERR | POLLHUP | POLLNVAL,
  275. },
  276. };
  277. char buf[512];
  278. ssize_t len;
  279. /* Poll to avoid excessive spinning, timeout set for 100ms. */
  280. poll(pfds, ARRAY_SIZE(pfds), /*timeout=*/100);
  281. if (!err_done && pfds[0].revents) {
  282. errno = 0;
  283. len = read(err, buf, sizeof(buf) - 1);
  284. if (len <= 0) {
  285. err_done = errno != EAGAIN;
  286. } else {
  287. buf[len] = '\0';
  288. if (verbose > 1)
  289. fprintf(stdout, "%s", buf);
  290. else
  291. strbuf_addstr(&err_output, buf);
  292. }
  293. }
  294. }
  295. /* Clean up child process. */
  296. ret = finish_command(&child_test->process);
  297. if (verbose == 1 && ret == TEST_FAIL) {
  298. /* Add header for test that was skipped above. */
  299. if (has_subtests(t))
  300. pr_info("%3d.%1d: %s:\n", i + 1, subi + 1, test_description(t, subi));
  301. else
  302. pr_info("%3d: %s:\n", i + 1, test_description(t, -1));
  303. fprintf(stderr, "%s", err_output.buf);
  304. }
  305. strbuf_release(&err_output);
  306. print_test_result(t, i, subi, ret, width);
  307. if (err > 0)
  308. close(err);
  309. return 0;
  310. }
  311. static int start_test(struct test_suite *test, int i, int subi, struct child_test **child,
  312. int width)
  313. {
  314. int err;
  315. *child = NULL;
  316. if (dont_fork) {
  317. pr_debug("--- start ---\n");
  318. err = test_function(test, subi)(test, subi);
  319. pr_debug("---- end ----\n");
  320. print_test_result(test, i, subi, err, width);
  321. return 0;
  322. }
  323. *child = zalloc(sizeof(**child));
  324. if (!*child)
  325. return -ENOMEM;
  326. (*child)->test = test;
  327. (*child)->test_num = i;
  328. (*child)->subtest = subi;
  329. (*child)->process.pid = -1;
  330. (*child)->process.no_stdin = 1;
  331. if (verbose <= 0) {
  332. (*child)->process.no_stdout = 1;
  333. (*child)->process.no_stderr = 1;
  334. } else {
  335. (*child)->process.stdout_to_stderr = 1;
  336. (*child)->process.out = -1;
  337. (*child)->process.err = -1;
  338. }
  339. (*child)->process.no_exec_cmd = run_test_child;
  340. err = start_command(&(*child)->process);
  341. if (err || !sequential)
  342. return err;
  343. return finish_test(*child, width);
  344. }
  345. #define for_each_test(j, k, t) \
  346. for (j = 0, k = 0; j < ARRAY_SIZE(tests); j++, k = 0) \
  347. while ((t = tests[j][k++]) != NULL)
  348. static int __cmd_test(int argc, const char *argv[], struct intlist *skiplist)
  349. {
  350. struct test_suite *t;
  351. unsigned int j, k;
  352. int i = 0;
  353. int width = 0;
  354. size_t num_tests = 0;
  355. struct child_test **child_tests;
  356. int child_test_num = 0;
  357. for_each_test(j, k, t) {
  358. int len = strlen(test_description(t, -1));
  359. if (width < len)
  360. width = len;
  361. if (has_subtests(t)) {
  362. for (int subi = 0, subn = num_subtests(t); subi < subn; subi++) {
  363. len = strlen(test_description(t, subi));
  364. if (width < len)
  365. width = len;
  366. num_tests++;
  367. }
  368. } else {
  369. num_tests++;
  370. }
  371. }
  372. child_tests = calloc(num_tests, sizeof(*child_tests));
  373. if (!child_tests)
  374. return -ENOMEM;
  375. for_each_test(j, k, t) {
  376. int curr = i++;
  377. if (!perf_test__matches(test_description(t, -1), curr, argc, argv)) {
  378. bool skip = true;
  379. for (int subi = 0, subn = num_subtests(t); subi < subn; subi++) {
  380. if (perf_test__matches(test_description(t, subi),
  381. curr, argc, argv))
  382. skip = false;
  383. }
  384. if (skip)
  385. continue;
  386. }
  387. if (intlist__find(skiplist, i)) {
  388. pr_info("%3d: %-*s:", curr + 1, width, test_description(t, -1));
  389. color_fprintf(stderr, PERF_COLOR_YELLOW, " Skip (user override)\n");
  390. continue;
  391. }
  392. if (!has_subtests(t)) {
  393. int err = start_test(t, curr, -1, &child_tests[child_test_num++], width);
  394. if (err) {
  395. /* TODO: if !sequential waitpid the already forked children. */
  396. free(child_tests);
  397. return err;
  398. }
  399. } else {
  400. for (int subi = 0, subn = num_subtests(t); subi < subn; subi++) {
  401. int err;
  402. if (!perf_test__matches(test_description(t, subi),
  403. curr, argc, argv))
  404. continue;
  405. err = start_test(t, curr, subi, &child_tests[child_test_num++],
  406. width);
  407. if (err)
  408. return err;
  409. }
  410. }
  411. }
  412. for (i = 0; i < child_test_num; i++) {
  413. if (!sequential) {
  414. int ret = finish_test(child_tests[i], width);
  415. if (ret)
  416. return ret;
  417. }
  418. free(child_tests[i]);
  419. }
  420. free(child_tests);
  421. return 0;
  422. }
  423. static int perf_test__list(int argc, const char **argv)
  424. {
  425. unsigned int j, k;
  426. struct test_suite *t;
  427. int i = 0;
  428. for_each_test(j, k, t) {
  429. int curr = i++;
  430. if (!perf_test__matches(test_description(t, -1), curr, argc, argv))
  431. continue;
  432. pr_info("%3d: %s\n", i, test_description(t, -1));
  433. if (has_subtests(t)) {
  434. int subn = num_subtests(t);
  435. int subi;
  436. for (subi = 0; subi < subn; subi++)
  437. pr_info("%3d:%1d: %s\n", i, subi + 1,
  438. test_description(t, subi));
  439. }
  440. }
  441. return 0;
  442. }
  443. static int run_workload(const char *work, int argc, const char **argv)
  444. {
  445. unsigned int i = 0;
  446. struct test_workload *twl;
  447. for (i = 0; i < ARRAY_SIZE(workloads); i++) {
  448. twl = workloads[i];
  449. if (!strcmp(twl->name, work))
  450. return twl->func(argc, argv);
  451. }
  452. pr_info("No workload found: %s\n", work);
  453. return -1;
  454. }
  455. static int perf_test__config(const char *var, const char *value,
  456. void *data __maybe_unused)
  457. {
  458. if (!strcmp(var, "annotate.objdump"))
  459. test_objdump_path = value;
  460. return 0;
  461. }
  462. int cmd_test(int argc, const char **argv)
  463. {
  464. const char *test_usage[] = {
  465. "perf test [<options>] [{list <test-name-fragment>|[<test-name-fragments>|<test-numbers>]}]",
  466. NULL,
  467. };
  468. const char *skip = NULL;
  469. const char *workload = NULL;
  470. const struct option test_options[] = {
  471. OPT_STRING('s', "skip", &skip, "tests", "tests to skip"),
  472. OPT_INCR('v', "verbose", &verbose,
  473. "be more verbose (show symbol address, etc)"),
  474. OPT_BOOLEAN('F', "dont-fork", &dont_fork,
  475. "Do not fork for testcase"),
  476. OPT_BOOLEAN('p', "parallel", &parallel, "Run the tests in parallel"),
  477. OPT_BOOLEAN('S', "sequential", &sequential,
  478. "Run the tests one after another rather than in parallel"),
  479. OPT_STRING('w', "workload", &workload, "work", "workload to run for testing"),
  480. OPT_STRING(0, "dso", &dso_to_test, "dso", "dso to test"),
  481. OPT_STRING(0, "objdump", &test_objdump_path, "path",
  482. "objdump binary to use for disassembly and annotations"),
  483. OPT_END()
  484. };
  485. const char * const test_subcommands[] = { "list", NULL };
  486. struct intlist *skiplist = NULL;
  487. int ret = hists__init();
  488. if (ret < 0)
  489. return ret;
  490. perf_config(perf_test__config, NULL);
  491. /* Unbuffered output */
  492. setvbuf(stdout, NULL, _IONBF, 0);
  493. tests[2] = create_script_test_suites();
  494. argc = parse_options_subcommand(argc, argv, test_options, test_subcommands, test_usage, 0);
  495. if (argc >= 1 && !strcmp(argv[0], "list"))
  496. return perf_test__list(argc - 1, argv + 1);
  497. if (workload)
  498. return run_workload(workload, argc, argv);
  499. if (dont_fork)
  500. sequential = true;
  501. else if (parallel)
  502. sequential = false;
  503. symbol_conf.priv_size = sizeof(int);
  504. symbol_conf.try_vmlinux_path = true;
  505. if (symbol__init(NULL) < 0)
  506. return -1;
  507. if (skip != NULL)
  508. skiplist = intlist__new(skip);
  509. /*
  510. * Tests that create BPF maps, for instance, need more than the 64K
  511. * default:
  512. */
  513. rlimit__bump_memlock();
  514. return __cmd_test(argc, argv, skiplist);
  515. }