1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * builtin-test.c
4 *
5 * Builtin regression testing command: ever growing number of sanity tests
6 */
7 #include <fcntl.h>
8 #include <errno.h>
9 #include <unistd.h>
10 #include <string.h>
11 #include <stdlib.h>
12 #include <sys/types.h>
13 #include <dirent.h>
14 #include <sys/wait.h>
15 #include <sys/stat.h>
16 #include "builtin.h"
17 #include "hist.h"
18 #include "intlist.h"
19 #include "tests.h"
20 #include "debug.h"
21 #include "color.h"
22 #include <subcmd/parse-options.h>
23 #include "string2.h"
24 #include "symbol.h"
25 #include "util/rlimit.h"
26 #include <linux/kernel.h>
27 #include <linux/string.h>
28 #include <subcmd/exec-cmd.h>
29 #include <linux/zalloc.h>
30
31 #include "builtin-test-list.h"
32
33 static bool dont_fork;
34 const char *dso_to_test;
35
36 struct test_suite *__weak arch_tests[] = {
37 NULL,
38 };
39
40 static struct test_suite *generic_tests[] = {
41 &suite__vmlinux_matches_kallsyms,
42 #ifdef HAVE_LIBTRACEEVENT
43 &suite__openat_syscall_event,
44 &suite__openat_syscall_event_on_all_cpus,
45 &suite__basic_mmap,
46 #endif
47 &suite__mem,
48 &suite__parse_events,
49 &suite__expr,
50 &suite__PERF_RECORD,
51 &suite__pmu,
52 &suite__pmu_events,
53 &suite__dso_data,
54 &suite__dso_data_cache,
55 &suite__dso_data_reopen,
56 &suite__perf_evsel__roundtrip_name_test,
57 #ifdef HAVE_LIBTRACEEVENT
58 &suite__perf_evsel__tp_sched_test,
59 &suite__syscall_openat_tp_fields,
60 #endif
61 &suite__attr,
62 &suite__hists_link,
63 &suite__python_use,
64 &suite__bp_signal,
65 &suite__bp_signal_overflow,
66 &suite__bp_accounting,
67 &suite__wp,
68 &suite__task_exit,
69 &suite__sw_clock_freq,
70 &suite__code_reading,
71 &suite__sample_parsing,
72 &suite__keep_tracking,
73 &suite__parse_no_sample_id_all,
74 &suite__hists_filter,
75 &suite__mmap_thread_lookup,
76 &suite__thread_maps_share,
77 &suite__hists_output,
78 &suite__hists_cumulate,
79 #ifdef HAVE_LIBTRACEEVENT
80 &suite__switch_tracking,
81 #endif
82 &suite__fdarray__filter,
83 &suite__fdarray__add,
84 &suite__kmod_path__parse,
85 &suite__thread_map,
86 &suite__llvm,
87 &suite__session_topology,
88 &suite__bpf,
89 &suite__thread_map_synthesize,
90 &suite__thread_map_remove,
91 &suite__cpu_map_synthesize,
92 &suite__synthesize_stat_config,
93 &suite__synthesize_stat,
94 &suite__synthesize_stat_round,
95 &suite__event_update,
96 &suite__event_times,
97 &suite__backward_ring_buffer,
98 &suite__cpu_map_print,
99 &suite__cpu_map_merge,
100 &suite__sdt_event,
101 &suite__is_printable_array,
102 &suite__bitmap_print,
103 &suite__perf_hooks,
104 &suite__clang,
105 &suite__unit_number__scnprint,
106 &suite__mem2node,
107 &suite__time_utils,
108 &suite__jit_write_elf,
109 &suite__pfm,
110 &suite__api_io,
111 &suite__maps__merge_in,
112 &suite__demangle_java,
113 &suite__demangle_ocaml,
114 &suite__parse_metric,
115 &suite__pe_file_parsing,
116 &suite__expand_cgroup_events,
117 &suite__perf_time_to_tsc,
118 &suite__dlfilter,
119 &suite__sigtrap,
120 &suite__event_groups,
121 &suite__symbols,
122 NULL,
123 };
124
125 static struct test_suite **tests[] = {
126 generic_tests,
127 arch_tests,
128 };
129
130 static struct test_workload *workloads[] = {
131 &workload__noploop,
132 &workload__thloop,
133 &workload__leafloop,
134 &workload__sqrtloop,
135 &workload__brstack,
136 &workload__datasym,
137 };
138
num_subtests(const struct test_suite * t)139 static int num_subtests(const struct test_suite *t)
140 {
141 int num;
142
143 if (!t->test_cases)
144 return 0;
145
146 num = 0;
147 while (t->test_cases[num].name)
148 num++;
149
150 return num;
151 }
152
has_subtests(const struct test_suite * t)153 static bool has_subtests(const struct test_suite *t)
154 {
155 return num_subtests(t) > 1;
156 }
157
skip_reason(const struct test_suite * t,int subtest)158 static const char *skip_reason(const struct test_suite *t, int subtest)
159 {
160 if (!t->test_cases)
161 return NULL;
162
163 return t->test_cases[subtest >= 0 ? subtest : 0].skip_reason;
164 }
165
test_description(const struct test_suite * t,int subtest)166 static const char *test_description(const struct test_suite *t, int subtest)
167 {
168 if (t->test_cases && subtest >= 0)
169 return t->test_cases[subtest].desc;
170
171 return t->desc;
172 }
173
test_function(const struct test_suite * t,int subtest)174 static test_fnptr test_function(const struct test_suite *t, int subtest)
175 {
176 if (subtest <= 0)
177 return t->test_cases[0].run_case;
178
179 return t->test_cases[subtest].run_case;
180 }
181
perf_test__matches(const char * desc,int curr,int argc,const char * argv[])182 static bool perf_test__matches(const char *desc, int curr, int argc, const char *argv[])
183 {
184 int i;
185
186 if (argc == 0)
187 return true;
188
189 for (i = 0; i < argc; ++i) {
190 char *end;
191 long nr = strtoul(argv[i], &end, 10);
192
193 if (*end == '\0') {
194 if (nr == curr + 1)
195 return true;
196 continue;
197 }
198
199 if (strcasestr(desc, argv[i]))
200 return true;
201 }
202
203 return false;
204 }
205
run_test(struct test_suite * test,int subtest)206 static int run_test(struct test_suite *test, int subtest)
207 {
208 int status, err = -1, child = dont_fork ? 0 : fork();
209 char sbuf[STRERR_BUFSIZE];
210
211 if (child < 0) {
212 pr_err("failed to fork test: %s\n",
213 str_error_r(errno, sbuf, sizeof(sbuf)));
214 return -1;
215 }
216
217 if (!child) {
218 if (!dont_fork) {
219 pr_debug("test child forked, pid %d\n", getpid());
220
221 if (verbose <= 0) {
222 int nullfd = open("/dev/null", O_WRONLY);
223
224 if (nullfd >= 0) {
225 close(STDERR_FILENO);
226 close(STDOUT_FILENO);
227
228 dup2(nullfd, STDOUT_FILENO);
229 dup2(STDOUT_FILENO, STDERR_FILENO);
230 close(nullfd);
231 }
232 } else {
233 signal(SIGSEGV, sighandler_dump_stack);
234 signal(SIGFPE, sighandler_dump_stack);
235 }
236 }
237
238 err = test_function(test, subtest)(test, subtest);
239 if (!dont_fork)
240 exit(err);
241 }
242
243 if (!dont_fork) {
244 wait(&status);
245
246 if (WIFEXITED(status)) {
247 err = (signed char)WEXITSTATUS(status);
248 pr_debug("test child finished with %d\n", err);
249 } else if (WIFSIGNALED(status)) {
250 err = -1;
251 pr_debug("test child interrupted\n");
252 }
253 }
254
255 return err;
256 }
257
258 #define for_each_test(j, k, t) \
259 for (j = 0; j < ARRAY_SIZE(tests); j++) \
260 for (k = 0, t = tests[j][k]; tests[j][k]; k++, t = tests[j][k])
261
test_and_print(struct test_suite * t,int subtest)262 static int test_and_print(struct test_suite *t, int subtest)
263 {
264 int err;
265
266 pr_debug("\n--- start ---\n");
267 err = run_test(t, subtest);
268 pr_debug("---- end ----\n");
269
270 if (!has_subtests(t))
271 pr_debug("%s:", t->desc);
272 else
273 pr_debug("%s subtest %d:", t->desc, subtest + 1);
274
275 switch (err) {
276 case TEST_OK:
277 pr_info(" Ok\n");
278 break;
279 case TEST_SKIP: {
280 const char *reason = skip_reason(t, subtest);
281
282 if (reason)
283 color_fprintf(stderr, PERF_COLOR_YELLOW, " Skip (%s)\n", reason);
284 else
285 color_fprintf(stderr, PERF_COLOR_YELLOW, " Skip\n");
286 }
287 break;
288 case TEST_FAIL:
289 default:
290 color_fprintf(stderr, PERF_COLOR_RED, " FAILED!\n");
291 break;
292 }
293
294 return err;
295 }
296
297 struct shell_test {
298 const char *dir;
299 const char *file;
300 };
301
shell_test__run(struct test_suite * test,int subdir __maybe_unused)302 static int shell_test__run(struct test_suite *test, int subdir __maybe_unused)
303 {
304 int err;
305 char script[PATH_MAX];
306 struct shell_test *st = test->priv;
307
308 path__join(script, sizeof(script) - 3, st->dir, st->file);
309
310 if (verbose > 0)
311 strncat(script, " -v", sizeof(script) - strlen(script) - 1);
312
313 err = system(script);
314 if (!err)
315 return TEST_OK;
316
317 return WEXITSTATUS(err) == 2 ? TEST_SKIP : TEST_FAIL;
318 }
319
run_shell_tests(int argc,const char * argv[],int i,int width,struct intlist * skiplist)320 static int run_shell_tests(int argc, const char *argv[], int i, int width,
321 struct intlist *skiplist)
322 {
323 struct shell_test st;
324 const struct script_file *files, *file;
325
326 files = list_script_files();
327 if (!files)
328 return 0;
329 for (file = files; file->dir; file++) {
330 int curr = i++;
331 struct test_case test_cases[] = {
332 {
333 .desc = file->desc,
334 .run_case = shell_test__run,
335 },
336 { .name = NULL, }
337 };
338 struct test_suite test_suite = {
339 .desc = test_cases[0].desc,
340 .test_cases = test_cases,
341 .priv = &st,
342 };
343 st.dir = file->dir;
344
345 if (test_suite.desc == NULL ||
346 !perf_test__matches(test_suite.desc, curr, argc, argv))
347 continue;
348
349 st.file = file->file;
350 pr_info("%3d: %-*s:", i, width, test_suite.desc);
351
352 if (intlist__find(skiplist, i)) {
353 color_fprintf(stderr, PERF_COLOR_YELLOW, " Skip (user override)\n");
354 continue;
355 }
356
357 test_and_print(&test_suite, 0);
358 }
359 return 0;
360 }
361
__cmd_test(int argc,const char * argv[],struct intlist * skiplist)362 static int __cmd_test(int argc, const char *argv[], struct intlist *skiplist)
363 {
364 struct test_suite *t;
365 unsigned int j, k;
366 int i = 0;
367 int width = list_script_max_width();
368
369 for_each_test(j, k, t) {
370 int len = strlen(test_description(t, -1));
371
372 if (width < len)
373 width = len;
374 }
375
376 for_each_test(j, k, t) {
377 int curr = i++;
378 int subi;
379
380 if (!perf_test__matches(test_description(t, -1), curr, argc, argv)) {
381 bool skip = true;
382 int subn;
383
384 subn = num_subtests(t);
385
386 for (subi = 0; subi < subn; subi++) {
387 if (perf_test__matches(test_description(t, subi),
388 curr, argc, argv))
389 skip = false;
390 }
391
392 if (skip)
393 continue;
394 }
395
396 pr_info("%3d: %-*s:", i, width, test_description(t, -1));
397
398 if (intlist__find(skiplist, i)) {
399 color_fprintf(stderr, PERF_COLOR_YELLOW, " Skip (user override)\n");
400 continue;
401 }
402
403 if (!has_subtests(t)) {
404 test_and_print(t, -1);
405 } else {
406 int subn = num_subtests(t);
407 /*
408 * minus 2 to align with normal testcases.
409 * For subtest we print additional '.x' in number.
410 * for example:
411 *
412 * 35: Test LLVM searching and compiling :
413 * 35.1: Basic BPF llvm compiling test : Ok
414 */
415 int subw = width > 2 ? width - 2 : width;
416
417 if (subn <= 0) {
418 color_fprintf(stderr, PERF_COLOR_YELLOW,
419 " Skip (not compiled in)\n");
420 continue;
421 }
422 pr_info("\n");
423
424 for (subi = 0; subi < subn; subi++) {
425 int len = strlen(test_description(t, subi));
426
427 if (subw < len)
428 subw = len;
429 }
430
431 for (subi = 0; subi < subn; subi++) {
432 if (!perf_test__matches(test_description(t, subi),
433 curr, argc, argv))
434 continue;
435
436 pr_info("%3d.%1d: %-*s:", i, subi + 1, subw,
437 test_description(t, subi));
438 test_and_print(t, subi);
439 }
440 }
441 }
442
443 return run_shell_tests(argc, argv, i, width, skiplist);
444 }
445
perf_test__list_shell(int argc,const char ** argv,int i)446 static int perf_test__list_shell(int argc, const char **argv, int i)
447 {
448 const struct script_file *files, *file;
449
450 files = list_script_files();
451 if (!files)
452 return 0;
453 for (file = files; file->dir; file++) {
454 int curr = i++;
455 struct test_suite t = {
456 .desc = file->desc
457 };
458
459 if (!perf_test__matches(t.desc, curr, argc, argv))
460 continue;
461
462 pr_info("%3d: %s\n", i, t.desc);
463 }
464 return 0;
465 }
466
perf_test__list(int argc,const char ** argv)467 static int perf_test__list(int argc, const char **argv)
468 {
469 unsigned int j, k;
470 struct test_suite *t;
471 int i = 0;
472
473 for_each_test(j, k, t) {
474 int curr = i++;
475
476 if (!perf_test__matches(test_description(t, -1), curr, argc, argv))
477 continue;
478
479 pr_info("%3d: %s\n", i, test_description(t, -1));
480
481 if (has_subtests(t)) {
482 int subn = num_subtests(t);
483 int subi;
484
485 for (subi = 0; subi < subn; subi++)
486 pr_info("%3d:%1d: %s\n", i, subi + 1,
487 test_description(t, subi));
488 }
489 }
490
491 perf_test__list_shell(argc, argv, i);
492
493 return 0;
494 }
495
run_workload(const char * work,int argc,const char ** argv)496 static int run_workload(const char *work, int argc, const char **argv)
497 {
498 unsigned int i = 0;
499 struct test_workload *twl;
500
501 for (i = 0; i < ARRAY_SIZE(workloads); i++) {
502 twl = workloads[i];
503 if (!strcmp(twl->name, work))
504 return twl->func(argc, argv);
505 }
506
507 pr_info("No workload found: %s\n", work);
508 return -1;
509 }
510
cmd_test(int argc,const char ** argv)511 int cmd_test(int argc, const char **argv)
512 {
513 const char *test_usage[] = {
514 "perf test [<options>] [{list <test-name-fragment>|[<test-name-fragments>|<test-numbers>]}]",
515 NULL,
516 };
517 const char *skip = NULL;
518 const char *workload = NULL;
519 const struct option test_options[] = {
520 OPT_STRING('s', "skip", &skip, "tests", "tests to skip"),
521 OPT_INCR('v', "verbose", &verbose,
522 "be more verbose (show symbol address, etc)"),
523 OPT_BOOLEAN('F', "dont-fork", &dont_fork,
524 "Do not fork for testcase"),
525 OPT_STRING('w', "workload", &workload, "work", "workload to run for testing"),
526 OPT_STRING(0, "dso", &dso_to_test, "dso", "dso to test"),
527 OPT_END()
528 };
529 const char * const test_subcommands[] = { "list", NULL };
530 struct intlist *skiplist = NULL;
531 int ret = hists__init();
532
533 if (ret < 0)
534 return ret;
535
536 /* Unbuffered output */
537 setvbuf(stdout, NULL, _IONBF, 0);
538
539 argc = parse_options_subcommand(argc, argv, test_options, test_subcommands, test_usage, 0);
540 if (argc >= 1 && !strcmp(argv[0], "list"))
541 return perf_test__list(argc - 1, argv + 1);
542
543 if (workload)
544 return run_workload(workload, argc, argv);
545
546 symbol_conf.priv_size = sizeof(int);
547 symbol_conf.sort_by_name = true;
548 symbol_conf.try_vmlinux_path = true;
549
550 if (symbol__init(NULL) < 0)
551 return -1;
552
553 if (skip != NULL)
554 skiplist = intlist__new(skip);
555 /*
556 * Tests that create BPF maps, for instance, need more than the 64K
557 * default:
558 */
559 rlimit__bump_memlock();
560
561 return __cmd_test(argc, argv, skiplist);
562 }
563