vdso_test_getrandom.c 8.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * Copyright (C) 2022-2024 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
  4. */
  5. #include <assert.h>
  6. #include <pthread.h>
  7. #include <stdint.h>
  8. #include <stdio.h>
  9. #include <stdlib.h>
  10. #include <string.h>
  11. #include <time.h>
  12. #include <unistd.h>
  13. #include <sched.h>
  14. #include <signal.h>
  15. #include <sys/auxv.h>
  16. #include <sys/mman.h>
  17. #include <sys/random.h>
  18. #include <sys/syscall.h>
  19. #include <sys/ptrace.h>
  20. #include <sys/wait.h>
  21. #include <sys/types.h>
  22. #include <linux/random.h>
  23. #include <linux/compiler.h>
  24. #include <linux/ptrace.h>
  25. #include "../kselftest.h"
  26. #include "parse_vdso.h"
  27. #include "vdso_config.h"
  28. #include "vdso_call.h"
  29. #ifndef timespecsub
  30. #define timespecsub(tsp, usp, vsp) \
  31. do { \
  32. (vsp)->tv_sec = (tsp)->tv_sec - (usp)->tv_sec; \
  33. (vsp)->tv_nsec = (tsp)->tv_nsec - (usp)->tv_nsec; \
  34. if ((vsp)->tv_nsec < 0) { \
  35. (vsp)->tv_sec--; \
  36. (vsp)->tv_nsec += 1000000000L; \
  37. } \
  38. } while (0)
  39. #endif
  40. #define ksft_assert(condition) \
  41. do { if (!(condition)) ksft_exit_fail_msg("Assertion failed: %s\n", #condition); } while (0)
  42. static struct {
  43. pthread_mutex_t lock;
  44. void **states;
  45. size_t len, cap;
  46. ssize_t(*fn)(void *, size_t, unsigned long, void *, size_t);
  47. struct vgetrandom_opaque_params params;
  48. } vgrnd = {
  49. .lock = PTHREAD_MUTEX_INITIALIZER
  50. };
  51. static void *vgetrandom_get_state(void)
  52. {
  53. void *state = NULL;
  54. pthread_mutex_lock(&vgrnd.lock);
  55. if (!vgrnd.len) {
  56. size_t page_size = getpagesize();
  57. size_t new_cap;
  58. size_t alloc_size, num = sysconf(_SC_NPROCESSORS_ONLN); /* Just a decent heuristic. */
  59. size_t state_size_aligned, cache_line_size = sysconf(_SC_LEVEL1_DCACHE_LINESIZE) ?: 1;
  60. void *new_block, *new_states;
  61. state_size_aligned = (vgrnd.params.size_of_opaque_state + cache_line_size - 1) & (~(cache_line_size - 1));
  62. alloc_size = (num * state_size_aligned + page_size - 1) & (~(page_size - 1));
  63. num = (page_size / state_size_aligned) * (alloc_size / page_size);
  64. new_block = mmap(0, alloc_size, vgrnd.params.mmap_prot, vgrnd.params.mmap_flags, -1, 0);
  65. if (new_block == MAP_FAILED)
  66. goto out;
  67. new_cap = vgrnd.cap + num;
  68. new_states = reallocarray(vgrnd.states, new_cap, sizeof(*vgrnd.states));
  69. if (!new_states)
  70. goto unmap;
  71. vgrnd.cap = new_cap;
  72. vgrnd.states = new_states;
  73. for (size_t i = 0; i < num; ++i) {
  74. if (((uintptr_t)new_block & (page_size - 1)) + vgrnd.params.size_of_opaque_state > page_size)
  75. new_block = (void *)(((uintptr_t)new_block + page_size - 1) & (~(page_size - 1)));
  76. vgrnd.states[i] = new_block;
  77. new_block += state_size_aligned;
  78. }
  79. vgrnd.len = num;
  80. goto success;
  81. unmap:
  82. munmap(new_block, alloc_size);
  83. goto out;
  84. }
  85. success:
  86. state = vgrnd.states[--vgrnd.len];
  87. out:
  88. pthread_mutex_unlock(&vgrnd.lock);
  89. return state;
  90. }
  91. static void vgetrandom_put_state(void *state)
  92. {
  93. if (!state)
  94. return;
  95. pthread_mutex_lock(&vgrnd.lock);
  96. vgrnd.states[vgrnd.len++] = state;
  97. pthread_mutex_unlock(&vgrnd.lock);
  98. }
  99. static void vgetrandom_init(void)
  100. {
  101. const char *version = versions[VDSO_VERSION];
  102. const char *name = names[VDSO_NAMES][6];
  103. unsigned long sysinfo_ehdr = getauxval(AT_SYSINFO_EHDR);
  104. ssize_t ret;
  105. if (!sysinfo_ehdr)
  106. ksft_exit_skip("AT_SYSINFO_EHDR is not present\n");
  107. vdso_init_from_sysinfo_ehdr(sysinfo_ehdr);
  108. vgrnd.fn = (__typeof__(vgrnd.fn))vdso_sym(version, name);
  109. if (!vgrnd.fn)
  110. ksft_exit_skip("%s@%s symbol is missing from vDSO\n", name, version);
  111. ret = VDSO_CALL(vgrnd.fn, 5, NULL, 0, 0, &vgrnd.params, ~0UL);
  112. if (ret == -ENOSYS)
  113. ksft_exit_skip("CPU does not have runtime support\n");
  114. else if (ret)
  115. ksft_exit_fail_msg("Failed to fetch vgetrandom params: %zd\n", ret);
  116. }
  117. static ssize_t vgetrandom(void *buf, size_t len, unsigned long flags)
  118. {
  119. static __thread void *state;
  120. if (!state) {
  121. state = vgetrandom_get_state();
  122. ksft_assert(state);
  123. }
  124. return VDSO_CALL(vgrnd.fn, 5, buf, len, flags, state, vgrnd.params.size_of_opaque_state);
  125. }
  126. enum { TRIALS = 25000000, THREADS = 256 };
  127. static void *test_vdso_getrandom(void *ctx)
  128. {
  129. for (size_t i = 0; i < TRIALS; ++i) {
  130. unsigned int val;
  131. ssize_t ret = vgetrandom(&val, sizeof(val), 0);
  132. ksft_assert(ret == sizeof(val));
  133. }
  134. return NULL;
  135. }
  136. static void *test_libc_getrandom(void *ctx)
  137. {
  138. for (size_t i = 0; i < TRIALS; ++i) {
  139. unsigned int val;
  140. ssize_t ret = getrandom(&val, sizeof(val), 0);
  141. ksft_assert(ret == sizeof(val));
  142. }
  143. return NULL;
  144. }
  145. static void *test_syscall_getrandom(void *ctx)
  146. {
  147. for (size_t i = 0; i < TRIALS; ++i) {
  148. unsigned int val;
  149. ssize_t ret = syscall(__NR_getrandom, &val, sizeof(val), 0);
  150. ksft_assert(ret == sizeof(val));
  151. }
  152. return NULL;
  153. }
  154. static void bench_single(void)
  155. {
  156. struct timespec start, end, diff;
  157. clock_gettime(CLOCK_MONOTONIC, &start);
  158. test_vdso_getrandom(NULL);
  159. clock_gettime(CLOCK_MONOTONIC, &end);
  160. timespecsub(&end, &start, &diff);
  161. printf(" vdso: %u times in %lu.%09lu seconds\n", TRIALS, diff.tv_sec, diff.tv_nsec);
  162. clock_gettime(CLOCK_MONOTONIC, &start);
  163. test_libc_getrandom(NULL);
  164. clock_gettime(CLOCK_MONOTONIC, &end);
  165. timespecsub(&end, &start, &diff);
  166. printf(" libc: %u times in %lu.%09lu seconds\n", TRIALS, diff.tv_sec, diff.tv_nsec);
  167. clock_gettime(CLOCK_MONOTONIC, &start);
  168. test_syscall_getrandom(NULL);
  169. clock_gettime(CLOCK_MONOTONIC, &end);
  170. timespecsub(&end, &start, &diff);
  171. printf("syscall: %u times in %lu.%09lu seconds\n", TRIALS, diff.tv_sec, diff.tv_nsec);
  172. }
  173. static void bench_multi(void)
  174. {
  175. struct timespec start, end, diff;
  176. pthread_t threads[THREADS];
  177. clock_gettime(CLOCK_MONOTONIC, &start);
  178. for (size_t i = 0; i < THREADS; ++i)
  179. ksft_assert(pthread_create(&threads[i], NULL, test_vdso_getrandom, NULL) == 0);
  180. for (size_t i = 0; i < THREADS; ++i)
  181. pthread_join(threads[i], NULL);
  182. clock_gettime(CLOCK_MONOTONIC, &end);
  183. timespecsub(&end, &start, &diff);
  184. printf(" vdso: %u x %u times in %lu.%09lu seconds\n", TRIALS, THREADS, diff.tv_sec, diff.tv_nsec);
  185. clock_gettime(CLOCK_MONOTONIC, &start);
  186. for (size_t i = 0; i < THREADS; ++i)
  187. ksft_assert(pthread_create(&threads[i], NULL, test_libc_getrandom, NULL) == 0);
  188. for (size_t i = 0; i < THREADS; ++i)
  189. pthread_join(threads[i], NULL);
  190. clock_gettime(CLOCK_MONOTONIC, &end);
  191. timespecsub(&end, &start, &diff);
  192. printf(" libc: %u x %u times in %lu.%09lu seconds\n", TRIALS, THREADS, diff.tv_sec, diff.tv_nsec);
  193. clock_gettime(CLOCK_MONOTONIC, &start);
  194. for (size_t i = 0; i < THREADS; ++i)
  195. ksft_assert(pthread_create(&threads[i], NULL, test_syscall_getrandom, NULL) == 0);
  196. for (size_t i = 0; i < THREADS; ++i)
  197. pthread_join(threads[i], NULL);
  198. clock_gettime(CLOCK_MONOTONIC, &end);
  199. timespecsub(&end, &start, &diff);
  200. printf(" syscall: %u x %u times in %lu.%09lu seconds\n", TRIALS, THREADS, diff.tv_sec, diff.tv_nsec);
  201. }
  202. static void fill(void)
  203. {
  204. uint8_t weird_size[323929];
  205. for (;;)
  206. vgetrandom(weird_size, sizeof(weird_size), 0);
  207. }
  208. static void kselftest(void)
  209. {
  210. uint8_t weird_size[1263];
  211. pid_t child;
  212. ksft_print_header();
  213. vgetrandom_init();
  214. ksft_set_plan(2);
  215. for (size_t i = 0; i < 1000; ++i) {
  216. ssize_t ret = vgetrandom(weird_size, sizeof(weird_size), 0);
  217. ksft_assert(ret == sizeof(weird_size));
  218. }
  219. ksft_test_result_pass("getrandom: PASS\n");
  220. unshare(CLONE_NEWUSER);
  221. ksft_assert(unshare(CLONE_NEWTIME) == 0);
  222. child = fork();
  223. ksft_assert(child >= 0);
  224. if (!child) {
  225. vgetrandom_init();
  226. child = getpid();
  227. ksft_assert(ptrace(PTRACE_TRACEME, 0, NULL, NULL) == 0);
  228. ksft_assert(kill(child, SIGSTOP) == 0);
  229. ksft_assert(vgetrandom(weird_size, sizeof(weird_size), 0) == sizeof(weird_size));
  230. _exit(0);
  231. }
  232. for (;;) {
  233. struct ptrace_syscall_info info = { 0 };
  234. int status, ret;
  235. ksft_assert(waitpid(child, &status, 0) >= 0);
  236. if (WIFEXITED(status)) {
  237. ksft_assert(WEXITSTATUS(status) == 0);
  238. break;
  239. }
  240. ksft_assert(WIFSTOPPED(status));
  241. if (WSTOPSIG(status) == SIGSTOP)
  242. ksft_assert(ptrace(PTRACE_SETOPTIONS, child, 0, PTRACE_O_TRACESYSGOOD) == 0);
  243. else if (WSTOPSIG(status) == (SIGTRAP | 0x80)) {
  244. ksft_assert(ptrace(PTRACE_GET_SYSCALL_INFO, child, sizeof(info), &info) > 0);
  245. if (info.op == PTRACE_SYSCALL_INFO_ENTRY && info.entry.nr == __NR_getrandom &&
  246. info.entry.args[0] == (uintptr_t)weird_size && info.entry.args[1] == sizeof(weird_size))
  247. ksft_exit_fail_msg("vgetrandom passed buffer to syscall getrandom unexpectedly\n");
  248. }
  249. ksft_assert(ptrace(PTRACE_SYSCALL, child, 0, 0) == 0);
  250. }
  251. ksft_test_result_pass("getrandom timens: PASS\n");
  252. ksft_exit_pass();
  253. }
  254. static void usage(const char *argv0)
  255. {
  256. fprintf(stderr, "Usage: %s [bench-single|bench-multi|fill]\n", argv0);
  257. }
  258. int main(int argc, char *argv[])
  259. {
  260. if (argc == 1) {
  261. kselftest();
  262. return 0;
  263. }
  264. if (argc != 2) {
  265. usage(argv[0]);
  266. return 1;
  267. }
  268. vgetrandom_init();
  269. if (!strcmp(argv[1], "bench-single"))
  270. bench_single();
  271. else if (!strcmp(argv[1], "bench-multi"))
  272. bench_multi();
  273. else if (!strcmp(argv[1], "fill"))
  274. fill();
  275. else {
  276. usage(argv[0]);
  277. return 1;
  278. }
  279. return 0;
  280. }