dirty_log_perf_test.c 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * KVM dirty page logging performance test
  4. *
  5. * Based on dirty_log_test.c
  6. *
  7. * Copyright (C) 2018, Red Hat, Inc.
  8. * Copyright (C) 2020, Google, Inc.
  9. */
  10. #include <stdio.h>
  11. #include <stdlib.h>
  12. #include <time.h>
  13. #include <pthread.h>
  14. #include <linux/bitmap.h>
  15. #include "kvm_util.h"
  16. #include "test_util.h"
  17. #include "memstress.h"
  18. #include "guest_modes.h"
  19. #include "ucall_common.h"
  20. #ifdef __aarch64__
  21. #include "aarch64/vgic.h"
  22. static int gic_fd;
  23. static void arch_setup_vm(struct kvm_vm *vm, unsigned int nr_vcpus)
  24. {
  25. /*
  26. * The test can still run even if hardware does not support GICv3, as it
  27. * is only an optimization to reduce guest exits.
  28. */
  29. gic_fd = vgic_v3_setup(vm, nr_vcpus, 64);
  30. }
  31. static void arch_cleanup_vm(struct kvm_vm *vm)
  32. {
  33. if (gic_fd > 0)
  34. close(gic_fd);
  35. }
  36. #else /* __aarch64__ */
  37. static void arch_setup_vm(struct kvm_vm *vm, unsigned int nr_vcpus)
  38. {
  39. }
  40. static void arch_cleanup_vm(struct kvm_vm *vm)
  41. {
  42. }
  43. #endif
  44. /* How many host loops to run by default (one KVM_GET_DIRTY_LOG for each loop)*/
  45. #define TEST_HOST_LOOP_N 2UL
  46. static int nr_vcpus = 1;
  47. static uint64_t guest_percpu_mem_size = DEFAULT_PER_VCPU_MEM_SIZE;
  48. static bool run_vcpus_while_disabling_dirty_logging;
  49. /* Host variables */
  50. static u64 dirty_log_manual_caps;
  51. static bool host_quit;
  52. static int iteration;
  53. static int vcpu_last_completed_iteration[KVM_MAX_VCPUS];
  54. static void vcpu_worker(struct memstress_vcpu_args *vcpu_args)
  55. {
  56. struct kvm_vcpu *vcpu = vcpu_args->vcpu;
  57. int vcpu_idx = vcpu_args->vcpu_idx;
  58. uint64_t pages_count = 0;
  59. struct kvm_run *run;
  60. struct timespec start;
  61. struct timespec ts_diff;
  62. struct timespec total = (struct timespec){0};
  63. struct timespec avg;
  64. int ret;
  65. run = vcpu->run;
  66. while (!READ_ONCE(host_quit)) {
  67. int current_iteration = READ_ONCE(iteration);
  68. clock_gettime(CLOCK_MONOTONIC, &start);
  69. ret = _vcpu_run(vcpu);
  70. ts_diff = timespec_elapsed(start);
  71. TEST_ASSERT(ret == 0, "vcpu_run failed: %d", ret);
  72. TEST_ASSERT(get_ucall(vcpu, NULL) == UCALL_SYNC,
  73. "Invalid guest sync status: exit_reason=%s",
  74. exit_reason_str(run->exit_reason));
  75. pr_debug("Got sync event from vCPU %d\n", vcpu_idx);
  76. vcpu_last_completed_iteration[vcpu_idx] = current_iteration;
  77. pr_debug("vCPU %d updated last completed iteration to %d\n",
  78. vcpu_idx, vcpu_last_completed_iteration[vcpu_idx]);
  79. if (current_iteration) {
  80. pages_count += vcpu_args->pages;
  81. total = timespec_add(total, ts_diff);
  82. pr_debug("vCPU %d iteration %d dirty memory time: %ld.%.9lds\n",
  83. vcpu_idx, current_iteration, ts_diff.tv_sec,
  84. ts_diff.tv_nsec);
  85. } else {
  86. pr_debug("vCPU %d iteration %d populate memory time: %ld.%.9lds\n",
  87. vcpu_idx, current_iteration, ts_diff.tv_sec,
  88. ts_diff.tv_nsec);
  89. }
  90. /*
  91. * Keep running the guest while dirty logging is being disabled
  92. * (iteration is negative) so that vCPUs are accessing memory
  93. * for the entire duration of zapping collapsible SPTEs.
  94. */
  95. while (current_iteration == READ_ONCE(iteration) &&
  96. READ_ONCE(iteration) >= 0 && !READ_ONCE(host_quit)) {}
  97. }
  98. avg = timespec_div(total, vcpu_last_completed_iteration[vcpu_idx]);
  99. pr_debug("\nvCPU %d dirtied 0x%lx pages over %d iterations in %ld.%.9lds. (Avg %ld.%.9lds/iteration)\n",
  100. vcpu_idx, pages_count, vcpu_last_completed_iteration[vcpu_idx],
  101. total.tv_sec, total.tv_nsec, avg.tv_sec, avg.tv_nsec);
  102. }
  103. struct test_params {
  104. unsigned long iterations;
  105. uint64_t phys_offset;
  106. bool partition_vcpu_memory_access;
  107. enum vm_mem_backing_src_type backing_src;
  108. int slots;
  109. uint32_t write_percent;
  110. bool random_access;
  111. };
  112. static void run_test(enum vm_guest_mode mode, void *arg)
  113. {
  114. struct test_params *p = arg;
  115. struct kvm_vm *vm;
  116. unsigned long **bitmaps;
  117. uint64_t guest_num_pages;
  118. uint64_t host_num_pages;
  119. uint64_t pages_per_slot;
  120. struct timespec start;
  121. struct timespec ts_diff;
  122. struct timespec get_dirty_log_total = (struct timespec){0};
  123. struct timespec vcpu_dirty_total = (struct timespec){0};
  124. struct timespec avg;
  125. struct timespec clear_dirty_log_total = (struct timespec){0};
  126. int i;
  127. vm = memstress_create_vm(mode, nr_vcpus, guest_percpu_mem_size,
  128. p->slots, p->backing_src,
  129. p->partition_vcpu_memory_access);
  130. memstress_set_write_percent(vm, p->write_percent);
  131. guest_num_pages = (nr_vcpus * guest_percpu_mem_size) >> vm->page_shift;
  132. guest_num_pages = vm_adjust_num_guest_pages(mode, guest_num_pages);
  133. host_num_pages = vm_num_host_pages(mode, guest_num_pages);
  134. pages_per_slot = host_num_pages / p->slots;
  135. bitmaps = memstress_alloc_bitmaps(p->slots, pages_per_slot);
  136. if (dirty_log_manual_caps)
  137. vm_enable_cap(vm, KVM_CAP_MANUAL_DIRTY_LOG_PROTECT2,
  138. dirty_log_manual_caps);
  139. arch_setup_vm(vm, nr_vcpus);
  140. /* Start the iterations */
  141. iteration = 0;
  142. host_quit = false;
  143. clock_gettime(CLOCK_MONOTONIC, &start);
  144. for (i = 0; i < nr_vcpus; i++)
  145. vcpu_last_completed_iteration[i] = -1;
  146. /*
  147. * Use 100% writes during the population phase to ensure all
  148. * memory is actually populated and not just mapped to the zero
  149. * page. The prevents expensive copy-on-write faults from
  150. * occurring during the dirty memory iterations below, which
  151. * would pollute the performance results.
  152. */
  153. memstress_set_write_percent(vm, 100);
  154. memstress_set_random_access(vm, false);
  155. memstress_start_vcpu_threads(nr_vcpus, vcpu_worker);
  156. /* Allow the vCPUs to populate memory */
  157. pr_debug("Starting iteration %d - Populating\n", iteration);
  158. for (i = 0; i < nr_vcpus; i++) {
  159. while (READ_ONCE(vcpu_last_completed_iteration[i]) !=
  160. iteration)
  161. ;
  162. }
  163. ts_diff = timespec_elapsed(start);
  164. pr_info("Populate memory time: %ld.%.9lds\n",
  165. ts_diff.tv_sec, ts_diff.tv_nsec);
  166. /* Enable dirty logging */
  167. clock_gettime(CLOCK_MONOTONIC, &start);
  168. memstress_enable_dirty_logging(vm, p->slots);
  169. ts_diff = timespec_elapsed(start);
  170. pr_info("Enabling dirty logging time: %ld.%.9lds\n\n",
  171. ts_diff.tv_sec, ts_diff.tv_nsec);
  172. memstress_set_write_percent(vm, p->write_percent);
  173. memstress_set_random_access(vm, p->random_access);
  174. while (iteration < p->iterations) {
  175. /*
  176. * Incrementing the iteration number will start the vCPUs
  177. * dirtying memory again.
  178. */
  179. clock_gettime(CLOCK_MONOTONIC, &start);
  180. iteration++;
  181. pr_debug("Starting iteration %d\n", iteration);
  182. for (i = 0; i < nr_vcpus; i++) {
  183. while (READ_ONCE(vcpu_last_completed_iteration[i])
  184. != iteration)
  185. ;
  186. }
  187. ts_diff = timespec_elapsed(start);
  188. vcpu_dirty_total = timespec_add(vcpu_dirty_total, ts_diff);
  189. pr_info("Iteration %d dirty memory time: %ld.%.9lds\n",
  190. iteration, ts_diff.tv_sec, ts_diff.tv_nsec);
  191. clock_gettime(CLOCK_MONOTONIC, &start);
  192. memstress_get_dirty_log(vm, bitmaps, p->slots);
  193. ts_diff = timespec_elapsed(start);
  194. get_dirty_log_total = timespec_add(get_dirty_log_total,
  195. ts_diff);
  196. pr_info("Iteration %d get dirty log time: %ld.%.9lds\n",
  197. iteration, ts_diff.tv_sec, ts_diff.tv_nsec);
  198. if (dirty_log_manual_caps) {
  199. clock_gettime(CLOCK_MONOTONIC, &start);
  200. memstress_clear_dirty_log(vm, bitmaps, p->slots,
  201. pages_per_slot);
  202. ts_diff = timespec_elapsed(start);
  203. clear_dirty_log_total = timespec_add(clear_dirty_log_total,
  204. ts_diff);
  205. pr_info("Iteration %d clear dirty log time: %ld.%.9lds\n",
  206. iteration, ts_diff.tv_sec, ts_diff.tv_nsec);
  207. }
  208. }
  209. /*
  210. * Run vCPUs while dirty logging is being disabled to stress disabling
  211. * in terms of both performance and correctness. Opt-in via command
  212. * line as this significantly increases time to disable dirty logging.
  213. */
  214. if (run_vcpus_while_disabling_dirty_logging)
  215. WRITE_ONCE(iteration, -1);
  216. /* Disable dirty logging */
  217. clock_gettime(CLOCK_MONOTONIC, &start);
  218. memstress_disable_dirty_logging(vm, p->slots);
  219. ts_diff = timespec_elapsed(start);
  220. pr_info("Disabling dirty logging time: %ld.%.9lds\n",
  221. ts_diff.tv_sec, ts_diff.tv_nsec);
  222. /*
  223. * Tell the vCPU threads to quit. No need to manually check that vCPUs
  224. * have stopped running after disabling dirty logging, the join will
  225. * wait for them to exit.
  226. */
  227. host_quit = true;
  228. memstress_join_vcpu_threads(nr_vcpus);
  229. avg = timespec_div(get_dirty_log_total, p->iterations);
  230. pr_info("Get dirty log over %lu iterations took %ld.%.9lds. (Avg %ld.%.9lds/iteration)\n",
  231. p->iterations, get_dirty_log_total.tv_sec,
  232. get_dirty_log_total.tv_nsec, avg.tv_sec, avg.tv_nsec);
  233. if (dirty_log_manual_caps) {
  234. avg = timespec_div(clear_dirty_log_total, p->iterations);
  235. pr_info("Clear dirty log over %lu iterations took %ld.%.9lds. (Avg %ld.%.9lds/iteration)\n",
  236. p->iterations, clear_dirty_log_total.tv_sec,
  237. clear_dirty_log_total.tv_nsec, avg.tv_sec, avg.tv_nsec);
  238. }
  239. memstress_free_bitmaps(bitmaps, p->slots);
  240. arch_cleanup_vm(vm);
  241. memstress_destroy_vm(vm);
  242. }
  243. static void help(char *name)
  244. {
  245. puts("");
  246. printf("usage: %s [-h] [-a] [-i iterations] [-p offset] [-g] "
  247. "[-m mode] [-n] [-b vcpu bytes] [-v vcpus] [-o] [-r random seed ] [-s mem type]"
  248. "[-x memslots] [-w percentage] [-c physical cpus to run test on]\n", name);
  249. puts("");
  250. printf(" -a: access memory randomly rather than in order.\n");
  251. printf(" -i: specify iteration counts (default: %"PRIu64")\n",
  252. TEST_HOST_LOOP_N);
  253. printf(" -g: Do not enable KVM_CAP_MANUAL_DIRTY_LOG_PROTECT2. This\n"
  254. " makes KVM_GET_DIRTY_LOG clear the dirty log (i.e.\n"
  255. " KVM_DIRTY_LOG_MANUAL_PROTECT_ENABLE is not enabled)\n"
  256. " and writes will be tracked as soon as dirty logging is\n"
  257. " enabled on the memslot (i.e. KVM_DIRTY_LOG_INITIALLY_SET\n"
  258. " is not enabled).\n");
  259. printf(" -p: specify guest physical test memory offset\n"
  260. " Warning: a low offset can conflict with the loaded test code.\n");
  261. guest_modes_help();
  262. printf(" -n: Run the vCPUs in nested mode (L2)\n");
  263. printf(" -e: Run vCPUs while dirty logging is being disabled. This\n"
  264. " can significantly increase runtime, especially if there\n"
  265. " isn't a dedicated pCPU for the main thread.\n");
  266. printf(" -b: specify the size of the memory region which should be\n"
  267. " dirtied by each vCPU. e.g. 10M or 3G.\n"
  268. " (default: 1G)\n");
  269. printf(" -v: specify the number of vCPUs to run.\n");
  270. printf(" -o: Overlap guest memory accesses instead of partitioning\n"
  271. " them into a separate region of memory for each vCPU.\n");
  272. printf(" -r: specify the starting random seed.\n");
  273. backing_src_help("-s");
  274. printf(" -x: Split the memory region into this number of memslots.\n"
  275. " (default: 1)\n");
  276. printf(" -w: specify the percentage of pages which should be written to\n"
  277. " as an integer from 0-100 inclusive. This is probabilistic,\n"
  278. " so -w X means each page has an X%% chance of writing\n"
  279. " and a (100-X)%% chance of reading.\n"
  280. " (default: 100 i.e. all pages are written to.)\n");
  281. kvm_print_vcpu_pinning_help();
  282. puts("");
  283. exit(0);
  284. }
  285. int main(int argc, char *argv[])
  286. {
  287. int max_vcpus = kvm_check_cap(KVM_CAP_MAX_VCPUS);
  288. const char *pcpu_list = NULL;
  289. struct test_params p = {
  290. .iterations = TEST_HOST_LOOP_N,
  291. .partition_vcpu_memory_access = true,
  292. .backing_src = DEFAULT_VM_MEM_SRC,
  293. .slots = 1,
  294. .write_percent = 100,
  295. };
  296. int opt;
  297. /* Override the seed to be deterministic by default. */
  298. guest_random_seed = 1;
  299. dirty_log_manual_caps =
  300. kvm_check_cap(KVM_CAP_MANUAL_DIRTY_LOG_PROTECT2);
  301. dirty_log_manual_caps &= (KVM_DIRTY_LOG_MANUAL_PROTECT_ENABLE |
  302. KVM_DIRTY_LOG_INITIALLY_SET);
  303. guest_modes_append_default();
  304. while ((opt = getopt(argc, argv, "ab:c:eghi:m:nop:r:s:v:x:w:")) != -1) {
  305. switch (opt) {
  306. case 'a':
  307. p.random_access = true;
  308. break;
  309. case 'b':
  310. guest_percpu_mem_size = parse_size(optarg);
  311. break;
  312. case 'c':
  313. pcpu_list = optarg;
  314. break;
  315. case 'e':
  316. /* 'e' is for evil. */
  317. run_vcpus_while_disabling_dirty_logging = true;
  318. break;
  319. case 'g':
  320. dirty_log_manual_caps = 0;
  321. break;
  322. case 'h':
  323. help(argv[0]);
  324. break;
  325. case 'i':
  326. p.iterations = atoi_positive("Number of iterations", optarg);
  327. break;
  328. case 'm':
  329. guest_modes_cmdline(optarg);
  330. break;
  331. case 'n':
  332. memstress_args.nested = true;
  333. break;
  334. case 'o':
  335. p.partition_vcpu_memory_access = false;
  336. break;
  337. case 'p':
  338. p.phys_offset = strtoull(optarg, NULL, 0);
  339. break;
  340. case 'r':
  341. guest_random_seed = atoi_positive("Random seed", optarg);
  342. break;
  343. case 's':
  344. p.backing_src = parse_backing_src_type(optarg);
  345. break;
  346. case 'v':
  347. nr_vcpus = atoi_positive("Number of vCPUs", optarg);
  348. TEST_ASSERT(nr_vcpus <= max_vcpus,
  349. "Invalid number of vcpus, must be between 1 and %d", max_vcpus);
  350. break;
  351. case 'w':
  352. p.write_percent = atoi_non_negative("Write percentage", optarg);
  353. TEST_ASSERT(p.write_percent <= 100,
  354. "Write percentage must be between 0 and 100");
  355. break;
  356. case 'x':
  357. p.slots = atoi_positive("Number of slots", optarg);
  358. break;
  359. default:
  360. help(argv[0]);
  361. break;
  362. }
  363. }
  364. if (pcpu_list) {
  365. kvm_parse_vcpu_pinning(pcpu_list, memstress_args.vcpu_to_pcpu,
  366. nr_vcpus);
  367. memstress_args.pin_vcpus = true;
  368. }
  369. TEST_ASSERT(p.iterations >= 2, "The test should have at least two iterations");
  370. pr_info("Test iterations: %"PRIu64"\n", p.iterations);
  371. for_each_guest_mode(run_test, &p);
  372. return 0;
  373. }