| // SPDX-License-Identifier: GPL-2.0 |
| /* |
| * KVM dirty logging page splitting test |
| * |
| * Based on dirty_log_perf.c |
| * |
| * Copyright (C) 2018, Red Hat, Inc. |
| * Copyright (C) 2023, Google, Inc. |
| */ |
| |
| #include <stdio.h> |
| #include <stdlib.h> |
| #include <pthread.h> |
| #include <linux/bitmap.h> |
| |
| #include "kvm_util.h" |
| #include "test_util.h" |
| #include "memstress.h" |
| #include "guest_modes.h" |
| |
| #define VCPUS 2 |
| #define SLOTS 2 |
| #define ITERATIONS 2 |
| |
| static uint64_t guest_percpu_mem_size = DEFAULT_PER_VCPU_MEM_SIZE; |
| |
| static enum vm_mem_backing_src_type backing_src = VM_MEM_SRC_ANONYMOUS_HUGETLB; |
| |
| static u64 dirty_log_manual_caps; |
| static bool host_quit; |
| static int iteration; |
| static int vcpu_last_completed_iteration[KVM_MAX_VCPUS]; |
| |
| struct kvm_page_stats { |
| uint64_t pages_4k; |
| uint64_t pages_2m; |
| uint64_t pages_1g; |
| uint64_t hugepages; |
| }; |
| |
| static void get_page_stats(struct kvm_vm *vm, struct kvm_page_stats *stats, const char *stage) |
| { |
| stats->pages_4k = vm_get_stat(vm, "pages_4k"); |
| stats->pages_2m = vm_get_stat(vm, "pages_2m"); |
| stats->pages_1g = vm_get_stat(vm, "pages_1g"); |
| stats->hugepages = stats->pages_2m + stats->pages_1g; |
| |
| pr_debug("\nPage stats after %s: 4K: %ld 2M: %ld 1G: %ld huge: %ld\n", |
| stage, stats->pages_4k, stats->pages_2m, stats->pages_1g, |
| stats->hugepages); |
| } |
| |
| static void run_vcpu_iteration(struct kvm_vm *vm) |
| { |
| int i; |
| |
| iteration++; |
| for (i = 0; i < VCPUS; i++) { |
| while (READ_ONCE(vcpu_last_completed_iteration[i]) != |
| iteration) |
| ; |
| } |
| } |
| |
| static void vcpu_worker(struct memstress_vcpu_args *vcpu_args) |
| { |
| struct kvm_vcpu *vcpu = vcpu_args->vcpu; |
| int vcpu_idx = vcpu_args->vcpu_idx; |
| |
| while (!READ_ONCE(host_quit)) { |
| int current_iteration = READ_ONCE(iteration); |
| |
| vcpu_run(vcpu); |
| |
| TEST_ASSERT_EQ(get_ucall(vcpu, NULL), UCALL_SYNC); |
| |
| vcpu_last_completed_iteration[vcpu_idx] = current_iteration; |
| |
| /* Wait for the start of the next iteration to be signaled. */ |
| while (current_iteration == READ_ONCE(iteration) && |
| READ_ONCE(iteration) >= 0 && |
| !READ_ONCE(host_quit)) |
| ; |
| } |
| } |
| |
| static void run_test(enum vm_guest_mode mode, void *unused) |
| { |
| struct kvm_vm *vm; |
| unsigned long **bitmaps; |
| uint64_t guest_num_pages; |
| uint64_t host_num_pages; |
| uint64_t pages_per_slot; |
| int i; |
| struct kvm_page_stats stats_populated; |
| struct kvm_page_stats stats_dirty_logging_enabled; |
| struct kvm_page_stats stats_dirty_pass[ITERATIONS]; |
| struct kvm_page_stats stats_clear_pass[ITERATIONS]; |
| struct kvm_page_stats stats_dirty_logging_disabled; |
| struct kvm_page_stats stats_repopulated; |
| |
| vm = memstress_create_vm(mode, VCPUS, guest_percpu_mem_size, |
| SLOTS, backing_src, false); |
| |
| guest_num_pages = (VCPUS * guest_percpu_mem_size) >> vm->page_shift; |
| guest_num_pages = vm_adjust_num_guest_pages(mode, guest_num_pages); |
| host_num_pages = vm_num_host_pages(mode, guest_num_pages); |
| pages_per_slot = host_num_pages / SLOTS; |
| TEST_ASSERT_EQ(host_num_pages, pages_per_slot * SLOTS); |
| TEST_ASSERT(!(host_num_pages % 512), |
| "Number of pages, '%lu' not a multiple of 2MiB", host_num_pages); |
| |
| bitmaps = memstress_alloc_bitmaps(SLOTS, pages_per_slot); |
| |
| if (dirty_log_manual_caps) |
| vm_enable_cap(vm, KVM_CAP_MANUAL_DIRTY_LOG_PROTECT2, |
| dirty_log_manual_caps); |
| |
| /* Start the iterations */ |
| iteration = -1; |
| host_quit = false; |
| |
| for (i = 0; i < VCPUS; i++) |
| vcpu_last_completed_iteration[i] = -1; |
| |
| memstress_start_vcpu_threads(VCPUS, vcpu_worker); |
| |
| run_vcpu_iteration(vm); |
| get_page_stats(vm, &stats_populated, "populating memory"); |
| |
| /* Enable dirty logging */ |
| memstress_enable_dirty_logging(vm, SLOTS); |
| |
| get_page_stats(vm, &stats_dirty_logging_enabled, "enabling dirty logging"); |
| |
| while (iteration < ITERATIONS) { |
| run_vcpu_iteration(vm); |
| get_page_stats(vm, &stats_dirty_pass[iteration - 1], |
| "dirtying memory"); |
| |
| memstress_get_dirty_log(vm, bitmaps, SLOTS); |
| |
| if (dirty_log_manual_caps) { |
| memstress_clear_dirty_log(vm, bitmaps, SLOTS, pages_per_slot); |
| |
| get_page_stats(vm, &stats_clear_pass[iteration - 1], "clearing dirty log"); |
| } |
| } |
| |
| /* Disable dirty logging */ |
| memstress_disable_dirty_logging(vm, SLOTS); |
| |
| get_page_stats(vm, &stats_dirty_logging_disabled, "disabling dirty logging"); |
| |
| /* Run vCPUs again to fault pages back in. */ |
| run_vcpu_iteration(vm); |
| get_page_stats(vm, &stats_repopulated, "repopulating memory"); |
| |
| /* |
| * Tell the vCPU threads to quit. No need to manually check that vCPUs |
| * have stopped running after disabling dirty logging, the join will |
| * wait for them to exit. |
| */ |
| host_quit = true; |
| memstress_join_vcpu_threads(VCPUS); |
| |
| memstress_free_bitmaps(bitmaps, SLOTS); |
| memstress_destroy_vm(vm); |
| |
| TEST_ASSERT_EQ((stats_populated.pages_2m * 512 + |
| stats_populated.pages_1g * 512 * 512), host_num_pages); |
| |
| /* |
| * Check that all huge pages were split. Since large pages can only |
| * exist in the data slot, and the vCPUs should have dirtied all pages |
| * in the data slot, there should be no huge pages left after splitting. |
| * Splitting happens at dirty log enable time without |
| * KVM_CAP_MANUAL_DIRTY_LOG_PROTECT2 and after the first clear pass |
| * with that capability. |
| */ |
| if (dirty_log_manual_caps) { |
| TEST_ASSERT_EQ(stats_clear_pass[0].hugepages, 0); |
| TEST_ASSERT(stats_clear_pass[0].pages_4k >= host_num_pages, |
| "Expected at least '%lu' 4KiB pages, found only '%lu'", |
| host_num_pages, stats_clear_pass[0].pages_4k); |
| TEST_ASSERT_EQ(stats_dirty_logging_enabled.hugepages, stats_populated.hugepages); |
| } else { |
| TEST_ASSERT_EQ(stats_dirty_logging_enabled.hugepages, 0); |
| TEST_ASSERT(stats_dirty_logging_enabled.pages_4k >= host_num_pages, |
| "Expected at least '%lu' 4KiB pages, found only '%lu'", |
| host_num_pages, stats_dirty_logging_enabled.pages_4k); |
| } |
| |
| /* |
| * Once dirty logging is disabled and the vCPUs have touched all their |
| * memory again, the hugepage counts should be the same as they were |
| * right after initial population of memory. |
| */ |
| TEST_ASSERT_EQ(stats_populated.pages_2m, stats_repopulated.pages_2m); |
| TEST_ASSERT_EQ(stats_populated.pages_1g, stats_repopulated.pages_1g); |
| } |
| |
| static void help(char *name) |
| { |
| puts(""); |
| printf("usage: %s [-h] [-b vcpu bytes] [-s mem type]\n", |
| name); |
| puts(""); |
| printf(" -b: specify the size of the memory region which should be\n" |
| " dirtied by each vCPU. e.g. 10M or 3G.\n" |
| " (default: 1G)\n"); |
| backing_src_help("-s"); |
| puts(""); |
| } |
| |
| int main(int argc, char *argv[]) |
| { |
| int opt; |
| |
| TEST_REQUIRE(get_kvm_param_bool("eager_page_split")); |
| TEST_REQUIRE(get_kvm_param_bool("tdp_mmu")); |
| |
| while ((opt = getopt(argc, argv, "b:hs:")) != -1) { |
| switch (opt) { |
| case 'b': |
| guest_percpu_mem_size = parse_size(optarg); |
| break; |
| case 'h': |
| help(argv[0]); |
| exit(0); |
| case 's': |
| backing_src = parse_backing_src_type(optarg); |
| break; |
| default: |
| help(argv[0]); |
| exit(1); |
| } |
| } |
| |
| if (!is_backing_src_hugetlb(backing_src)) { |
| pr_info("This test will only work reliably with HugeTLB memory. " |
| "It can work with THP, but that is best effort.\n"); |
| } |
| |
| guest_modes_append_default(); |
| |
| dirty_log_manual_caps = 0; |
| for_each_guest_mode(run_test, NULL); |
| |
| dirty_log_manual_caps = |
| kvm_check_cap(KVM_CAP_MANUAL_DIRTY_LOG_PROTECT2); |
| |
| if (dirty_log_manual_caps) { |
| dirty_log_manual_caps &= (KVM_DIRTY_LOG_MANUAL_PROTECT_ENABLE | |
| KVM_DIRTY_LOG_INITIALLY_SET); |
| for_each_guest_mode(run_test, NULL); |
| } else { |
| pr_info("Skipping testing with MANUAL_PROTECT as it is not supported"); |
| } |
| |
| return 0; |
| } |