| #include "linux/err.h" | |
| #include "kvm/kvm-cpu.h" | |
| #include "kvm/symbol.h" | |
| #include "kvm/util.h" | |
| #include "kvm/kvm.h" | |
| #include "kvm/virtio.h" | |
| #include "kvm/mutex.h" | |
| #include "kvm/barrier.h" | |
| #include <sys/ioctl.h> | |
| #include <sys/mman.h> | |
| #include <sys/eventfd.h> | |
| #include <signal.h> | |
| #include <stdlib.h> | |
| #include <string.h> | |
| #include <errno.h> | |
| #include <stdio.h> | |
| extern __thread struct kvm_cpu *current_kvm_cpu; | |
| int __attribute__((weak)) kvm_cpu__get_endianness(struct kvm_cpu *vcpu) | |
| { | |
| return VIRTIO_ENDIAN_HOST; | |
| } | |
| void kvm_cpu__enable_singlestep(struct kvm_cpu *vcpu) | |
| { | |
| struct kvm_guest_debug debug = { | |
| .control = KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_SINGLESTEP, | |
| }; | |
| if (ioctl(vcpu->vcpu_fd, KVM_SET_GUEST_DEBUG, &debug) < 0) | |
| pr_warning("KVM_SET_GUEST_DEBUG failed"); | |
| } | |
| void kvm_cpu__run(struct kvm_cpu *vcpu) | |
| { | |
| int err; | |
| if (!vcpu->is_running) | |
| return; | |
| err = ioctl(vcpu->vcpu_fd, KVM_RUN, 0); | |
| if (err < 0 && (errno != EINTR && errno != EAGAIN)) | |
| die_perror("KVM_RUN failed"); | |
| } | |
| static void kvm_cpu_signal_handler(int signum) | |
| { | |
| if (signum == SIGKVMEXIT) { | |
| if (current_kvm_cpu && current_kvm_cpu->is_running) | |
| current_kvm_cpu->is_running = false; | |
| } else if (signum == SIGKVMPAUSE) { | |
| if (current_kvm_cpu->paused) | |
| die("Pause signaled for already paused CPU\n"); | |
| /* pause_lock is held by kvm__pause() */ | |
| current_kvm_cpu->paused = 1; | |
| /* | |
| * This is a blocking function and uses locks. It is safe | |
| * to call it for this signal as a second pause event should | |
| * not be send to this thread until it acquires and releases | |
| * the pause_lock. | |
| */ | |
| kvm__notify_paused(); | |
| } | |
| /* For SIGKVMTASK cpu->task is already set */ | |
| } | |
| static void kvm_cpu__handle_coalesced_mmio(struct kvm_cpu *cpu) | |
| { | |
| if (cpu->ring) { | |
| while (cpu->ring->first != cpu->ring->last) { | |
| struct kvm_coalesced_mmio *m; | |
| m = &cpu->ring->coalesced_mmio[cpu->ring->first]; | |
| kvm_cpu__emulate_mmio(cpu, | |
| m->phys_addr, | |
| m->data, | |
| m->len, | |
| 1); | |
| cpu->ring->first = (cpu->ring->first + 1) % KVM_COALESCED_MMIO_MAX; | |
| } | |
| } | |
| } | |
| static DEFINE_MUTEX(task_lock); | |
| static int task_eventfd; | |
| static void kvm_cpu__run_task(struct kvm_cpu *cpu) | |
| { | |
| u64 inc = 1; | |
| pr_debug("Running task %p on cpu %lu", cpu->task, cpu->cpu_id); | |
| /* Make sure we see the store to cpu->task */ | |
| rmb(); | |
| cpu->task->func(cpu, cpu->task->data); | |
| /* Clear task before we signal completion */ | |
| cpu->task = NULL; | |
| wmb(); | |
| if (write(task_eventfd, &inc, sizeof(inc)) < 0) | |
| die("Failed notifying of completed task."); | |
| } | |
| void kvm_cpu__run_on_all_cpus(struct kvm *kvm, struct kvm_cpu_task *task) | |
| { | |
| int i, done = 0; | |
| pr_debug("Running task %p on all cpus", task); | |
| mutex_lock(&task_lock); | |
| for (i = 0; i < kvm->nrcpus; i++) { | |
| if (kvm->cpus[i]->task) { | |
| /* Should never happen */ | |
| die("CPU %d already has a task pending!", i); | |
| } | |
| kvm->cpus[i]->task = task; | |
| wmb(); | |
| if (kvm->cpus[i] == current_kvm_cpu) | |
| kvm_cpu__run_task(current_kvm_cpu); | |
| else | |
| pthread_kill(kvm->cpus[i]->thread, SIGKVMTASK); | |
| } | |
| while (done < kvm->nrcpus) { | |
| u64 count; | |
| if (read(task_eventfd, &count, sizeof(count)) < 0) | |
| die("Failed reading task eventfd"); | |
| done += count; | |
| } | |
| mutex_unlock(&task_lock); | |
| } | |
| static void handle_wakeup(struct kvm_cpu *vcpu) | |
| { | |
| struct kvm_mp_state mp_state = { | |
| .mp_state = KVM_MP_STATE_RUNNABLE, | |
| }; | |
| if (ioctl(vcpu->vcpu_fd, KVM_SET_MP_STATE, &mp_state)) | |
| die_perror("KVM_SET_MP_STATE failed"); | |
| } | |
| int kvm_cpu__start(struct kvm_cpu *cpu) | |
| { | |
| sigset_t sigset; | |
| sigemptyset(&sigset); | |
| sigaddset(&sigset, SIGALRM); | |
| pthread_sigmask(SIG_BLOCK, &sigset, NULL); | |
| signal(SIGKVMEXIT, kvm_cpu_signal_handler); | |
| signal(SIGKVMPAUSE, kvm_cpu_signal_handler); | |
| signal(SIGKVMTASK, kvm_cpu_signal_handler); | |
| if (cpu->kvm->cfg.single_step) | |
| kvm_cpu__enable_singlestep(cpu); | |
| while (cpu->is_running) { | |
| if (cpu->needs_nmi) { | |
| kvm_cpu__arch_nmi(cpu); | |
| cpu->needs_nmi = 0; | |
| } | |
| if (cpu->task) | |
| kvm_cpu__run_task(cpu); | |
| kvm_cpu__run(cpu); | |
| switch (cpu->kvm_run->exit_reason) { | |
| case KVM_EXIT_UNKNOWN: | |
| break; | |
| case KVM_EXIT_DEBUG: | |
| kvm_cpu__show_registers(cpu); | |
| kvm_cpu__show_code(cpu); | |
| break; | |
| case KVM_EXIT_IO: { | |
| bool ret; | |
| ret = kvm_cpu__emulate_io(cpu, | |
| cpu->kvm_run->io.port, | |
| (u8 *)cpu->kvm_run + | |
| cpu->kvm_run->io.data_offset, | |
| cpu->kvm_run->io.direction, | |
| cpu->kvm_run->io.size, | |
| cpu->kvm_run->io.count); | |
| if (!ret) | |
| goto panic_kvm; | |
| break; | |
| } | |
| case KVM_EXIT_MMIO: { | |
| bool ret; | |
| /* | |
| * If we had MMIO exit, coalesced ring should be processed | |
| * *before* processing the exit itself | |
| */ | |
| kvm_cpu__handle_coalesced_mmio(cpu); | |
| ret = kvm_cpu__emulate_mmio(cpu, | |
| cpu->kvm_run->mmio.phys_addr, | |
| cpu->kvm_run->mmio.data, | |
| cpu->kvm_run->mmio.len, | |
| cpu->kvm_run->mmio.is_write); | |
| if (!ret) | |
| goto panic_kvm; | |
| break; | |
| } | |
| case KVM_EXIT_INTR: | |
| if (cpu->is_running) | |
| break; | |
| goto exit_kvm; | |
| case KVM_EXIT_SHUTDOWN: | |
| goto exit_kvm; | |
| case KVM_EXIT_SYSTEM_EVENT: | |
| /* | |
| * Print the type of system event and | |
| * treat all system events as shutdown request. | |
| */ | |
| switch (cpu->kvm_run->system_event.type) { | |
| default: | |
| pr_warning("unknown system event type %d", | |
| cpu->kvm_run->system_event.type); | |
| /* fall through for now */ | |
| case KVM_SYSTEM_EVENT_RESET: | |
| /* Fall through for now */ | |
| case KVM_SYSTEM_EVENT_SHUTDOWN: | |
| /* | |
| * Ensure that all VCPUs are torn down, | |
| * regardless of which CPU generated the event. | |
| */ | |
| kvm__reboot(cpu->kvm); | |
| goto exit_kvm; | |
| case KVM_SYSTEM_EVENT_WAKEUP: | |
| handle_wakeup(cpu); | |
| break; | |
| }; | |
| break; | |
| default: { | |
| bool ret; | |
| ret = kvm_cpu__handle_exit(cpu); | |
| if (!ret) | |
| goto panic_kvm; | |
| break; | |
| } | |
| } | |
| kvm_cpu__handle_coalesced_mmio(cpu); | |
| } | |
| exit_kvm: | |
| return 0; | |
| panic_kvm: | |
| return 1; | |
| } | |
| int kvm_cpu__init(struct kvm *kvm) | |
| { | |
| int max_cpus, recommended_cpus, i; | |
| max_cpus = kvm__max_cpus(kvm); | |
| recommended_cpus = kvm__recommended_cpus(kvm); | |
| if (kvm->cfg.nrcpus > max_cpus) { | |
| pr_warning("Limiting the number of CPUs to %d", max_cpus); | |
| kvm->cfg.nrcpus = max_cpus; | |
| } else if (kvm->cfg.nrcpus > recommended_cpus) { | |
| pr_warning("The maximum recommended amount of VCPUs is %d", | |
| recommended_cpus); | |
| } | |
| kvm->nrcpus = kvm->cfg.nrcpus; | |
| task_eventfd = eventfd(0, 0); | |
| if (task_eventfd < 0) { | |
| pr_err("Couldn't create task_eventfd"); | |
| return task_eventfd; | |
| } | |
| /* Alloc one pointer too many, so array ends up 0-terminated */ | |
| kvm->cpus = calloc(kvm->nrcpus + 1, sizeof(void *)); | |
| if (!kvm->cpus) { | |
| pr_err("Couldn't allocate array for %d CPUs", kvm->nrcpus); | |
| return -ENOMEM; | |
| } | |
| for (i = 0; i < kvm->nrcpus; i++) { | |
| kvm->cpus[i] = kvm_cpu__arch_init(kvm, i); | |
| if (!kvm->cpus[i]) { | |
| pr_err("unable to initialize KVM VCPU"); | |
| goto fail_alloc; | |
| } | |
| kvm_cpu__reset_vcpu(kvm->cpus[i]); | |
| } | |
| return 0; | |
| fail_alloc: | |
| for (i = 0; i < kvm->nrcpus; i++) | |
| free(kvm->cpus[i]); | |
| return -ENOMEM; | |
| } | |
| base_init(kvm_cpu__init); | |
| int kvm_cpu__exit(struct kvm *kvm, int vcpu0_ret) | |
| { | |
| int ret = vcpu0_ret; | |
| void *vcpu_ret; | |
| int i; | |
| kvm_cpu__delete(kvm->cpus[0]); | |
| kvm->cpus[0] = NULL; | |
| kvm__pause(kvm); | |
| for (i = 1; i < kvm->nrcpus; i++) { | |
| if (kvm->cpus[i]->is_running) { | |
| pthread_kill(kvm->cpus[i]->thread, SIGKVMEXIT); | |
| if (pthread_join(kvm->cpus[i]->thread, &vcpu_ret) != 0) | |
| die("pthread_join"); | |
| kvm_cpu__delete(kvm->cpus[i]); | |
| if (!ret && PTR_ERR(vcpu_ret)) | |
| ret = PTR_ERR(vcpu_ret); | |
| } | |
| } | |
| kvm__continue(kvm); | |
| free(kvm->cpus); | |
| kvm->nrcpus = 0; | |
| close(task_eventfd); | |
| return ret; | |
| } |