blob: 5f5eb577d583cec1aeb0a51711c963c9828e4877 [file] [log] [blame]
Thomas Gleixner20c8ccb2019-06-04 10:11:32 +02001// SPDX-License-Identifier: GPL-2.0-only
Carsten Otte043405e2007-10-10 17:16:19 +02002/*
3 * Kernel-based Virtual Machine driver for Linux
4 *
5 * derived from drivers/kvm/kvm_main.c
6 *
7 * Copyright (C) 2006 Qumranet, Inc.
Ben-Ami Yassour4d5c5d02008-07-28 19:26:26 +03008 * Copyright (C) 2008 Qumranet, Inc.
9 * Copyright IBM Corporation, 2008
Nicolas Kaiser9611c182010-10-06 14:23:22 +020010 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
Carsten Otte043405e2007-10-10 17:16:19 +020011 *
12 * Authors:
13 * Avi Kivity <avi@qumranet.com>
14 * Yaniv Kamay <yaniv@qumranet.com>
Ben-Ami Yassour4d5c5d02008-07-28 19:26:26 +030015 * Amit Shah <amit.shah@qumranet.com>
16 * Ben-Ami Yassour <benami@il.ibm.com>
Carsten Otte043405e2007-10-10 17:16:19 +020017 */
18
Avi Kivityedf88412007-12-16 11:02:48 +020019#include <linux/kvm_host.h>
Carsten Otte313a3dc2007-10-11 19:16:52 +020020#include "irq.h"
彭浩(Richard)88197e62020-05-21 05:57:49 +000021#include "ioapic.h"
Zhang Xiantao1d737c82007-12-14 09:35:10 +080022#include "mmu.h"
Sheng Yang78376992008-01-28 05:10:22 +080023#include "i8254.h"
Izik Eidus37817f22008-03-24 23:14:53 +020024#include "tss.h"
Marcelo Tosatti5fdbf972008-06-27 14:58:02 -030025#include "kvm_cache_regs.h"
Sean Christopherson2f728d62020-02-18 15:29:49 -080026#include "kvm_emulate.h"
Avi Kivity26eef702008-07-03 14:59:22 +030027#include "x86.h"
Avi Kivity00b27a32011-11-23 16:30:32 +020028#include "cpuid.h"
Wei Huang474a5bb2015-06-19 13:54:23 +020029#include "pmu.h"
Andrey Smetanine83d5882015-07-03 15:01:34 +030030#include "hyperv.h"
Suravee Suthikulpanit8df14af2019-11-14 14:15:06 -060031#include "lapic.h"
Joao Martins23200b72018-06-13 09:55:44 -040032#include "xen.h"
Carsten Otte313a3dc2007-10-11 19:16:52 +020033
Glauber de Oliveira Costa18068522008-02-15 17:52:47 -020034#include <linux/clocksource.h>
Ben-Ami Yassour4d5c5d02008-07-28 19:26:26 +030035#include <linux/interrupt.h>
Carsten Otte313a3dc2007-10-11 19:16:52 +020036#include <linux/kvm.h>
37#include <linux/fs.h>
38#include <linux/vmalloc.h>
Paul Gortmaker1767e932016-07-13 20:19:00 -040039#include <linux/export.h>
40#include <linux/moduleparam.h>
Zhang Xiantao0de10342007-11-20 16:25:04 +080041#include <linux/mman.h>
Marcelo Tosatti2bacc552007-12-12 10:46:12 -050042#include <linux/highmem.h>
Joerg Roedel19de40a2008-12-03 14:43:34 +010043#include <linux/iommu.h>
Gerd Hoffmannc8076602009-02-04 17:52:04 +010044#include <linux/cpufreq.h>
Avi Kivity18863bd2009-09-07 11:12:18 +030045#include <linux/user-return-notifier.h>
Marcelo Tosattia983fb22009-12-23 14:35:23 -020046#include <linux/srcu.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090047#include <linux/slab.h>
Zhang, Yanminff9d07a2010-04-19 13:32:45 +080048#include <linux/perf_event.h>
Lai Jiangshan7bee3422010-06-02 17:06:03 +080049#include <linux/uaccess.h>
Gleb Natapovaf585b92010-10-14 11:22:46 +020050#include <linux/hash.h>
Joerg Roedela1b60c12011-09-06 18:46:34 +020051#include <linux/pci.h>
Marcelo Tosatti16e8d742012-11-27 23:29:00 -020052#include <linux/timekeeper_internal.h>
53#include <linux/pvclock_gtod.h>
Feng Wu87276882015-09-18 22:29:40 +080054#include <linux/kvm_irqfd.h>
55#include <linux/irqbypass.h>
Ingo Molnar3905f9a2017-02-05 12:07:04 +010056#include <linux/sched/stat.h>
Wanpeng Li0c5f81d2019-07-06 09:26:51 +080057#include <linux/sched/isolation.h>
Tom Lendackyd0ec49d2017-07-17 16:10:27 -050058#include <linux/mem_encrypt.h>
Thomas Gleixner72c3c0f2020-07-23 00:00:09 +020059#include <linux/entry-kvm.h>
Sergey Senozhatsky7d628742021-06-06 11:10:45 +090060#include <linux/suspend.h>
Ingo Molnar3905f9a2017-02-05 12:07:04 +010061
Avi Kivityaec51dc2009-07-01 16:01:02 +030062#include <trace/events/kvm.h>
Xiao Guangrong2ed152a2010-03-10 19:00:43 +080063
Frederic Weisbecker24f1e32c2009-09-09 19:22:48 +020064#include <asm/debugreg.h>
Zhang Xiantaod825ed02007-11-14 20:08:51 +080065#include <asm/msr.h>
Avi Kivitya5f61302008-02-20 17:57:21 +020066#include <asm/desc.h>
Huang Ying890ca9a2009-05-11 16:48:15 +080067#include <asm/mce.h>
Dave Hansen784a46612021-06-23 14:02:05 +020068#include <asm/pkru.h>
Ingo Molnarf89e32e2015-04-22 10:58:10 +020069#include <linux/kernel_stat.h>
Thomas Gleixnera0ff0612021-10-15 03:16:12 +020070#include <asm/fpu/api.h>
71#include <asm/fpu/xcr.h>
72#include <asm/fpu/xstate.h>
Zachary Amsden1d5f0662010-08-19 22:07:30 -100073#include <asm/pvclock.h>
Avi Kivity217fc9c2010-08-26 13:38:03 +030074#include <asm/div64.h>
Feng Wuefc64402015-09-18 22:29:51 +080075#include <asm/irq_remapping.h>
Vitaly Kuznetsovb0c39dc2018-01-24 14:23:36 +010076#include <asm/mshyperv.h>
Vitaly Kuznetsov0092e432018-01-24 14:23:37 +010077#include <asm/hypervisor.h>
Babu Moger97150922020-09-11 14:29:12 -050078#include <asm/tlbflush.h>
Chao Pengbf8c55d2018-10-24 16:05:14 +080079#include <asm/intel_pt.h>
Masami Hiramatsub3dc0692019-09-06 22:13:59 +090080#include <asm/emulate_prefix.h>
Sean Christophersonfe7e9482021-04-12 16:21:43 +120081#include <asm/sgx.h>
Michael Kelleydd2cb3482019-07-01 04:26:06 +000082#include <clocksource/hyperv_timer.h>
Carsten Otte043405e2007-10-10 17:16:19 +020083
Dave Hansend1898b72016-06-01 10:42:20 -070084#define CREATE_TRACE_POINTS
85#include "trace.h"
86
Carsten Otte313a3dc2007-10-11 19:16:52 +020087#define MAX_IO_MSRS 256
Huang Ying890ca9a2009-05-11 16:48:15 +080088#define KVM_MAX_MCE_BANKS 32
Sean Christopherson938c8742022-05-24 21:56:23 +080089
90struct kvm_caps kvm_caps __read_mostly = {
91 .supported_mce_cap = MCG_CTL_P | MCG_SER_P,
92};
93EXPORT_SYMBOL_GPL(kvm_caps);
Huang Ying890ca9a2009-05-11 16:48:15 +080094
Sean Christopherson6e37ec882022-02-02 00:51:57 +000095#define ERR_PTR_USR(e) ((void __user *)ERR_PTR(e))
96
Avi Kivity0f65dd72011-04-20 13:37:53 +030097#define emul_to_vcpu(ctxt) \
Sean Christophersonc9b8b072020-02-18 15:29:48 -080098 ((struct kvm_vcpu *)(ctxt)->vcpu)
Avi Kivity0f65dd72011-04-20 13:37:53 +030099
Joerg Roedel50a37eb2008-01-31 14:57:38 +0100100/* EFER defaults:
101 * - enable syscall per default because its emulated by KVM
102 * - enable LME and LMA per default on 64 bit KVM
103 */
104#ifdef CONFIG_X86_64
Lai Jiangshan1260edbe2011-02-21 11:51:35 +0800105static
106u64 __read_mostly efer_reserved_bits = ~((u64)(EFER_SCE | EFER_LME | EFER_LMA));
Joerg Roedel50a37eb2008-01-31 14:57:38 +0100107#else
Lai Jiangshan1260edbe2011-02-21 11:51:35 +0800108static u64 __read_mostly efer_reserved_bits = ~((u64)EFER_SCE);
Joerg Roedel50a37eb2008-01-31 14:57:38 +0100109#endif
Carsten Otte313a3dc2007-10-11 19:16:52 +0200110
Sean Christophersonb11306b2019-12-10 14:44:13 -0800111static u64 __read_mostly cr4_reserved_bits = CR4_RESERVED_BITS;
112
Ashish Kalra0dbb1122021-06-08 18:05:43 +0000113#define KVM_EXIT_HYPERCALL_VALID_MASK (1 << KVM_HC_MAP_GPA_RANGE)
114
David Dunnba7bb6632022-02-23 22:57:41 +0000115#define KVM_CAP_PMU_VALID_MASK KVM_PMU_CAP_DISABLE
116
Radim Krčmářc5192652016-07-12 22:09:28 +0200117#define KVM_X2APIC_API_VALID_FLAGS (KVM_X2APIC_API_USE_32BIT_IDS | \
118 KVM_X2APIC_API_DISABLE_BROADCAST_QUIRK)
Radim Krčmář371313132016-07-12 22:09:27 +0200119
Gleb Natapovcb142eb2009-08-09 15:17:40 +0300120static void update_cr8_intercept(struct kvm_vcpu *vcpu);
Avi Kivity7460fb4a2011-09-20 13:43:14 +0300121static void process_nmi(struct kvm_vcpu *vcpu);
Jay Zhou1f7becf2021-01-18 16:47:20 +0800122static void process_smi(struct kvm_vcpu *vcpu);
Paolo Bonziniee2cd4b2016-06-01 22:26:01 +0200123static void enter_smm(struct kvm_vcpu *vcpu);
Paolo Bonzini6addfc42014-03-27 11:29:28 +0100124static void __kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags);
Ken Hofsass01643c52018-01-31 16:03:36 -0800125static void store_regs(struct kvm_vcpu *vcpu);
126static int sync_regs(struct kvm_vcpu *vcpu);
Hou Wenlongd2f7d492021-11-02 17:15:31 +0800127static int kvm_vcpu_do_singlestep(struct kvm_vcpu *vcpu);
Avi Kivity674eea0f2008-02-11 18:37:23 +0200128
Maxim Levitsky6dba9402021-06-07 12:02:02 +0300129static int __set_sregs2(struct kvm_vcpu *vcpu, struct kvm_sregs2 *sregs2);
130static void __get_sregs2(struct kvm_vcpu *vcpu, struct kvm_sregs2 *sregs2);
131
Sean Christophersonafaf0b22020-03-21 13:26:00 -0700132struct kvm_x86_ops kvm_x86_ops __read_mostly;
Zhang Xiantao97896d02007-11-14 20:09:30 +0800133
Jason Baron9af54712021-01-14 22:27:55 -0500134#define KVM_X86_OP(func) \
135 DEFINE_STATIC_CALL_NULL(kvm_x86_##func, \
136 *(((struct kvm_x86_ops *)0)->func));
Paolo Bonzinie4fc23b2021-12-09 08:12:28 -0500137#define KVM_X86_OP_OPTIONAL KVM_X86_OP
Paolo Bonzini5be22262022-02-15 13:07:10 -0500138#define KVM_X86_OP_OPTIONAL_RET0 KVM_X86_OP
Jason Baron9af54712021-01-14 22:27:55 -0500139#include <asm/kvm-x86-ops.h>
140EXPORT_STATIC_CALL_GPL(kvm_x86_get_cs_db_l_bits);
141EXPORT_STATIC_CALL_GPL(kvm_x86_cache_reg);
Jason Baron9af54712021-01-14 22:27:55 -0500142
Paolo Bonzini893590c2015-11-06 11:46:24 +0100143static bool __read_mostly ignore_msrs = 0;
Rusty Russell476bc002012-01-13 09:32:18 +1030144module_param(ignore_msrs, bool, S_IRUGO | S_IWUSR);
Andre Przywaraed85c062009-06-25 12:36:49 +0200145
Like Xud8550662021-01-08 09:36:55 +0800146bool __read_mostly report_ignored_msrs = true;
Eyal Moscovicifab0aa32017-11-08 14:32:08 +0200147module_param(report_ignored_msrs, bool, S_IRUGO | S_IWUSR);
Like Xud8550662021-01-08 09:36:55 +0800148EXPORT_SYMBOL_GPL(report_ignored_msrs);
Eyal Moscovicifab0aa32017-11-08 14:32:08 +0200149
Wanpeng Li4c276252018-05-05 04:02:32 -0700150unsigned int min_timer_period_us = 200;
Marcelo Tosatti9ed96e82014-01-06 12:00:02 -0200151module_param(min_timer_period_us, uint, S_IRUGO | S_IWUSR);
152
Marcelo Tosatti630994b2015-05-12 22:42:04 -0300153static bool __read_mostly kvmclock_periodic_sync = true;
154module_param(kvmclock_periodic_sync, bool, S_IRUGO);
155
Zachary Amsdencc578282012-02-03 15:43:50 -0200156/* tsc tolerance in parts per million - default to 1/2 of the NTP threshold */
Paolo Bonzini893590c2015-11-06 11:46:24 +0100157static u32 __read_mostly tsc_tolerance_ppm = 250;
Zachary Amsdencc578282012-02-03 15:43:50 -0200158module_param(tsc_tolerance_ppm, uint, S_IRUGO | S_IWUSR);
159
Sean Christophersonc3941d92019-04-17 10:15:33 -0700160/*
161 * lapic timer advance (tscdeadline mode only) in nanoseconds. '-1' enables
Ingo Molnard9f6e122021-03-18 15:28:01 +0100162 * adaptive tuning starting from default advancement of 1000ns. '0' disables
Sean Christophersonc3941d92019-04-17 10:15:33 -0700163 * advancement entirely. Any other value is used as-is and disables adaptive
Ingo Molnard9f6e122021-03-18 15:28:01 +0100164 * tuning, i.e. allows privileged userspace to set an exact advancement time.
Sean Christophersonc3941d92019-04-17 10:15:33 -0700165 */
166static int __read_mostly lapic_timer_advance_ns = -1;
Wanpeng Li0e6edce2019-05-20 16:18:06 +0800167module_param(lapic_timer_advance_ns, int, S_IRUGO | S_IWUSR);
Marcelo Tosattid0659d92014-12-16 09:08:15 -0500168
Feng Wu520040142016-01-25 16:53:33 +0800169static bool __read_mostly vector_hashing = true;
170module_param(vector_hashing, bool, S_IRUGO);
171
Liran Alonc4ae60e2018-03-12 13:12:47 +0200172bool __read_mostly enable_vmware_backdoor = false;
173module_param(enable_vmware_backdoor, bool, S_IRUGO);
174EXPORT_SYMBOL_GPL(enable_vmware_backdoor);
175
Sean Christophersond500e1e2022-08-30 23:15:51 +0000176/*
177 * Flags to manipulate forced emulation behavior (any non-zero value will
178 * enable forced emulation).
179 */
180#define KVM_FEP_CLEAR_RFLAGS_RF BIT(1)
181static int __read_mostly force_emulation_prefix;
Sean Christopherson40aaa5b2022-08-30 23:16:14 +0000182module_param(force_emulation_prefix, int, 0644);
Wanpeng Li6c86eed2018-04-03 16:28:49 -0700183
Wanpeng Li0c5f81d2019-07-06 09:26:51 +0800184int __read_mostly pi_inject_timer = -1;
185module_param(pi_inject_timer, bint, S_IRUGO | S_IWUSR);
186
Like Xu4732f242022-01-11 15:38:23 +0800187/* Enable/disable PMU virtualization */
188bool __read_mostly enable_pmu = true;
189EXPORT_SYMBOL_GPL(enable_pmu);
190module_param(enable_pmu, bool, 0444);
191
David Matlackcb00a702022-01-19 23:07:37 +0000192bool __read_mostly eager_page_split = true;
David Matlacka3fe5db2022-01-19 23:07:36 +0000193module_param(eager_page_split, bool, 0644);
194
Sean Christopherson7e34fbd2020-09-23 11:03:55 -0700195/*
196 * Restoring the host value for MSRs that are only consumed when running in
197 * usermode, e.g. SYSCALL MSRs and TSC_AUX, can be deferred until the CPU
198 * returns to userspace, i.e. the kernel can run with the guest's value.
199 */
200#define KVM_MAX_NR_USER_RETURN_MSRS 16
Avi Kivity18863bd2009-09-07 11:12:18 +0300201
Sean Christopherson7e34fbd2020-09-23 11:03:55 -0700202struct kvm_user_return_msrs {
Avi Kivity18863bd2009-09-07 11:12:18 +0300203 struct user_return_notifier urn;
204 bool registered;
Sean Christopherson7e34fbd2020-09-23 11:03:55 -0700205 struct kvm_user_return_msr_values {
Sheng Yang2bf78fa2009-12-18 16:48:44 +0800206 u64 host;
207 u64 curr;
Sean Christopherson7e34fbd2020-09-23 11:03:55 -0700208 } values[KVM_MAX_NR_USER_RETURN_MSRS];
Avi Kivity18863bd2009-09-07 11:12:18 +0300209};
210
Sean Christopherson9cc39a52021-05-04 10:17:31 -0700211u32 __read_mostly kvm_nr_uret_msrs;
212EXPORT_SYMBOL_GPL(kvm_nr_uret_msrs);
213static u32 __read_mostly kvm_uret_msrs_list[KVM_MAX_NR_USER_RETURN_MSRS];
Sean Christopherson7e34fbd2020-09-23 11:03:55 -0700214static struct kvm_user_return_msrs __percpu *user_return_msrs;
Avi Kivity18863bd2009-09-07 11:12:18 +0300215
Sean Christophersoncfc48182020-03-02 15:56:23 -0800216#define KVM_SUPPORTED_XCR0 (XFEATURE_MASK_FP | XFEATURE_MASK_SSE \
217 | XFEATURE_MASK_YMM | XFEATURE_MASK_BNDREGS \
218 | XFEATURE_MASK_BNDCSR | XFEATURE_MASK_AVX512 \
Jing Liu86aff7a42022-01-05 04:35:26 -0800219 | XFEATURE_MASK_PKRU | XFEATURE_MASK_XTILE)
Sean Christophersoncfc48182020-03-02 15:56:23 -0800220
Sean Christopherson91661982020-03-02 15:57:06 -0800221u64 __read_mostly host_efer;
222EXPORT_SYMBOL_GPL(host_efer);
223
Mohammed Gamalb96e6502020-09-03 16:11:22 +0200224bool __read_mostly allow_smaller_maxphyaddr = 0;
Mohammed Gamal3edd6832020-07-10 17:48:11 +0200225EXPORT_SYMBOL_GPL(allow_smaller_maxphyaddr);
226
Vitaly Kuznetsovfdf513e2021-06-09 17:09:08 +0200227bool __read_mostly enable_apicv = true;
228EXPORT_SYMBOL_GPL(enable_apicv);
229
Tom Lendacky86137772020-12-10 11:10:07 -0600230u64 __read_mostly host_xss;
231EXPORT_SYMBOL_GPL(host_xss);
Aaron Lewis139a12c2019-10-21 16:30:25 -0700232
Jing Zhangfcfe1ba2021-06-18 22:27:05 +0000233const struct _kvm_stats_desc kvm_vm_stats_desc[] = {
234 KVM_GENERIC_VM_STATS(),
235 STATS_DESC_COUNTER(VM, mmu_shadow_zapped),
236 STATS_DESC_COUNTER(VM, mmu_pte_write),
237 STATS_DESC_COUNTER(VM, mmu_pde_zapped),
238 STATS_DESC_COUNTER(VM, mmu_flooded),
239 STATS_DESC_COUNTER(VM, mmu_recycled),
240 STATS_DESC_COUNTER(VM, mmu_cache_miss),
241 STATS_DESC_ICOUNTER(VM, mmu_unsync),
Mingwei Zhang71f51d22021-08-02 21:46:07 -0700242 STATS_DESC_ICOUNTER(VM, pages_4k),
243 STATS_DESC_ICOUNTER(VM, pages_2m),
244 STATS_DESC_ICOUNTER(VM, pages_1g),
Jing Zhangfcfe1ba2021-06-18 22:27:05 +0000245 STATS_DESC_ICOUNTER(VM, nx_lpage_splits),
Peter Xuec1cf692021-06-25 11:32:06 -0400246 STATS_DESC_PCOUNTER(VM, max_mmu_rmap_size),
Jing Zhangbc9e9e62021-06-23 17:28:46 -0400247 STATS_DESC_PCOUNTER(VM, max_mmu_page_hash_collisions)
Jing Zhangfcfe1ba2021-06-18 22:27:05 +0000248};
Jing Zhangfcfe1ba2021-06-18 22:27:05 +0000249
250const struct kvm_stats_header kvm_vm_stats_header = {
251 .name_size = KVM_STATS_NAME_SIZE,
252 .num_desc = ARRAY_SIZE(kvm_vm_stats_desc),
253 .id_offset = sizeof(struct kvm_stats_header),
254 .desc_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE,
255 .data_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE +
256 sizeof(kvm_vm_stats_desc),
257};
258
Jing Zhangce55c042021-06-18 22:27:06 +0000259const struct _kvm_stats_desc kvm_vcpu_stats_desc[] = {
260 KVM_GENERIC_VCPU_STATS(),
Sean Christopherson1075d412022-04-23 03:47:49 +0000261 STATS_DESC_COUNTER(VCPU, pf_taken),
Jing Zhangce55c042021-06-18 22:27:06 +0000262 STATS_DESC_COUNTER(VCPU, pf_fixed),
Sean Christopherson1075d412022-04-23 03:47:49 +0000263 STATS_DESC_COUNTER(VCPU, pf_emulate),
264 STATS_DESC_COUNTER(VCPU, pf_spurious),
265 STATS_DESC_COUNTER(VCPU, pf_fast),
266 STATS_DESC_COUNTER(VCPU, pf_mmio_spte_created),
Jing Zhangce55c042021-06-18 22:27:06 +0000267 STATS_DESC_COUNTER(VCPU, pf_guest),
268 STATS_DESC_COUNTER(VCPU, tlb_flush),
269 STATS_DESC_COUNTER(VCPU, invlpg),
270 STATS_DESC_COUNTER(VCPU, exits),
271 STATS_DESC_COUNTER(VCPU, io_exits),
272 STATS_DESC_COUNTER(VCPU, mmio_exits),
273 STATS_DESC_COUNTER(VCPU, signal_exits),
274 STATS_DESC_COUNTER(VCPU, irq_window_exits),
275 STATS_DESC_COUNTER(VCPU, nmi_window_exits),
276 STATS_DESC_COUNTER(VCPU, l1d_flush),
277 STATS_DESC_COUNTER(VCPU, halt_exits),
278 STATS_DESC_COUNTER(VCPU, request_irq_exits),
279 STATS_DESC_COUNTER(VCPU, irq_exits),
280 STATS_DESC_COUNTER(VCPU, host_state_reload),
281 STATS_DESC_COUNTER(VCPU, fpu_reload),
282 STATS_DESC_COUNTER(VCPU, insn_emulation),
283 STATS_DESC_COUNTER(VCPU, insn_emulation_fail),
284 STATS_DESC_COUNTER(VCPU, hypercalls),
285 STATS_DESC_COUNTER(VCPU, irq_injections),
286 STATS_DESC_COUNTER(VCPU, nmi_injections),
287 STATS_DESC_COUNTER(VCPU, req_event),
288 STATS_DESC_COUNTER(VCPU, nested_run),
289 STATS_DESC_COUNTER(VCPU, directed_yield_attempted),
290 STATS_DESC_COUNTER(VCPU, directed_yield_successful),
Paolo Bonzini6cd88242022-06-07 10:09:03 -0400291 STATS_DESC_COUNTER(VCPU, preemption_reported),
292 STATS_DESC_COUNTER(VCPU, preemption_other),
Paolo Bonzini63f4b212022-07-29 09:46:01 -0400293 STATS_DESC_IBOOLEAN(VCPU, guest_mode),
Tao Xu2f4073e2022-05-24 21:56:24 +0800294 STATS_DESC_COUNTER(VCPU, notify_window_exits),
Jing Zhangce55c042021-06-18 22:27:06 +0000295};
Jing Zhangce55c042021-06-18 22:27:06 +0000296
297const struct kvm_stats_header kvm_vcpu_stats_header = {
298 .name_size = KVM_STATS_NAME_SIZE,
299 .num_desc = ARRAY_SIZE(kvm_vcpu_stats_desc),
300 .id_offset = sizeof(struct kvm_stats_header),
301 .desc_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE,
302 .data_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE +
303 sizeof(kvm_vcpu_stats_desc),
304};
305
Dexuan Cui2acf9232010-06-10 11:27:12 +0800306u64 __read_mostly host_xcr0;
307
Sean Christophersonc9b8b072020-02-18 15:29:48 -0800308static struct kmem_cache *x86_emulator_cache;
309
Peter Xu6abe9c12020-06-22 18:04:41 -0400310/*
311 * When called, it means the previous get/set msr reached an invalid msr.
Maxim Levitskycc4cb012020-11-01 13:55:23 +0200312 * Return true if we want to ignore/silent this failed msr access.
Peter Xu6abe9c12020-06-22 18:04:41 -0400313 */
Haiwei Lid6328262021-03-13 13:10:32 +0800314static bool kvm_msr_ignored_check(u32 msr, u64 data, bool write)
Peter Xu6abe9c12020-06-22 18:04:41 -0400315{
316 const char *op = write ? "wrmsr" : "rdmsr";
317
318 if (ignore_msrs) {
319 if (report_ignored_msrs)
Takashi Iwaid383b312020-10-30 16:14:14 +0100320 kvm_pr_unimpl("ignored %s: 0x%x data 0x%llx\n",
321 op, msr, data);
Peter Xu6abe9c12020-06-22 18:04:41 -0400322 /* Mask the error */
Maxim Levitskycc4cb012020-11-01 13:55:23 +0200323 return true;
Peter Xu6abe9c12020-06-22 18:04:41 -0400324 } else {
Takashi Iwaid383b312020-10-30 16:14:14 +0100325 kvm_debug_ratelimited("unhandled %s: 0x%x data 0x%llx\n",
326 op, msr, data);
Maxim Levitskycc4cb012020-11-01 13:55:23 +0200327 return false;
Peter Xu6abe9c12020-06-22 18:04:41 -0400328 }
329}
330
Sean Christophersonc9b8b072020-02-18 15:29:48 -0800331static struct kmem_cache *kvm_alloc_emulator_cache(void)
332{
Sean Christopherson06add252020-02-18 15:29:50 -0800333 unsigned int useroffset = offsetof(struct x86_emulate_ctxt, src);
334 unsigned int size = sizeof(struct x86_emulate_ctxt);
335
336 return kmem_cache_create_usercopy("x86_emulator", size,
Sean Christophersonc9b8b072020-02-18 15:29:48 -0800337 __alignof__(struct x86_emulate_ctxt),
Sean Christopherson06add252020-02-18 15:29:50 -0800338 SLAB_ACCOUNT, useroffset,
339 size - useroffset, NULL);
Sean Christophersonc9b8b072020-02-18 15:29:48 -0800340}
341
Jan Kiszkab6785de2012-09-20 07:43:17 +0200342static int emulator_fix_hypercall(struct x86_emulate_ctxt *ctxt);
Avi Kivityd6aa1002011-04-20 15:47:13 +0300343
Gleb Natapovaf585b92010-10-14 11:22:46 +0200344static inline void kvm_async_pf_hash_reset(struct kvm_vcpu *vcpu)
345{
346 int i;
Peter Xudd03bca2020-04-16 11:58:59 -0400347 for (i = 0; i < ASYNC_PF_PER_VCPU; i++)
Gleb Natapovaf585b92010-10-14 11:22:46 +0200348 vcpu->arch.apf.gfns[i] = ~0;
349}
350
Avi Kivity18863bd2009-09-07 11:12:18 +0300351static void kvm_on_user_return(struct user_return_notifier *urn)
352{
353 unsigned slot;
Sean Christopherson7e34fbd2020-09-23 11:03:55 -0700354 struct kvm_user_return_msrs *msrs
355 = container_of(urn, struct kvm_user_return_msrs, urn);
356 struct kvm_user_return_msr_values *values;
Ignacio Alvarado1650b4e2016-11-04 12:15:55 -0700357 unsigned long flags;
Avi Kivity18863bd2009-09-07 11:12:18 +0300358
Ignacio Alvarado1650b4e2016-11-04 12:15:55 -0700359 /*
360 * Disabling irqs at this point since the following code could be
361 * interrupted and executed through kvm_arch_hardware_disable()
362 */
363 local_irq_save(flags);
Sean Christopherson7e34fbd2020-09-23 11:03:55 -0700364 if (msrs->registered) {
365 msrs->registered = false;
Ignacio Alvarado1650b4e2016-11-04 12:15:55 -0700366 user_return_notifier_unregister(urn);
367 }
368 local_irq_restore(flags);
Sean Christopherson9cc39a52021-05-04 10:17:31 -0700369 for (slot = 0; slot < kvm_nr_uret_msrs; ++slot) {
Sean Christopherson7e34fbd2020-09-23 11:03:55 -0700370 values = &msrs->values[slot];
Sheng Yang2bf78fa2009-12-18 16:48:44 +0800371 if (values->host != values->curr) {
Sean Christopherson9cc39a52021-05-04 10:17:31 -0700372 wrmsrl(kvm_uret_msrs_list[slot], values->host);
Sheng Yang2bf78fa2009-12-18 16:48:44 +0800373 values->curr = values->host;
Avi Kivity18863bd2009-09-07 11:12:18 +0300374 }
375 }
Avi Kivity18863bd2009-09-07 11:12:18 +0300376}
377
Sean Christophersone5fda4b2021-05-04 10:17:32 -0700378static int kvm_probe_user_return_msr(u32 msr)
Sean Christopherson5104d7f2021-05-04 10:17:24 -0700379{
380 u64 val;
381 int ret;
382
383 preempt_disable();
384 ret = rdmsrl_safe(msr, &val);
385 if (ret)
386 goto out;
387 ret = wrmsrl_safe(msr, val);
388out:
389 preempt_enable();
390 return ret;
391}
Sean Christopherson5104d7f2021-05-04 10:17:24 -0700392
Sean Christophersone5fda4b2021-05-04 10:17:32 -0700393int kvm_add_user_return_msr(u32 msr)
Sheng Yang2bf78fa2009-12-18 16:48:44 +0800394{
Sean Christophersone5fda4b2021-05-04 10:17:32 -0700395 BUG_ON(kvm_nr_uret_msrs >= KVM_MAX_NR_USER_RETURN_MSRS);
396
397 if (kvm_probe_user_return_msr(msr))
398 return -1;
399
400 kvm_uret_msrs_list[kvm_nr_uret_msrs] = msr;
401 return kvm_nr_uret_msrs++;
Avi Kivity18863bd2009-09-07 11:12:18 +0300402}
Sean Christophersone5fda4b2021-05-04 10:17:32 -0700403EXPORT_SYMBOL_GPL(kvm_add_user_return_msr);
Avi Kivity18863bd2009-09-07 11:12:18 +0300404
Sean Christopherson8ea8b8d2021-05-04 10:17:29 -0700405int kvm_find_user_return_msr(u32 msr)
406{
407 int i;
408
Sean Christopherson9cc39a52021-05-04 10:17:31 -0700409 for (i = 0; i < kvm_nr_uret_msrs; ++i) {
410 if (kvm_uret_msrs_list[i] == msr)
Sean Christopherson8ea8b8d2021-05-04 10:17:29 -0700411 return i;
412 }
413 return -1;
414}
415EXPORT_SYMBOL_GPL(kvm_find_user_return_msr);
416
Sean Christopherson7e34fbd2020-09-23 11:03:55 -0700417static void kvm_user_return_msr_cpu_online(void)
Avi Kivity18863bd2009-09-07 11:12:18 +0300418{
Sean Christopherson05c19c22019-11-22 12:04:50 -0800419 unsigned int cpu = smp_processor_id();
Sean Christopherson7e34fbd2020-09-23 11:03:55 -0700420 struct kvm_user_return_msrs *msrs = per_cpu_ptr(user_return_msrs, cpu);
Sean Christopherson05c19c22019-11-22 12:04:50 -0800421 u64 value;
422 int i;
Avi Kivity18863bd2009-09-07 11:12:18 +0300423
Sean Christopherson9cc39a52021-05-04 10:17:31 -0700424 for (i = 0; i < kvm_nr_uret_msrs; ++i) {
425 rdmsrl_safe(kvm_uret_msrs_list[i], &value);
Sean Christopherson7e34fbd2020-09-23 11:03:55 -0700426 msrs->values[i].host = value;
427 msrs->values[i].curr = value;
Sean Christopherson05c19c22019-11-22 12:04:50 -0800428 }
Avi Kivity18863bd2009-09-07 11:12:18 +0300429}
430
Sean Christopherson7e34fbd2020-09-23 11:03:55 -0700431int kvm_set_user_return_msr(unsigned slot, u64 value, u64 mask)
Avi Kivity18863bd2009-09-07 11:12:18 +0300432{
Marcelo Tosatti013f6a52013-01-03 11:41:39 -0200433 unsigned int cpu = smp_processor_id();
Sean Christopherson7e34fbd2020-09-23 11:03:55 -0700434 struct kvm_user_return_msrs *msrs = per_cpu_ptr(user_return_msrs, cpu);
Andy Honig8b3c3102014-08-27 11:16:44 -0700435 int err;
Avi Kivity18863bd2009-09-07 11:12:18 +0300436
Sean Christopherson7e34fbd2020-09-23 11:03:55 -0700437 value = (value & mask) | (msrs->values[slot].host & ~mask);
438 if (value == msrs->values[slot].curr)
Andy Honig8b3c3102014-08-27 11:16:44 -0700439 return 0;
Sean Christopherson9cc39a52021-05-04 10:17:31 -0700440 err = wrmsrl_safe(kvm_uret_msrs_list[slot], value);
Andy Honig8b3c3102014-08-27 11:16:44 -0700441 if (err)
442 return 1;
443
Sean Christopherson7e34fbd2020-09-23 11:03:55 -0700444 msrs->values[slot].curr = value;
445 if (!msrs->registered) {
446 msrs->urn.on_user_return = kvm_on_user_return;
447 user_return_notifier_register(&msrs->urn);
448 msrs->registered = true;
Avi Kivity18863bd2009-09-07 11:12:18 +0300449 }
Andy Honig8b3c3102014-08-27 11:16:44 -0700450 return 0;
Avi Kivity18863bd2009-09-07 11:12:18 +0300451}
Sean Christopherson7e34fbd2020-09-23 11:03:55 -0700452EXPORT_SYMBOL_GPL(kvm_set_user_return_msr);
Avi Kivity18863bd2009-09-07 11:12:18 +0300453
Radim Krčmář13a34e02014-08-28 15:13:03 +0200454static void drop_user_return_notifiers(void)
Avi Kivity3548bab2009-11-28 14:18:47 +0200455{
Marcelo Tosatti013f6a52013-01-03 11:41:39 -0200456 unsigned int cpu = smp_processor_id();
Sean Christopherson7e34fbd2020-09-23 11:03:55 -0700457 struct kvm_user_return_msrs *msrs = per_cpu_ptr(user_return_msrs, cpu);
Avi Kivity3548bab2009-11-28 14:18:47 +0200458
Sean Christopherson7e34fbd2020-09-23 11:03:55 -0700459 if (msrs->registered)
460 kvm_on_user_return(&msrs->urn);
Avi Kivity3548bab2009-11-28 14:18:47 +0200461}
462
Carsten Otte6866b832007-10-29 16:09:10 +0100463u64 kvm_get_apic_base(struct kvm_vcpu *vcpu)
464{
Gleb Natapov8a5a87d2012-08-05 15:58:26 +0300465 return vcpu->arch.apic_base;
Carsten Otte6866b832007-10-29 16:09:10 +0100466}
467EXPORT_SYMBOL_GPL(kvm_get_apic_base);
468
Jim Mattson58871642018-05-09 16:56:04 -0400469enum lapic_mode kvm_get_apic_mode(struct kvm_vcpu *vcpu)
470{
471 return kvm_apic_mode(kvm_get_apic_base(vcpu));
472}
473EXPORT_SYMBOL_GPL(kvm_get_apic_mode);
474
Jan Kiszka58cb6282014-01-24 16:48:44 +0100475int kvm_set_apic_base(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
Carsten Otte6866b832007-10-29 16:09:10 +0100476{
Jim Mattson58871642018-05-09 16:56:04 -0400477 enum lapic_mode old_mode = kvm_get_apic_mode(vcpu);
478 enum lapic_mode new_mode = kvm_apic_mode(msr_info->data);
Sean Christophersona8ac8642021-02-03 16:01:15 -0800479 u64 reserved_bits = kvm_vcpu_reserved_gpa_bits_raw(vcpu) | 0x2ff |
Radim Krčmářd6321d42017-08-05 00:12:49 +0200480 (guest_cpuid_has(vcpu, X86_FEATURE_X2APIC) ? 0 : X2APIC_ENABLE);
Jan Kiszka58cb6282014-01-24 16:48:44 +0100481
Jim Mattson58871642018-05-09 16:56:04 -0400482 if ((msr_info->data & reserved_bits) != 0 || new_mode == LAPIC_MODE_INVALID)
Jim Mattsond3802282017-08-10 10:14:13 -0700483 return 1;
Jim Mattson58871642018-05-09 16:56:04 -0400484 if (!msr_info->host_initiated) {
485 if (old_mode == LAPIC_MODE_X2APIC && new_mode == LAPIC_MODE_XAPIC)
486 return 1;
487 if (old_mode == LAPIC_MODE_DISABLED && new_mode == LAPIC_MODE_X2APIC)
488 return 1;
489 }
Jan Kiszka58cb6282014-01-24 16:48:44 +0100490
491 kvm_lapic_set_base(vcpu, msr_info->data);
Wanpeng Li4abaffc2020-02-26 10:41:02 +0800492 kvm_recalculate_apic_map(vcpu->kvm);
Jan Kiszka58cb6282014-01-24 16:48:44 +0100493 return 0;
Carsten Otte6866b832007-10-29 16:09:10 +0100494}
495EXPORT_SYMBOL_GPL(kvm_set_apic_base);
496
Sean Christophersonad0577c2021-08-09 10:39:54 -0700497/*
498 * Handle a fault on a hardware virtualization (VMX or SVM) instruction.
499 *
500 * Hardware virtualization extension instructions may fault if a reboot turns
501 * off virtualization while processes are running. Usually after catching the
502 * fault we just panic; during reboot instead the instruction is ignored.
503 */
504noinstr void kvm_spurious_fault(void)
Geoff Levande3ba45b2013-04-05 19:20:30 +0000505{
506 /* Fault while not rebooting. We want the trace. */
kbuild test robotb4fdcf62019-09-29 18:43:28 +0200507 BUG_ON(!kvm_rebooting);
Geoff Levande3ba45b2013-04-05 19:20:30 +0000508}
509EXPORT_SYMBOL_GPL(kvm_spurious_fault);
510
Eddie Dong3fd28fc2009-11-19 17:54:07 +0200511#define EXCPT_BENIGN 0
512#define EXCPT_CONTRIBUTORY 1
513#define EXCPT_PF 2
514
515static int exception_class(int vector)
516{
517 switch (vector) {
518 case PF_VECTOR:
519 return EXCPT_PF;
520 case DE_VECTOR:
521 case TS_VECTOR:
522 case NP_VECTOR:
523 case SS_VECTOR:
524 case GP_VECTOR:
525 return EXCPT_CONTRIBUTORY;
526 default:
527 break;
528 }
529 return EXCPT_BENIGN;
530}
531
Nadav Amitd6e8c852014-07-24 14:51:24 +0300532#define EXCPT_FAULT 0
533#define EXCPT_TRAP 1
534#define EXCPT_ABORT 2
535#define EXCPT_INTERRUPT 3
Sean Christopherson5623f752022-08-30 23:15:55 +0000536#define EXCPT_DB 4
Nadav Amitd6e8c852014-07-24 14:51:24 +0300537
538static int exception_type(int vector)
539{
540 unsigned int mask;
541
542 if (WARN_ON(vector > 31 || vector == NMI_VECTOR))
543 return EXCPT_INTERRUPT;
544
545 mask = 1 << vector;
546
Sean Christopherson5623f752022-08-30 23:15:55 +0000547 /*
548 * #DBs can be trap-like or fault-like, the caller must check other CPU
549 * state, e.g. DR6, to determine whether a #DB is a trap or fault.
550 */
551 if (mask & (1 << DB_VECTOR))
552 return EXCPT_DB;
553
554 if (mask & ((1 << BP_VECTOR) | (1 << OF_VECTOR)))
Nadav Amitd6e8c852014-07-24 14:51:24 +0300555 return EXCPT_TRAP;
556
557 if (mask & ((1 << DF_VECTOR) | (1 << MC_VECTOR)))
558 return EXCPT_ABORT;
559
560 /* Reserved exceptions will result in fault */
561 return EXCPT_FAULT;
562}
563
Sean Christophersond4963e32022-08-30 23:16:01 +0000564void kvm_deliver_exception_payload(struct kvm_vcpu *vcpu,
565 struct kvm_queued_exception *ex)
Jim Mattsonda998b42018-10-16 14:29:22 -0700566{
Sean Christophersond4963e32022-08-30 23:16:01 +0000567 if (!ex->has_payload)
Jim Mattsonda998b42018-10-16 14:29:22 -0700568 return;
569
Sean Christophersond4963e32022-08-30 23:16:01 +0000570 switch (ex->vector) {
Jim Mattsonf10c7292018-10-16 14:29:23 -0700571 case DB_VECTOR:
572 /*
573 * "Certain debug exceptions may clear bit 0-3. The
574 * remaining contents of the DR6 register are never
575 * cleared by the processor".
576 */
577 vcpu->arch.dr6 &= ~DR_TRAP_BITS;
578 /*
Chenyi Qiang9a3ecd52021-02-02 17:04:31 +0800579 * In order to reflect the #DB exception payload in guest
580 * dr6, three components need to be considered: active low
581 * bit, FIXED_1 bits and active high bits (e.g. DR6_BD,
582 * DR6_BS and DR6_BT)
583 * DR6_ACTIVE_LOW contains the FIXED_1 and active low bits.
584 * In the target guest dr6:
585 * FIXED_1 bits should always be set.
586 * Active low bits should be cleared if 1-setting in payload.
587 * Active high bits should be set if 1-setting in payload.
588 *
589 * Note, the payload is compatible with the pending debug
590 * exceptions/exit qualification under VMX, that active_low bits
591 * are active high in payload.
592 * So they need to be flipped for DR6.
Jim Mattsonf10c7292018-10-16 14:29:23 -0700593 */
Chenyi Qiang9a3ecd52021-02-02 17:04:31 +0800594 vcpu->arch.dr6 |= DR6_ACTIVE_LOW;
Sean Christophersond4963e32022-08-30 23:16:01 +0000595 vcpu->arch.dr6 |= ex->payload;
596 vcpu->arch.dr6 ^= ex->payload & DR6_ACTIVE_LOW;
Oliver Upton307f1cf2020-02-07 02:36:04 -0800597
598 /*
599 * The #DB payload is defined as compatible with the 'pending
600 * debug exceptions' field under VMX, not DR6. While bit 12 is
601 * defined in the 'pending debug exceptions' field (enabled
602 * breakpoint), it is reserved and must be zero in DR6.
603 */
604 vcpu->arch.dr6 &= ~BIT(12);
Jim Mattsonf10c7292018-10-16 14:29:23 -0700605 break;
Jim Mattsonda998b42018-10-16 14:29:22 -0700606 case PF_VECTOR:
Sean Christophersond4963e32022-08-30 23:16:01 +0000607 vcpu->arch.cr2 = ex->payload;
Jim Mattsonda998b42018-10-16 14:29:22 -0700608 break;
609 }
610
Sean Christophersond4963e32022-08-30 23:16:01 +0000611 ex->has_payload = false;
612 ex->payload = 0;
Jim Mattsonda998b42018-10-16 14:29:22 -0700613}
614EXPORT_SYMBOL_GPL(kvm_deliver_exception_payload);
615
Sean Christopherson7709aba2022-08-30 23:16:08 +0000616static void kvm_queue_exception_vmexit(struct kvm_vcpu *vcpu, unsigned int vector,
617 bool has_error_code, u32 error_code,
618 bool has_payload, unsigned long payload)
619{
620 struct kvm_queued_exception *ex = &vcpu->arch.exception_vmexit;
621
622 ex->vector = vector;
623 ex->injected = false;
624 ex->pending = true;
625 ex->has_error_code = has_error_code;
626 ex->error_code = error_code;
627 ex->has_payload = has_payload;
628 ex->payload = payload;
629}
630
Eddie Dong3fd28fc2009-11-19 17:54:07 +0200631static void kvm_multiple_exception(struct kvm_vcpu *vcpu,
Joerg Roedelce7ddec2010-04-22 12:33:13 +0200632 unsigned nr, bool has_error, u32 error_code,
Jim Mattson91e86d22018-10-16 14:29:21 -0700633 bool has_payload, unsigned long payload, bool reinject)
Eddie Dong3fd28fc2009-11-19 17:54:07 +0200634{
635 u32 prev_nr;
636 int class1, class2;
637
Avi Kivity3842d132010-07-27 12:30:24 +0300638 kvm_make_request(KVM_REQ_EVENT, vcpu);
639
Sean Christopherson7709aba2022-08-30 23:16:08 +0000640 /*
641 * If the exception is destined for L2 and isn't being reinjected,
642 * morph it to a VM-Exit if L1 wants to intercept the exception. A
643 * previously injected exception is not checked because it was checked
644 * when it was original queued, and re-checking is incorrect if _L1_
645 * injected the exception, in which case it's exempt from interception.
646 */
647 if (!reinject && is_guest_mode(vcpu) &&
648 kvm_x86_ops.nested_ops->is_exception_vmexit(vcpu, nr, error_code)) {
649 kvm_queue_exception_vmexit(vcpu, nr, has_error, error_code,
650 has_payload, payload);
651 return;
652 }
653
Wanpeng Li664f8e22017-08-24 03:35:09 -0700654 if (!vcpu->arch.exception.pending && !vcpu->arch.exception.injected) {
Eddie Dong3fd28fc2009-11-19 17:54:07 +0200655 queue:
Wanpeng Li664f8e22017-08-24 03:35:09 -0700656 if (reinject) {
657 /*
Sean Christopherson7709aba2022-08-30 23:16:08 +0000658 * On VM-Entry, an exception can be pending if and only
659 * if event injection was blocked by nested_run_pending.
660 * In that case, however, vcpu_enter_guest() requests an
661 * immediate exit, and the guest shouldn't proceed far
662 * enough to need reinjection.
Wanpeng Li664f8e22017-08-24 03:35:09 -0700663 */
Sean Christopherson7709aba2022-08-30 23:16:08 +0000664 WARN_ON_ONCE(kvm_is_exception_pending(vcpu));
Wanpeng Li664f8e22017-08-24 03:35:09 -0700665 vcpu->arch.exception.injected = true;
Jim Mattson91e86d22018-10-16 14:29:21 -0700666 if (WARN_ON_ONCE(has_payload)) {
667 /*
668 * A reinjected event has already
669 * delivered its payload.
670 */
671 has_payload = false;
672 payload = 0;
673 }
Wanpeng Li664f8e22017-08-24 03:35:09 -0700674 } else {
675 vcpu->arch.exception.pending = true;
676 vcpu->arch.exception.injected = false;
677 }
Eddie Dong3fd28fc2009-11-19 17:54:07 +0200678 vcpu->arch.exception.has_error_code = has_error;
Sean Christophersond4963e32022-08-30 23:16:01 +0000679 vcpu->arch.exception.vector = nr;
Eddie Dong3fd28fc2009-11-19 17:54:07 +0200680 vcpu->arch.exception.error_code = error_code;
Jim Mattson91e86d22018-10-16 14:29:21 -0700681 vcpu->arch.exception.has_payload = has_payload;
682 vcpu->arch.exception.payload = payload;
Oliver Uptona06230b2020-02-07 02:36:06 -0800683 if (!is_guest_mode(vcpu))
Sean Christophersond4963e32022-08-30 23:16:01 +0000684 kvm_deliver_exception_payload(vcpu,
685 &vcpu->arch.exception);
Eddie Dong3fd28fc2009-11-19 17:54:07 +0200686 return;
687 }
688
689 /* to check exception */
Sean Christophersond4963e32022-08-30 23:16:01 +0000690 prev_nr = vcpu->arch.exception.vector;
Eddie Dong3fd28fc2009-11-19 17:54:07 +0200691 if (prev_nr == DF_VECTOR) {
692 /* triple fault -> shutdown */
Avi Kivitya8eeb042010-05-10 12:34:53 +0300693 kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
Eddie Dong3fd28fc2009-11-19 17:54:07 +0200694 return;
695 }
696 class1 = exception_class(prev_nr);
697 class2 = exception_class(nr);
Sean Christopherson81601492022-08-30 23:16:03 +0000698 if ((class1 == EXCPT_CONTRIBUTORY && class2 == EXCPT_CONTRIBUTORY) ||
699 (class1 == EXCPT_PF && class2 != EXCPT_BENIGN)) {
Wanpeng Li664f8e22017-08-24 03:35:09 -0700700 /*
Sean Christopherson81601492022-08-30 23:16:03 +0000701 * Synthesize #DF. Clear the previously injected or pending
702 * exception so as not to incorrectly trigger shutdown.
Wanpeng Li664f8e22017-08-24 03:35:09 -0700703 */
Wanpeng Li664f8e22017-08-24 03:35:09 -0700704 vcpu->arch.exception.injected = false;
Sean Christopherson81601492022-08-30 23:16:03 +0000705 vcpu->arch.exception.pending = false;
706
707 kvm_queue_exception_e(vcpu, DF_VECTOR, 0);
708 } else {
Eddie Dong3fd28fc2009-11-19 17:54:07 +0200709 /* replace previous exception with a new one in a hope
710 that instruction re-execution will regenerate lost
711 exception */
712 goto queue;
Sean Christopherson81601492022-08-30 23:16:03 +0000713 }
Eddie Dong3fd28fc2009-11-19 17:54:07 +0200714}
715
Avi Kivity298101d2007-11-25 13:41:11 +0200716void kvm_queue_exception(struct kvm_vcpu *vcpu, unsigned nr)
717{
Jim Mattson91e86d22018-10-16 14:29:21 -0700718 kvm_multiple_exception(vcpu, nr, false, 0, false, 0, false);
Avi Kivity298101d2007-11-25 13:41:11 +0200719}
720EXPORT_SYMBOL_GPL(kvm_queue_exception);
721
Joerg Roedelce7ddec2010-04-22 12:33:13 +0200722void kvm_requeue_exception(struct kvm_vcpu *vcpu, unsigned nr)
723{
Jim Mattson91e86d22018-10-16 14:29:21 -0700724 kvm_multiple_exception(vcpu, nr, false, 0, false, 0, true);
Joerg Roedelce7ddec2010-04-22 12:33:13 +0200725}
726EXPORT_SYMBOL_GPL(kvm_requeue_exception);
727
Paolo Bonzini4d5523c2020-05-05 07:33:20 -0400728void kvm_queue_exception_p(struct kvm_vcpu *vcpu, unsigned nr,
729 unsigned long payload)
Jim Mattsonf10c7292018-10-16 14:29:23 -0700730{
731 kvm_multiple_exception(vcpu, nr, false, 0, true, payload, false);
732}
Paolo Bonzini4d5523c2020-05-05 07:33:20 -0400733EXPORT_SYMBOL_GPL(kvm_queue_exception_p);
Jim Mattsonf10c7292018-10-16 14:29:23 -0700734
Jim Mattsonda998b42018-10-16 14:29:22 -0700735static void kvm_queue_exception_e_p(struct kvm_vcpu *vcpu, unsigned nr,
736 u32 error_code, unsigned long payload)
737{
738 kvm_multiple_exception(vcpu, nr, true, error_code,
739 true, payload, false);
740}
741
Kyle Huey6affcbe2016-11-29 12:40:40 -0800742int kvm_complete_insn_gp(struct kvm_vcpu *vcpu, int err)
Avi Kivityc3c91fe2007-11-25 14:04:58 +0200743{
Andre Przywaradb8fcef2010-12-21 11:12:01 +0100744 if (err)
745 kvm_inject_gp(vcpu, 0);
746 else
Kyle Huey6affcbe2016-11-29 12:40:40 -0800747 return kvm_skip_emulated_instruction(vcpu);
748
749 return 1;
Andre Przywaradb8fcef2010-12-21 11:12:01 +0100750}
751EXPORT_SYMBOL_GPL(kvm_complete_insn_gp);
Joerg Roedel8df25a32010-09-10 17:30:46 +0200752
Hou Wenlongd2f7d492021-11-02 17:15:31 +0800753static int complete_emulated_insn_gp(struct kvm_vcpu *vcpu, int err)
754{
755 if (err) {
756 kvm_inject_gp(vcpu, 0);
757 return 1;
758 }
759
760 return kvm_emulate_instruction(vcpu, EMULTYPE_NO_DECODE | EMULTYPE_SKIP |
761 EMULTYPE_COMPLETE_USER_EXIT);
762}
763
Avi Kivity6389ee92010-11-29 16:12:30 +0200764void kvm_inject_page_fault(struct kvm_vcpu *vcpu, struct x86_exception *fault)
Avi Kivityc3c91fe2007-11-25 14:04:58 +0200765{
Avi Kivityc3c91fe2007-11-25 14:04:58 +0200766 ++vcpu->stat.pf_guest;
Sean Christopherson7709aba2022-08-30 23:16:08 +0000767
768 /*
769 * Async #PF in L2 is always forwarded to L1 as a VM-Exit regardless of
770 * whether or not L1 wants to intercept "regular" #PF.
771 */
772 if (is_guest_mode(vcpu) && fault->async_page_fault)
773 kvm_queue_exception_vmexit(vcpu, PF_VECTOR,
774 true, fault->error_code,
775 true, fault->address);
776 else
Jim Mattsonda998b42018-10-16 14:29:22 -0700777 kvm_queue_exception_e_p(vcpu, PF_VECTOR, fault->error_code,
778 fault->address);
Avi Kivityc3c91fe2007-11-25 14:04:58 +0200779}
Nadav Har'El27d6c862011-05-25 23:06:59 +0300780EXPORT_SYMBOL_GPL(kvm_inject_page_fault);
Avi Kivityc3c91fe2007-11-25 14:04:58 +0200781
Sean Christopherson7709aba2022-08-30 23:16:08 +0000782void kvm_inject_emulated_page_fault(struct kvm_vcpu *vcpu,
Sean Christopherson53b3d8e2020-03-20 14:28:01 -0700783 struct x86_exception *fault)
Joerg Roedeld4f8cf62010-09-10 17:30:55 +0200784{
Paolo Bonzini0cd665b2020-03-25 12:50:03 -0400785 struct kvm_mmu *fault_mmu;
Sean Christopherson53b3d8e2020-03-20 14:28:01 -0700786 WARN_ON_ONCE(fault->vector != PF_VECTOR);
787
Paolo Bonzini0cd665b2020-03-25 12:50:03 -0400788 fault_mmu = fault->nested_page_fault ? vcpu->arch.mmu :
789 vcpu->arch.walk_mmu;
Paolo Bonzinief54bcf2014-09-04 19:46:15 +0200790
Junaid Shahidee1fa202020-03-20 14:28:03 -0700791 /*
792 * Invalidate the TLB entry for the faulting address, if it exists,
793 * else the access will fault indefinitely (and to emulate hardware).
794 */
795 if ((fault->error_code & PFERR_PRESENT_MASK) &&
796 !(fault->error_code & PFERR_RSVD_MASK))
797 kvm_mmu_invalidate_gva(vcpu, fault_mmu, fault->address,
Paolo Bonzinib9e56032022-02-21 09:28:33 -0500798 fault_mmu->root.hpa);
Junaid Shahidee1fa202020-03-20 14:28:03 -0700799
800 fault_mmu->inject_page_fault(vcpu, fault);
Joerg Roedeld4f8cf62010-09-10 17:30:55 +0200801}
Sean Christopherson53b3d8e2020-03-20 14:28:01 -0700802EXPORT_SYMBOL_GPL(kvm_inject_emulated_page_fault);
Joerg Roedeld4f8cf62010-09-10 17:30:55 +0200803
Sheng Yang3419ffc2008-05-15 09:52:48 +0800804void kvm_inject_nmi(struct kvm_vcpu *vcpu)
805{
Avi Kivity7460fb4a2011-09-20 13:43:14 +0300806 atomic_inc(&vcpu->arch.nmi_queued);
807 kvm_make_request(KVM_REQ_NMI, vcpu);
Sheng Yang3419ffc2008-05-15 09:52:48 +0800808}
809EXPORT_SYMBOL_GPL(kvm_inject_nmi);
810
Avi Kivity298101d2007-11-25 13:41:11 +0200811void kvm_queue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code)
812{
Jim Mattson91e86d22018-10-16 14:29:21 -0700813 kvm_multiple_exception(vcpu, nr, true, error_code, false, 0, false);
Avi Kivity298101d2007-11-25 13:41:11 +0200814}
815EXPORT_SYMBOL_GPL(kvm_queue_exception_e);
816
Joerg Roedelce7ddec2010-04-22 12:33:13 +0200817void kvm_requeue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code)
818{
Jim Mattson91e86d22018-10-16 14:29:21 -0700819 kvm_multiple_exception(vcpu, nr, true, error_code, false, 0, true);
Joerg Roedelce7ddec2010-04-22 12:33:13 +0200820}
821EXPORT_SYMBOL_GPL(kvm_requeue_exception_e);
822
Carsten Ottea03490e2007-10-29 16:09:35 +0100823/*
Avi Kivity0a79b002009-09-01 12:03:25 +0300824 * Checks if cpl <= required_cpl; if true, return true. Otherwise queue
825 * a #GP and return false.
826 */
827bool kvm_require_cpl(struct kvm_vcpu *vcpu, int required_cpl)
Carsten Otte043405e2007-10-10 17:16:19 +0200828{
Jason Baronb36464772021-01-14 22:27:56 -0500829 if (static_call(kvm_x86_get_cpl)(vcpu) <= required_cpl)
Avi Kivity0a79b002009-09-01 12:03:25 +0300830 return true;
831 kvm_queue_exception_e(vcpu, GP_VECTOR, 0);
832 return false;
Carsten Ottea03490e2007-10-29 16:09:35 +0100833}
Avi Kivity0a79b002009-09-01 12:03:25 +0300834EXPORT_SYMBOL_GPL(kvm_require_cpl);
Carsten Ottea03490e2007-10-29 16:09:35 +0100835
Nadav Amit16f8a6f2014-10-03 01:10:05 +0300836bool kvm_require_dr(struct kvm_vcpu *vcpu, int dr)
837{
838 if ((dr != 4 && dr != 5) || !kvm_read_cr4_bits(vcpu, X86_CR4_DE))
839 return true;
840
841 kvm_queue_exception(vcpu, UD_VECTOR);
842 return false;
843}
844EXPORT_SYMBOL_GPL(kvm_require_dr);
845
Sean Christopherson16cfacc2019-09-03 16:36:45 -0700846static inline u64 pdptr_rsvd_bits(struct kvm_vcpu *vcpu)
847{
Sean Christopherson5b7f5752021-02-03 16:01:13 -0800848 return vcpu->arch.reserved_gpa_bits | rsvd_bits(5, 8) | rsvd_bits(1, 2);
Sean Christopherson16cfacc2019-09-03 16:36:45 -0700849}
850
Joerg Roedelec92fe42010-09-10 17:30:51 +0200851/*
Sean Christopherson16cfacc2019-09-03 16:36:45 -0700852 * Load the pae pdptrs. Return 1 if they are all valid, 0 otherwise.
Carsten Ottea03490e2007-10-29 16:09:35 +0100853 */
Lai Jiangshan2df4a5e2021-11-24 20:20:52 +0800854int load_pdptrs(struct kvm_vcpu *vcpu, unsigned long cr3)
Carsten Ottea03490e2007-10-29 16:09:35 +0100855{
Lai Jiangshan2df4a5e2021-11-24 20:20:52 +0800856 struct kvm_mmu *mmu = vcpu->arch.walk_mmu;
Carsten Ottea03490e2007-10-29 16:09:35 +0100857 gfn_t pdpt_gfn = cr3 >> PAGE_SHIFT;
Sean Christopherson15cabbc2021-08-31 09:42:23 -0700858 gpa_t real_gpa;
Carsten Ottea03490e2007-10-29 16:09:35 +0100859 int i;
860 int ret;
Joerg Roedelff03a072010-09-10 17:30:57 +0200861 u64 pdpte[ARRAY_SIZE(mmu->pdptrs)];
Carsten Ottea03490e2007-10-29 16:09:35 +0100862
Sean Christopherson15cabbc2021-08-31 09:42:23 -0700863 /*
864 * If the MMU is nested, CR3 holds an L2 GPA and needs to be translated
865 * to an L1 GPA.
866 */
Lai Jiangshanc59a0f52021-11-24 20:20:45 +0800867 real_gpa = kvm_translate_gpa(vcpu, mmu, gfn_to_gpa(pdpt_gfn),
868 PFERR_USER_MASK | PFERR_WRITE_MASK, NULL);
Hou Wenlong6e1d2a32022-07-01 17:24:13 +0800869 if (real_gpa == INVALID_GPA)
Sean Christopherson15cabbc2021-08-31 09:42:23 -0700870 return 0;
871
Sean Christopherson94c641b2021-08-31 09:42:24 -0700872 /* Note the offset, PDPTRs are 32 byte aligned when using PAE paging. */
Sean Christopherson15cabbc2021-08-31 09:42:23 -0700873 ret = kvm_vcpu_read_guest_page(vcpu, gpa_to_gfn(real_gpa), pdpte,
Sean Christopherson94c641b2021-08-31 09:42:24 -0700874 cr3 & GENMASK(11, 5), sizeof(pdpte));
Sean Christopherson15cabbc2021-08-31 09:42:23 -0700875 if (ret < 0)
876 return 0;
877
Carsten Ottea03490e2007-10-29 16:09:35 +0100878 for (i = 0; i < ARRAY_SIZE(pdpte); ++i) {
Bandan Das812f30b2016-07-12 18:18:50 -0400879 if ((pdpte[i] & PT_PRESENT_MASK) &&
Sean Christopherson16cfacc2019-09-03 16:36:45 -0700880 (pdpte[i] & pdptr_rsvd_bits(vcpu))) {
Sean Christopherson15cabbc2021-08-31 09:42:23 -0700881 return 0;
Carsten Ottea03490e2007-10-29 16:09:35 +0100882 }
883 }
Carsten Ottea03490e2007-10-29 16:09:35 +0100884
Lai Jiangshan6b123c32021-12-16 10:19:37 +0800885 /*
886 * Marking VCPU_EXREG_PDPTR dirty doesn't work for !tdp_enabled.
887 * Shadow page roots need to be reconstructed instead.
888 */
889 if (!tdp_enabled && memcmp(mmu->pdptrs, pdpte, sizeof(mmu->pdptrs)))
Paolo Bonzini0c1c92f2022-02-21 09:31:51 -0500890 kvm_mmu_free_roots(vcpu->kvm, mmu, KVM_MMU_ROOT_CURRENT);
Lai Jiangshan6b123c32021-12-16 10:19:37 +0800891
Paolo Bonzini46cbc042021-12-10 18:13:37 -0500892 memcpy(mmu->pdptrs, pdpte, sizeof(mmu->pdptrs));
893 kvm_register_mark_dirty(vcpu, VCPU_EXREG_PDPTR);
894 kvm_make_request(KVM_REQ_LOAD_MMU_PGD, vcpu);
Maxim Levitsky158a48e2021-06-07 12:02:03 +0300895 vcpu->arch.pdptrs_from_userspace = false;
896
Sean Christopherson15cabbc2021-08-31 09:42:23 -0700897 return 1;
Carsten Ottea03490e2007-10-29 16:09:35 +0100898}
Joerg Roedelcc4b6872008-02-07 13:47:43 +0100899EXPORT_SYMBOL_GPL(load_pdptrs);
Carsten Ottea03490e2007-10-29 16:09:35 +0100900
Tom Lendackyf27ad382020-12-10 11:09:56 -0600901void kvm_post_set_cr0(struct kvm_vcpu *vcpu, unsigned long old_cr0, unsigned long cr0)
902{
Tom Lendackyf27ad382020-12-10 11:09:56 -0600903 if ((cr0 ^ old_cr0) & X86_CR0_PG) {
904 kvm_clear_async_pf_completion_queue(vcpu);
905 kvm_async_pf_hash_reset(vcpu);
Paolo Bonzinib5f61c02022-02-14 11:50:36 -0500906
907 /*
908 * Clearing CR0.PG is defined to flush the TLB from the guest's
909 * perspective.
910 */
911 if (!(cr0 & X86_CR0_PG))
912 kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu);
Tom Lendackyf27ad382020-12-10 11:09:56 -0600913 }
914
Sean Christopherson20f632b2021-06-22 10:57:02 -0700915 if ((cr0 ^ old_cr0) & KVM_MMU_CR0_ROLE_BITS)
Tom Lendackyf27ad382020-12-10 11:09:56 -0600916 kvm_mmu_reset_context(vcpu);
917
918 if (((cr0 ^ old_cr0) & X86_CR0_CD) &&
919 kvm_arch_has_noncoherent_dma(vcpu->kvm) &&
920 !kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_CD_NW_CLEARED))
921 kvm_zap_gfn_range(vcpu->kvm, 0, ~0ULL);
922}
923EXPORT_SYMBOL_GPL(kvm_post_set_cr0);
924
Avi Kivity49a9b072010-06-10 17:02:14 +0300925int kvm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
Carsten Ottea03490e2007-10-29 16:09:35 +0100926{
Sheng Yangaad82702010-05-12 16:40:42 +0800927 unsigned long old_cr0 = kvm_read_cr0(vcpu);
Sheng Yangaad82702010-05-12 16:40:42 +0800928
Avi Kivityf9a48e62010-01-06 19:10:22 +0200929 cr0 |= X86_CR0_ET;
930
Gleb Natapovab344822010-01-21 15:28:46 +0200931#ifdef CONFIG_X86_64
Gleb Natapov0f122442010-04-28 19:15:31 +0300932 if (cr0 & 0xffffffff00000000UL)
933 return 1;
Gleb Natapovab344822010-01-21 15:28:46 +0200934#endif
935
936 cr0 &= ~CR0_RESERVED_BITS;
Carsten Ottea03490e2007-10-29 16:09:35 +0100937
Gleb Natapov0f122442010-04-28 19:15:31 +0300938 if ((cr0 & X86_CR0_NW) && !(cr0 & X86_CR0_CD))
939 return 1;
Carsten Ottea03490e2007-10-29 16:09:35 +0100940
Gleb Natapov0f122442010-04-28 19:15:31 +0300941 if ((cr0 & X86_CR0_PG) && !(cr0 & X86_CR0_PE))
942 return 1;
Carsten Ottea03490e2007-10-29 16:09:35 +0100943
Carsten Ottea03490e2007-10-29 16:09:35 +0100944#ifdef CONFIG_X86_64
Sean Christopherson05487212020-07-13 18:57:32 -0700945 if ((vcpu->arch.efer & EFER_LME) && !is_paging(vcpu) &&
946 (cr0 & X86_CR0_PG)) {
947 int cs_db, cs_l;
Carsten Ottea03490e2007-10-29 16:09:35 +0100948
Sean Christopherson05487212020-07-13 18:57:32 -0700949 if (!is_pae(vcpu))
950 return 1;
Jason Baronb36464772021-01-14 22:27:56 -0500951 static_call(kvm_x86_get_cs_db_l_bits)(vcpu, &cs_db, &cs_l);
Sean Christopherson05487212020-07-13 18:57:32 -0700952 if (cs_l)
Gleb Natapov0f122442010-04-28 19:15:31 +0300953 return 1;
Carsten Ottea03490e2007-10-29 16:09:35 +0100954 }
Sean Christopherson05487212020-07-13 18:57:32 -0700955#endif
956 if (!(vcpu->arch.efer & EFER_LME) && (cr0 & X86_CR0_PG) &&
Lai Jiangshane63f3152021-11-08 20:43:58 +0800957 is_pae(vcpu) && ((cr0 ^ old_cr0) & X86_CR0_PDPTR_BITS) &&
Lai Jiangshan2df4a5e2021-11-24 20:20:52 +0800958 !load_pdptrs(vcpu, kvm_read_cr3(vcpu)))
Sean Christopherson05487212020-07-13 18:57:32 -0700959 return 1;
Carsten Ottea03490e2007-10-29 16:09:35 +0100960
Lai Jiangshan777ab822021-12-07 17:52:30 +0800961 if (!(cr0 & X86_CR0_PG) &&
962 (is_64_bit_mode(vcpu) || kvm_read_cr4_bits(vcpu, X86_CR4_PCIDE)))
Mao, Junjiead756a12012-07-02 01:18:48 +0000963 return 1;
964
Jason Baronb36464772021-01-14 22:27:56 -0500965 static_call(kvm_x86_set_cr0)(vcpu, cr0);
Carsten Ottea03490e2007-10-29 16:09:35 +0100966
Tom Lendackyf27ad382020-12-10 11:09:56 -0600967 kvm_post_set_cr0(vcpu, old_cr0, cr0);
Xiao Guangrongb18d5432015-06-15 16:55:21 +0800968
Gleb Natapov0f122442010-04-28 19:15:31 +0300969 return 0;
970}
Avi Kivity2d3ad1f2008-02-24 11:20:43 +0200971EXPORT_SYMBOL_GPL(kvm_set_cr0);
Carsten Ottea03490e2007-10-29 16:09:35 +0100972
Avi Kivity2d3ad1f2008-02-24 11:20:43 +0200973void kvm_lmsw(struct kvm_vcpu *vcpu, unsigned long msw)
Carsten Ottea03490e2007-10-29 16:09:35 +0100974{
Avi Kivity49a9b072010-06-10 17:02:14 +0300975 (void)kvm_set_cr0(vcpu, kvm_read_cr0_bits(vcpu, ~0x0eul) | (msw & 0x0f));
Carsten Ottea03490e2007-10-29 16:09:35 +0100976}
Avi Kivity2d3ad1f2008-02-24 11:20:43 +0200977EXPORT_SYMBOL_GPL(kvm_lmsw);
Carsten Ottea03490e2007-10-29 16:09:35 +0100978
Aaron Lewis139a12c2019-10-21 16:30:25 -0700979void kvm_load_guest_xsave_state(struct kvm_vcpu *vcpu)
Marcelo Tosatti42bdf9912013-04-15 23:30:13 -0300980{
Tom Lendacky16809ec2020-12-10 11:10:08 -0600981 if (vcpu->arch.guest_state_protected)
982 return;
983
Aaron Lewis139a12c2019-10-21 16:30:25 -0700984 if (kvm_read_cr4_bits(vcpu, X86_CR4_OSXSAVE)) {
Marcelo Tosatti42bdf9912013-04-15 23:30:13 -0300985
Aaron Lewis139a12c2019-10-21 16:30:25 -0700986 if (vcpu->arch.xcr0 != host_xcr0)
987 xsetbv(XCR_XFEATURE_ENABLED_MASK, vcpu->arch.xcr0);
988
989 if (vcpu->arch.xsaves_enabled &&
990 vcpu->arch.ia32_xss != host_xss)
991 wrmsrl(MSR_IA32_XSS, vcpu->arch.ia32_xss);
992 }
Babu Moger37486132020-05-12 18:59:06 -0500993
Jon Kohler945024d2022-03-23 20:44:39 -0400994#ifdef CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS
Babu Moger37486132020-05-12 18:59:06 -0500995 if (static_cpu_has(X86_FEATURE_PKU) &&
Jon Kohler945024d2022-03-23 20:44:39 -0400996 vcpu->arch.pkru != vcpu->arch.host_pkru &&
997 ((vcpu->arch.xcr0 & XFEATURE_MASK_PKRU) ||
998 kvm_read_cr4_bits(vcpu, X86_CR4_PKE)))
Thomas Gleixner72a6c082021-06-23 14:02:23 +0200999 write_pkru(vcpu->arch.pkru);
Jon Kohler945024d2022-03-23 20:44:39 -04001000#endif /* CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS */
Marcelo Tosatti42bdf9912013-04-15 23:30:13 -03001001}
Aaron Lewis139a12c2019-10-21 16:30:25 -07001002EXPORT_SYMBOL_GPL(kvm_load_guest_xsave_state);
1003
1004void kvm_load_host_xsave_state(struct kvm_vcpu *vcpu)
1005{
Tom Lendacky16809ec2020-12-10 11:10:08 -06001006 if (vcpu->arch.guest_state_protected)
1007 return;
1008
Jon Kohler945024d2022-03-23 20:44:39 -04001009#ifdef CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS
Babu Moger37486132020-05-12 18:59:06 -05001010 if (static_cpu_has(X86_FEATURE_PKU) &&
Jon Kohler945024d2022-03-23 20:44:39 -04001011 ((vcpu->arch.xcr0 & XFEATURE_MASK_PKRU) ||
1012 kvm_read_cr4_bits(vcpu, X86_CR4_PKE))) {
Babu Moger37486132020-05-12 18:59:06 -05001013 vcpu->arch.pkru = rdpkru();
1014 if (vcpu->arch.pkru != vcpu->arch.host_pkru)
Thomas Gleixner72a6c082021-06-23 14:02:23 +02001015 write_pkru(vcpu->arch.host_pkru);
Babu Moger37486132020-05-12 18:59:06 -05001016 }
Jon Kohler945024d2022-03-23 20:44:39 -04001017#endif /* CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS */
Babu Moger37486132020-05-12 18:59:06 -05001018
Aaron Lewis139a12c2019-10-21 16:30:25 -07001019 if (kvm_read_cr4_bits(vcpu, X86_CR4_OSXSAVE)) {
1020
1021 if (vcpu->arch.xcr0 != host_xcr0)
1022 xsetbv(XCR_XFEATURE_ENABLED_MASK, host_xcr0);
1023
1024 if (vcpu->arch.xsaves_enabled &&
1025 vcpu->arch.ia32_xss != host_xss)
1026 wrmsrl(MSR_IA32_XSS, host_xss);
1027 }
1028
1029}
1030EXPORT_SYMBOL_GPL(kvm_load_host_xsave_state);
Marcelo Tosatti42bdf9912013-04-15 23:30:13 -03001031
Leonardo Brasba1f77c2022-02-18 00:41:00 -03001032#ifdef CONFIG_X86_64
Leonardo Bras988896b2022-02-17 02:30:30 -03001033static inline u64 kvm_guest_supported_xfd(struct kvm_vcpu *vcpu)
1034{
Sean Christophersonee519b32022-08-24 03:30:55 +00001035 return vcpu->arch.guest_supported_xcr0 & XFEATURE_MASK_USER_DYNAMIC;
Leonardo Bras988896b2022-02-17 02:30:30 -03001036}
Leonardo Brasba1f77c2022-02-18 00:41:00 -03001037#endif
Leonardo Bras988896b2022-02-17 02:30:30 -03001038
Fengguang Wu69b00492015-01-19 22:33:39 +08001039static int __kvm_set_xcr(struct kvm_vcpu *vcpu, u32 index, u64 xcr)
Dexuan Cui2acf9232010-06-10 11:27:12 +08001040{
Liu, Jinsong56c103e2014-02-21 17:39:02 +00001041 u64 xcr0 = xcr;
1042 u64 old_xcr0 = vcpu->arch.xcr0;
Paolo Bonzini46c34cb2013-10-17 16:50:46 +02001043 u64 valid_bits;
Dexuan Cui2acf9232010-06-10 11:27:12 +08001044
1045 /* Only support XCR_XFEATURE_ENABLED_MASK(xcr0) now */
1046 if (index != XCR_XFEATURE_ENABLED_MASK)
1047 return 1;
Dave Hansend91cab72015-09-02 16:31:26 -07001048 if (!(xcr0 & XFEATURE_MASK_FP))
Dexuan Cui2acf9232010-06-10 11:27:12 +08001049 return 1;
Dave Hansend91cab72015-09-02 16:31:26 -07001050 if ((xcr0 & XFEATURE_MASK_YMM) && !(xcr0 & XFEATURE_MASK_SSE))
Dexuan Cui2acf9232010-06-10 11:27:12 +08001051 return 1;
Paolo Bonzini46c34cb2013-10-17 16:50:46 +02001052
1053 /*
1054 * Do not allow the guest to set bits that we do not support
1055 * saving. However, xcr0 bit 0 is always set, even if the
Sean Christophersone8f65b92021-09-20 17:02:58 -07001056 * emulated CPU does not support XSAVE (see kvm_vcpu_reset()).
Paolo Bonzini46c34cb2013-10-17 16:50:46 +02001057 */
Sean Christophersonee519b32022-08-24 03:30:55 +00001058 valid_bits = vcpu->arch.guest_supported_xcr0 | XFEATURE_MASK_FP;
Paolo Bonzini46c34cb2013-10-17 16:50:46 +02001059 if (xcr0 & ~valid_bits)
Dexuan Cui2acf9232010-06-10 11:27:12 +08001060 return 1;
Paolo Bonzini46c34cb2013-10-17 16:50:46 +02001061
Dave Hansend91cab72015-09-02 16:31:26 -07001062 if ((!(xcr0 & XFEATURE_MASK_BNDREGS)) !=
1063 (!(xcr0 & XFEATURE_MASK_BNDCSR)))
Liu, Jinsong390bd5282014-02-24 10:58:09 +00001064 return 1;
1065
Dave Hansend91cab72015-09-02 16:31:26 -07001066 if (xcr0 & XFEATURE_MASK_AVX512) {
1067 if (!(xcr0 & XFEATURE_MASK_YMM))
Chao Peng612263b2014-10-22 17:35:24 +08001068 return 1;
Dave Hansend91cab72015-09-02 16:31:26 -07001069 if ((xcr0 & XFEATURE_MASK_AVX512) != XFEATURE_MASK_AVX512)
Chao Peng612263b2014-10-22 17:35:24 +08001070 return 1;
1071 }
Jing Liu86aff7a42022-01-05 04:35:26 -08001072
1073 if ((xcr0 & XFEATURE_MASK_XTILE) &&
1074 ((xcr0 & XFEATURE_MASK_XTILE) != XFEATURE_MASK_XTILE))
1075 return 1;
1076
Dexuan Cui2acf9232010-06-10 11:27:12 +08001077 vcpu->arch.xcr0 = xcr0;
Liu, Jinsong56c103e2014-02-21 17:39:02 +00001078
Dave Hansend91cab72015-09-02 16:31:26 -07001079 if ((xcr0 ^ old_xcr0) & XFEATURE_MASK_EXTEND)
Xiaoyao Liaedbaf42020-07-09 12:34:23 +08001080 kvm_update_cpuid_runtime(vcpu);
Dexuan Cui2acf9232010-06-10 11:27:12 +08001081 return 0;
1082}
1083
Sean Christopherson92f98952021-02-04 16:57:46 -08001084int kvm_emulate_xsetbv(struct kvm_vcpu *vcpu)
Dexuan Cui2acf9232010-06-10 11:27:12 +08001085{
Sean Christopherson50b2d492022-08-24 03:30:57 +00001086 /* Note, #UD due to CR4.OSXSAVE=0 has priority over the intercept. */
Sean Christopherson92f98952021-02-04 16:57:46 -08001087 if (static_call(kvm_x86_get_cpl)(vcpu) != 0 ||
1088 __kvm_set_xcr(vcpu, kvm_rcx_read(vcpu), kvm_read_edx_eax(vcpu))) {
1089 kvm_inject_gp(vcpu, 0);
1090 return 1;
1091 }
Paolo Bonzinibbefd4f2020-12-14 07:49:54 -05001092
Sean Christopherson92f98952021-02-04 16:57:46 -08001093 return kvm_skip_emulated_instruction(vcpu);
Dexuan Cui2acf9232010-06-10 11:27:12 +08001094}
Sean Christopherson92f98952021-02-04 16:57:46 -08001095EXPORT_SYMBOL_GPL(kvm_emulate_xsetbv);
Dexuan Cui2acf9232010-06-10 11:27:12 +08001096
Sean Christophersonc33f6f2222022-06-07 21:35:50 +00001097bool __kvm_is_valid_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
Wanpeng Li3ca94192019-09-18 17:50:10 +08001098{
Sean Christophersonb11306b2019-12-10 14:44:13 -08001099 if (cr4 & cr4_reserved_bits)
Sean Christophersonee69c922020-10-06 18:44:16 -07001100 return false;
Wanpeng Li3ca94192019-09-18 17:50:10 +08001101
Krish Sadhukhanb899c132020-07-08 00:39:55 +00001102 if (cr4 & vcpu->arch.cr4_guest_rsvd_bits)
Sean Christophersonee69c922020-10-06 18:44:16 -07001103 return false;
Wanpeng Li3ca94192019-09-18 17:50:10 +08001104
Sean Christophersonc33f6f2222022-06-07 21:35:50 +00001105 return true;
Wanpeng Li3ca94192019-09-18 17:50:10 +08001106}
Sean Christophersonc33f6f2222022-06-07 21:35:50 +00001107EXPORT_SYMBOL_GPL(__kvm_is_valid_cr4);
1108
1109static bool kvm_is_valid_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
1110{
1111 return __kvm_is_valid_cr4(vcpu, cr4) &&
1112 static_call(kvm_x86_is_valid_cr4)(vcpu, cr4);
1113}
Wanpeng Li3ca94192019-09-18 17:50:10 +08001114
Tom Lendacky5b51cb12020-12-10 11:09:57 -06001115void kvm_post_set_cr4(struct kvm_vcpu *vcpu, unsigned long old_cr4, unsigned long cr4)
1116{
Paolo Bonzinib5f61c02022-02-14 11:50:36 -05001117 if ((cr4 ^ old_cr4) & KVM_MMU_CR4_ROLE_BITS)
1118 kvm_mmu_reset_context(vcpu);
1119
Lai Jiangshan509bfe32021-10-19 19:01:52 +08001120 /*
Lai Jiangshan509bfe32021-10-19 19:01:52 +08001121 * If CR4.PCIDE is changed 0 -> 1, there is no need to flush the TLB
1122 * according to the SDM; however, stale prev_roots could be reused
1123 * incorrectly in the future after a MOV to CR3 with NOFLUSH=1, so we
Paolo Bonzinib5f61c02022-02-14 11:50:36 -05001124 * free them all. This is *not* a superset of KVM_REQ_TLB_FLUSH_GUEST
1125 * or KVM_REQ_TLB_FLUSH_CURRENT, because the hardware TLB is not flushed,
1126 * so fall through.
Lai Jiangshan509bfe32021-10-19 19:01:52 +08001127 */
Paolo Bonzinib5f61c02022-02-14 11:50:36 -05001128 if (!tdp_enabled &&
1129 (cr4 & X86_CR4_PCIDE) && !(old_cr4 & X86_CR4_PCIDE))
Sean Christophersonf6d0a252022-02-25 18:22:43 +00001130 kvm_mmu_unload(vcpu);
Paolo Bonzinib5f61c02022-02-14 11:50:36 -05001131
1132 /*
1133 * The TLB has to be flushed for all PCIDs if any of the following
1134 * (architecturally required) changes happen:
1135 * - CR4.PCIDE is changed from 1 to 0
1136 * - CR4.PGE is toggled
1137 *
1138 * This is a superset of KVM_REQ_TLB_FLUSH_CURRENT.
1139 */
1140 if (((cr4 ^ old_cr4) & X86_CR4_PGE) ||
1141 (!(cr4 & X86_CR4_PCIDE) && (old_cr4 & X86_CR4_PCIDE)))
Lai Jiangshan55261732021-09-19 10:42:45 +08001142 kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu);
Paolo Bonzinib5f61c02022-02-14 11:50:36 -05001143
1144 /*
1145 * The TLB has to be flushed for the current PCID if any of the
1146 * following (architecturally required) changes happen:
1147 * - CR4.SMEP is changed from 0 to 1
1148 * - CR4.PAE is toggled
1149 */
1150 else if (((cr4 ^ old_cr4) & X86_CR4_PAE) ||
1151 ((cr4 & X86_CR4_SMEP) && !(old_cr4 & X86_CR4_SMEP)))
1152 kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu);
1153
Tom Lendacky5b51cb12020-12-10 11:09:57 -06001154}
1155EXPORT_SYMBOL_GPL(kvm_post_set_cr4);
Dexuan Cui2acf9232010-06-10 11:27:12 +08001156
Avi Kivitya83b29c2010-06-10 17:02:15 +03001157int kvm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
Carsten Ottea03490e2007-10-29 16:09:35 +01001158{
Avi Kivityfc78f512009-12-07 12:16:48 +02001159 unsigned long old_cr4 = kvm_read_cr4(vcpu);
Xiao Guangrong0be02262015-05-11 22:55:21 +08001160
Sean Christophersonee69c922020-10-06 18:44:16 -07001161 if (!kvm_is_valid_cr4(vcpu, cr4))
Paolo Bonziniae3e61e2016-07-12 10:36:41 +02001162 return 1;
1163
Carsten Ottea03490e2007-10-29 16:09:35 +01001164 if (is_long_mode(vcpu)) {
Gleb Natapov0f122442010-04-28 19:15:31 +03001165 if (!(cr4 & X86_CR4_PAE))
1166 return 1;
Sean Christophersond74fcfc2020-07-02 19:17:14 -07001167 if ((cr4 ^ old_cr4) & X86_CR4_LA57)
1168 return 1;
Avi Kivitya2edf572009-05-24 22:19:00 +03001169 } else if (is_paging(vcpu) && (cr4 & X86_CR4_PAE)
Lai Jiangshana37ebdc2021-11-08 20:43:57 +08001170 && ((cr4 ^ old_cr4) & X86_CR4_PDPTR_BITS)
Lai Jiangshan2df4a5e2021-11-24 20:20:52 +08001171 && !load_pdptrs(vcpu, kvm_read_cr3(vcpu)))
Gleb Natapov0f122442010-04-28 19:15:31 +03001172 return 1;
Carsten Ottea03490e2007-10-29 16:09:35 +01001173
Mao, Junjiead756a12012-07-02 01:18:48 +00001174 if ((cr4 & X86_CR4_PCIDE) && !(old_cr4 & X86_CR4_PCIDE)) {
Radim Krčmářd6321d42017-08-05 00:12:49 +02001175 if (!guest_cpuid_has(vcpu, X86_FEATURE_PCID))
Mao, Junjiead756a12012-07-02 01:18:48 +00001176 return 1;
1177
1178 /* PCID can not be enabled when cr3[11:0]!=000H or EFER.LMA=0 */
1179 if ((kvm_read_cr3(vcpu) & X86_CR3_PCID_MASK) || !is_long_mode(vcpu))
1180 return 1;
1181 }
1182
Jason Baronb36464772021-01-14 22:27:56 -05001183 static_call(kvm_x86_set_cr4)(vcpu, cr4);
Gleb Natapov0f122442010-04-28 19:15:31 +03001184
Tom Lendacky5b51cb12020-12-10 11:09:57 -06001185 kvm_post_set_cr4(vcpu, old_cr4, cr4);
Dexuan Cui2acf9232010-06-10 11:27:12 +08001186
Gleb Natapov0f122442010-04-28 19:15:31 +03001187 return 0;
1188}
Avi Kivity2d3ad1f2008-02-24 11:20:43 +02001189EXPORT_SYMBOL_GPL(kvm_set_cr4);
Carsten Ottea03490e2007-10-29 16:09:35 +01001190
Sean Christopherson21823fb2021-06-09 16:42:24 -07001191static void kvm_invalidate_pcid(struct kvm_vcpu *vcpu, unsigned long pcid)
1192{
1193 struct kvm_mmu *mmu = vcpu->arch.mmu;
1194 unsigned long roots_to_free = 0;
1195 int i;
1196
1197 /*
Lai Jiangshane45e9e32021-10-19 19:01:51 +08001198 * MOV CR3 and INVPCID are usually not intercepted when using TDP, but
1199 * this is reachable when running EPT=1 and unrestricted_guest=0, and
1200 * also via the emulator. KVM's TDP page tables are not in the scope of
1201 * the invalidation, but the guest's TLB entries need to be flushed as
1202 * the CPU may have cached entries in its TLB for the target PCID.
1203 */
1204 if (unlikely(tdp_enabled)) {
1205 kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu);
1206 return;
1207 }
1208
1209 /*
Sean Christopherson21823fb2021-06-09 16:42:24 -07001210 * If neither the current CR3 nor any of the prev_roots use the given
1211 * PCID, then nothing needs to be done here because a resync will
1212 * happen anyway before switching to any other CR3.
1213 */
1214 if (kvm_get_active_pcid(vcpu) == pcid) {
Sean Christophersone62f1aa2021-06-09 16:42:32 -07001215 kvm_make_request(KVM_REQ_MMU_SYNC, vcpu);
Sean Christopherson21823fb2021-06-09 16:42:24 -07001216 kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu);
1217 }
1218
Lai Jiangshan509bfe32021-10-19 19:01:52 +08001219 /*
1220 * If PCID is disabled, there is no need to free prev_roots even if the
1221 * PCIDs for them are also 0, because MOV to CR3 always flushes the TLB
1222 * with PCIDE=0.
1223 */
1224 if (!kvm_read_cr4_bits(vcpu, X86_CR4_PCIDE))
1225 return;
1226
Sean Christopherson21823fb2021-06-09 16:42:24 -07001227 for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++)
1228 if (kvm_get_pcid(vcpu, mmu->prev_roots[i].pgd) == pcid)
1229 roots_to_free |= KVM_MMU_ROOT_PREVIOUS(i);
1230
Paolo Bonzini0c1c92f2022-02-21 09:31:51 -05001231 kvm_mmu_free_roots(vcpu->kvm, mmu, roots_to_free);
Sean Christopherson21823fb2021-06-09 16:42:24 -07001232}
1233
Avi Kivity23902182010-06-10 17:02:16 +03001234int kvm_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3)
Carsten Ottea03490e2007-10-29 16:09:35 +01001235{
Junaid Shahidade61e22018-06-27 14:59:15 -07001236 bool skip_tlb_flush = false;
Sean Christopherson21823fb2021-06-09 16:42:24 -07001237 unsigned long pcid = 0;
Paolo Bonziniac146232014-11-10 13:53:25 +01001238#ifdef CONFIG_X86_64
Junaid Shahidc19986f2018-05-04 11:37:13 -07001239 bool pcid_enabled = kvm_read_cr4_bits(vcpu, X86_CR4_PCIDE);
1240
Junaid Shahidade61e22018-06-27 14:59:15 -07001241 if (pcid_enabled) {
Junaid Shahid208320b2018-06-27 14:59:21 -07001242 skip_tlb_flush = cr3 & X86_CR3_PCID_NOFLUSH;
1243 cr3 &= ~X86_CR3_PCID_NOFLUSH;
Sean Christopherson21823fb2021-06-09 16:42:24 -07001244 pcid = cr3 & X86_CR3_PCID_MASK;
Junaid Shahidade61e22018-06-27 14:59:15 -07001245 }
Paolo Bonziniac146232014-11-10 13:53:25 +01001246#endif
Nadav Amit9d88fca2014-11-02 11:54:52 +02001247
Sean Christophersonc7313152021-06-07 12:01:58 +03001248 /* PDPTRs are always reloaded for PAE paging. */
Sean Christopherson21823fb2021-06-09 16:42:24 -07001249 if (cr3 == kvm_read_cr3(vcpu) && !is_pae_paging(vcpu))
1250 goto handle_tlb_flush;
Avi Kivityd835dfe2007-11-21 02:57:59 +02001251
Sean Christopherson886bbcc2021-04-21 19:21:21 -07001252 /*
1253 * Do not condition the GPA check on long mode, this helper is used to
1254 * stuff CR3, e.g. for RSM emulation, and there is no guarantee that
1255 * the current vCPU mode is accurate.
1256 */
1257 if (kvm_vcpu_is_illegal_gpa(vcpu, cr3))
Yu Zhangd1cd3ce2017-08-24 20:27:53 +08001258 return 1;
Sean Christopherson886bbcc2021-04-21 19:21:21 -07001259
Lai Jiangshan2df4a5e2021-11-24 20:20:52 +08001260 if (is_pae_paging(vcpu) && !load_pdptrs(vcpu, cr3))
Nadav Amit346874c2014-04-18 03:35:09 +03001261 return 1;
Carsten Ottea03490e2007-10-29 16:09:35 +01001262
Sean Christopherson21823fb2021-06-09 16:42:24 -07001263 if (cr3 != kvm_read_cr3(vcpu))
Sean Christophersonb5129102021-06-09 16:42:27 -07001264 kvm_mmu_new_pgd(vcpu, cr3);
Sean Christopherson21823fb2021-06-09 16:42:24 -07001265
Gleb Natapov0f122442010-04-28 19:15:31 +03001266 vcpu->arch.cr3 = cr3;
Lai Jiangshan3883bc9d2021-11-08 20:44:02 +08001267 kvm_register_mark_dirty(vcpu, VCPU_EXREG_CR3);
Michael Roth405329f2021-12-16 11:13:54 -06001268 /* Do not call post_set_cr3, we do not get here for confidential guests. */
Junaid Shahid7c390d32018-06-27 14:59:06 -07001269
Sean Christopherson21823fb2021-06-09 16:42:24 -07001270handle_tlb_flush:
1271 /*
1272 * A load of CR3 that flushes the TLB flushes only the current PCID,
1273 * even if PCID is disabled, in which case PCID=0 is flushed. It's a
1274 * moot point in the end because _disabling_ PCID will flush all PCIDs,
1275 * and it's impossible to use a non-zero PCID when PCID is disabled,
1276 * i.e. only PCID=0 can be relevant.
1277 */
1278 if (!skip_tlb_flush)
1279 kvm_invalidate_pcid(vcpu, pcid);
1280
Gleb Natapov0f122442010-04-28 19:15:31 +03001281 return 0;
1282}
Avi Kivity2d3ad1f2008-02-24 11:20:43 +02001283EXPORT_SYMBOL_GPL(kvm_set_cr3);
Carsten Ottea03490e2007-10-29 16:09:35 +01001284
Andre Przywaraeea1cff2010-12-21 11:12:00 +01001285int kvm_set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8)
Carsten Ottea03490e2007-10-29 16:09:35 +01001286{
Gleb Natapov0f122442010-04-28 19:15:31 +03001287 if (cr8 & CR8_RESERVED_BITS)
1288 return 1;
Paolo Bonzini35754c92015-07-29 12:05:37 +02001289 if (lapic_in_kernel(vcpu))
Carsten Ottea03490e2007-10-29 16:09:35 +01001290 kvm_lapic_set_tpr(vcpu, cr8);
1291 else
Zhang Xiantaoad312c72007-12-13 23:50:52 +08001292 vcpu->arch.cr8 = cr8;
Gleb Natapov0f122442010-04-28 19:15:31 +03001293 return 0;
1294}
Avi Kivity2d3ad1f2008-02-24 11:20:43 +02001295EXPORT_SYMBOL_GPL(kvm_set_cr8);
Carsten Ottea03490e2007-10-29 16:09:35 +01001296
Avi Kivity2d3ad1f2008-02-24 11:20:43 +02001297unsigned long kvm_get_cr8(struct kvm_vcpu *vcpu)
Carsten Ottea03490e2007-10-29 16:09:35 +01001298{
Paolo Bonzini35754c92015-07-29 12:05:37 +02001299 if (lapic_in_kernel(vcpu))
Carsten Ottea03490e2007-10-29 16:09:35 +01001300 return kvm_lapic_get_cr8(vcpu);
1301 else
Zhang Xiantaoad312c72007-12-13 23:50:52 +08001302 return vcpu->arch.cr8;
Carsten Ottea03490e2007-10-29 16:09:35 +01001303}
Avi Kivity2d3ad1f2008-02-24 11:20:43 +02001304EXPORT_SYMBOL_GPL(kvm_get_cr8);
Carsten Ottea03490e2007-10-29 16:09:35 +01001305
Nadav Amitae561ed2015-04-02 03:10:37 +03001306static void kvm_update_dr0123(struct kvm_vcpu *vcpu)
1307{
1308 int i;
1309
1310 if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)) {
1311 for (i = 0; i < KVM_NR_DB_REGS; i++)
1312 vcpu->arch.eff_db[i] = vcpu->arch.db[i];
Nadav Amitae561ed2015-04-02 03:10:37 +03001313 }
1314}
1315
Paolo Bonzini7c866632020-05-16 08:42:28 -04001316void kvm_update_dr7(struct kvm_vcpu *vcpu)
Jan Kiszkac8639012012-09-21 05:42:55 +02001317{
1318 unsigned long dr7;
1319
1320 if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)
1321 dr7 = vcpu->arch.guest_debug_dr7;
1322 else
1323 dr7 = vcpu->arch.dr7;
Jason Baronb36464772021-01-14 22:27:56 -05001324 static_call(kvm_x86_set_dr7)(vcpu, dr7);
Paolo Bonzini360b9482014-02-21 09:55:56 +01001325 vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_BP_ENABLED;
1326 if (dr7 & DR7_BP_EN_MASK)
1327 vcpu->arch.switch_db_regs |= KVM_DEBUGREG_BP_ENABLED;
Jan Kiszkac8639012012-09-21 05:42:55 +02001328}
Paolo Bonzini7c866632020-05-16 08:42:28 -04001329EXPORT_SYMBOL_GPL(kvm_update_dr7);
Jan Kiszkac8639012012-09-21 05:42:55 +02001330
Nadav Amit6f43ed02014-07-15 17:37:46 +03001331static u64 kvm_dr6_fixed(struct kvm_vcpu *vcpu)
1332{
1333 u64 fixed = DR6_FIXED_1;
1334
Radim Krčmářd6321d42017-08-05 00:12:49 +02001335 if (!guest_cpuid_has(vcpu, X86_FEATURE_RTM))
Nadav Amit6f43ed02014-07-15 17:37:46 +03001336 fixed |= DR6_RTM;
Chenyi Qiange8ea85f2021-02-02 17:04:32 +08001337
1338 if (!guest_cpuid_has(vcpu, X86_FEATURE_BUS_LOCK_DETECT))
1339 fixed |= DR6_BUS_LOCK;
Nadav Amit6f43ed02014-07-15 17:37:46 +03001340 return fixed;
1341}
1342
Paolo Bonzini996ff542020-12-14 07:49:54 -05001343int kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val)
Gleb Natapov020df072010-04-13 10:05:23 +03001344{
Marios Pomonisea740052019-12-11 12:47:52 -08001345 size_t size = ARRAY_SIZE(vcpu->arch.db);
1346
Gleb Natapov020df072010-04-13 10:05:23 +03001347 switch (dr) {
1348 case 0 ... 3:
Marios Pomonisea740052019-12-11 12:47:52 -08001349 vcpu->arch.db[array_index_nospec(dr, size)] = val;
Gleb Natapov020df072010-04-13 10:05:23 +03001350 if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP))
1351 vcpu->arch.eff_db[dr] = val;
1352 break;
1353 case 4:
Gleb Natapov020df072010-04-13 10:05:23 +03001354 case 6:
Krish Sadhukhanf5f61452020-05-22 18:19:51 -04001355 if (!kvm_dr6_valid(val))
Paolo Bonzini996ff542020-12-14 07:49:54 -05001356 return 1; /* #GP */
Nadav Amit6f43ed02014-07-15 17:37:46 +03001357 vcpu->arch.dr6 = (val & DR6_VOLATILE) | kvm_dr6_fixed(vcpu);
Gleb Natapov020df072010-04-13 10:05:23 +03001358 break;
1359 case 5:
Gleb Natapov020df072010-04-13 10:05:23 +03001360 default: /* 7 */
Krish Sadhukhanb91991b2020-01-15 19:54:32 -05001361 if (!kvm_dr7_valid(val))
Paolo Bonzini996ff542020-12-14 07:49:54 -05001362 return 1; /* #GP */
Gleb Natapov020df072010-04-13 10:05:23 +03001363 vcpu->arch.dr7 = (val & DR7_VOLATILE) | DR7_FIXED_1;
Jan Kiszkac8639012012-09-21 05:42:55 +02001364 kvm_update_dr7(vcpu);
Gleb Natapov020df072010-04-13 10:05:23 +03001365 break;
1366 }
1367
1368 return 0;
1369}
1370EXPORT_SYMBOL_GPL(kvm_set_dr);
1371
Paolo Bonzini29d6ca42021-02-03 03:42:41 -05001372void kvm_get_dr(struct kvm_vcpu *vcpu, int dr, unsigned long *val)
Gleb Natapov020df072010-04-13 10:05:23 +03001373{
Marios Pomonisea740052019-12-11 12:47:52 -08001374 size_t size = ARRAY_SIZE(vcpu->arch.db);
1375
Gleb Natapov020df072010-04-13 10:05:23 +03001376 switch (dr) {
1377 case 0 ... 3:
Marios Pomonisea740052019-12-11 12:47:52 -08001378 *val = vcpu->arch.db[array_index_nospec(dr, size)];
Gleb Natapov020df072010-04-13 10:05:23 +03001379 break;
1380 case 4:
Gleb Natapov020df072010-04-13 10:05:23 +03001381 case 6:
Paolo Bonzini5679b802020-05-04 11:28:25 -04001382 *val = vcpu->arch.dr6;
Gleb Natapov020df072010-04-13 10:05:23 +03001383 break;
1384 case 5:
Gleb Natapov020df072010-04-13 10:05:23 +03001385 default: /* 7 */
1386 *val = vcpu->arch.dr7;
1387 break;
1388 }
Gleb Natapov338dbc92010-04-28 19:15:32 +03001389}
Gleb Natapov020df072010-04-13 10:05:23 +03001390EXPORT_SYMBOL_GPL(kvm_get_dr);
1391
Sean Christophersonc483c452021-02-04 16:57:48 -08001392int kvm_emulate_rdpmc(struct kvm_vcpu *vcpu)
Avi Kivity022cd0e2011-11-10 14:57:23 +02001393{
Sean Christophersonde3cd112019-04-30 10:36:17 -07001394 u32 ecx = kvm_rcx_read(vcpu);
Avi Kivity022cd0e2011-11-10 14:57:23 +02001395 u64 data;
Avi Kivity022cd0e2011-11-10 14:57:23 +02001396
Sean Christophersonc483c452021-02-04 16:57:48 -08001397 if (kvm_pmu_rdpmc(vcpu, ecx, &data)) {
1398 kvm_inject_gp(vcpu, 0);
1399 return 1;
1400 }
1401
Sean Christophersonde3cd112019-04-30 10:36:17 -07001402 kvm_rax_write(vcpu, (u32)data);
1403 kvm_rdx_write(vcpu, data >> 32);
Sean Christophersonc483c452021-02-04 16:57:48 -08001404 return kvm_skip_emulated_instruction(vcpu);
Avi Kivity022cd0e2011-11-10 14:57:23 +02001405}
Sean Christophersonc483c452021-02-04 16:57:48 -08001406EXPORT_SYMBOL_GPL(kvm_emulate_rdpmc);
Avi Kivity022cd0e2011-11-10 14:57:23 +02001407
Carsten Otte043405e2007-10-10 17:16:19 +02001408/*
1409 * List of msr numbers which we expose to userspace through KVM_GET_MSRS
1410 * and KVM_SET_MSRS, and KVM_GET_MSR_INDEX_LIST.
1411 *
Chenyi Qiang7a5ee6e2019-11-06 14:35:20 +08001412 * The three MSR lists(msrs_to_save, emulated_msrs, msr_based_features)
1413 * extract the supported MSRs from the related const lists.
1414 * msrs_to_save is selected from the msrs_to_save_all to reflect the
Glauber Costae3267cb2009-10-06 13:24:50 -04001415 * capabilities of the host cpu. This capabilities test skips MSRs that are
Chenyi Qiang7a5ee6e2019-11-06 14:35:20 +08001416 * kvm-specific. Those are put in emulated_msrs_all; filtering of emulated_msrs
Paolo Bonzini62ef68b2015-05-05 12:08:55 +02001417 * may depend on host virtualization features rather than host cpu features.
Carsten Otte043405e2007-10-10 17:16:19 +02001418 */
Glauber Costae3267cb2009-10-06 13:24:50 -04001419
Chenyi Qiang7a5ee6e2019-11-06 14:35:20 +08001420static const u32 msrs_to_save_all[] = {
Carsten Otte043405e2007-10-10 17:16:19 +02001421 MSR_IA32_SYSENTER_CS, MSR_IA32_SYSENTER_ESP, MSR_IA32_SYSENTER_EIP,
Brian Gerst8c065852010-07-17 09:03:26 -04001422 MSR_STAR,
Carsten Otte043405e2007-10-10 17:16:19 +02001423#ifdef CONFIG_X86_64
1424 MSR_CSTAR, MSR_KERNEL_GS_BASE, MSR_SYSCALL_MASK, MSR_LSTAR,
1425#endif
Nadav Har'Elb3897a42013-07-08 19:12:35 +08001426 MSR_IA32_TSC, MSR_IA32_CR_PAT, MSR_VM_HSAVE_PA,
Sean Christopherson32ad73d2019-12-20 20:44:55 -08001427 MSR_IA32_FEAT_CTL, MSR_IA32_BNDCFGS, MSR_TSC_AUX,
Xiaoyao Li2bdb76c2019-03-08 15:57:20 +08001428 MSR_IA32_SPEC_CTRL,
Chao Pengbf8c55d2018-10-24 16:05:14 +08001429 MSR_IA32_RTIT_CTL, MSR_IA32_RTIT_STATUS, MSR_IA32_RTIT_CR3_MATCH,
1430 MSR_IA32_RTIT_OUTPUT_BASE, MSR_IA32_RTIT_OUTPUT_MASK,
1431 MSR_IA32_RTIT_ADDR0_A, MSR_IA32_RTIT_ADDR0_B,
1432 MSR_IA32_RTIT_ADDR1_A, MSR_IA32_RTIT_ADDR1_B,
1433 MSR_IA32_RTIT_ADDR2_A, MSR_IA32_RTIT_ADDR2_B,
1434 MSR_IA32_RTIT_ADDR3_A, MSR_IA32_RTIT_ADDR3_B,
Tao Xu6e3ba4a2019-07-16 14:55:50 +08001435 MSR_IA32_UMWAIT_CONTROL,
1436
Jim Mattsone2ada662019-08-21 11:20:04 -07001437 MSR_ARCH_PERFMON_FIXED_CTR0, MSR_ARCH_PERFMON_FIXED_CTR1,
Wei Wang9fb12fe2021-12-17 07:49:34 -05001438 MSR_ARCH_PERFMON_FIXED_CTR0 + 2,
Jim Mattsone2ada662019-08-21 11:20:04 -07001439 MSR_CORE_PERF_FIXED_CTR_CTRL, MSR_CORE_PERF_GLOBAL_STATUS,
1440 MSR_CORE_PERF_GLOBAL_CTRL, MSR_CORE_PERF_GLOBAL_OVF_CTRL,
1441 MSR_ARCH_PERFMON_PERFCTR0, MSR_ARCH_PERFMON_PERFCTR1,
1442 MSR_ARCH_PERFMON_PERFCTR0 + 2, MSR_ARCH_PERFMON_PERFCTR0 + 3,
1443 MSR_ARCH_PERFMON_PERFCTR0 + 4, MSR_ARCH_PERFMON_PERFCTR0 + 5,
1444 MSR_ARCH_PERFMON_PERFCTR0 + 6, MSR_ARCH_PERFMON_PERFCTR0 + 7,
1445 MSR_ARCH_PERFMON_PERFCTR0 + 8, MSR_ARCH_PERFMON_PERFCTR0 + 9,
1446 MSR_ARCH_PERFMON_PERFCTR0 + 10, MSR_ARCH_PERFMON_PERFCTR0 + 11,
1447 MSR_ARCH_PERFMON_PERFCTR0 + 12, MSR_ARCH_PERFMON_PERFCTR0 + 13,
1448 MSR_ARCH_PERFMON_PERFCTR0 + 14, MSR_ARCH_PERFMON_PERFCTR0 + 15,
1449 MSR_ARCH_PERFMON_PERFCTR0 + 16, MSR_ARCH_PERFMON_PERFCTR0 + 17,
Jim Mattsone2ada662019-08-21 11:20:04 -07001450 MSR_ARCH_PERFMON_EVENTSEL0, MSR_ARCH_PERFMON_EVENTSEL1,
1451 MSR_ARCH_PERFMON_EVENTSEL0 + 2, MSR_ARCH_PERFMON_EVENTSEL0 + 3,
1452 MSR_ARCH_PERFMON_EVENTSEL0 + 4, MSR_ARCH_PERFMON_EVENTSEL0 + 5,
1453 MSR_ARCH_PERFMON_EVENTSEL0 + 6, MSR_ARCH_PERFMON_EVENTSEL0 + 7,
1454 MSR_ARCH_PERFMON_EVENTSEL0 + 8, MSR_ARCH_PERFMON_EVENTSEL0 + 9,
1455 MSR_ARCH_PERFMON_EVENTSEL0 + 10, MSR_ARCH_PERFMON_EVENTSEL0 + 11,
1456 MSR_ARCH_PERFMON_EVENTSEL0 + 12, MSR_ARCH_PERFMON_EVENTSEL0 + 13,
1457 MSR_ARCH_PERFMON_EVENTSEL0 + 14, MSR_ARCH_PERFMON_EVENTSEL0 + 15,
1458 MSR_ARCH_PERFMON_EVENTSEL0 + 16, MSR_ARCH_PERFMON_EVENTSEL0 + 17,
Like Xu902caeb2022-04-11 18:19:40 +08001459 MSR_IA32_PEBS_ENABLE, MSR_IA32_DS_AREA, MSR_PEBS_DATA_CFG,
Fares Mehannae1fc1552021-09-15 13:39:50 +00001460
1461 MSR_K7_EVNTSEL0, MSR_K7_EVNTSEL1, MSR_K7_EVNTSEL2, MSR_K7_EVNTSEL3,
1462 MSR_K7_PERFCTR0, MSR_K7_PERFCTR1, MSR_K7_PERFCTR2, MSR_K7_PERFCTR3,
1463 MSR_F15H_PERF_CTL0, MSR_F15H_PERF_CTL1, MSR_F15H_PERF_CTL2,
1464 MSR_F15H_PERF_CTL3, MSR_F15H_PERF_CTL4, MSR_F15H_PERF_CTL5,
1465 MSR_F15H_PERF_CTR0, MSR_F15H_PERF_CTR1, MSR_F15H_PERF_CTR2,
1466 MSR_F15H_PERF_CTR3, MSR_F15H_PERF_CTR4, MSR_F15H_PERF_CTR5,
Jing Liu548e8362022-01-05 04:35:24 -08001467 MSR_IA32_XFD, MSR_IA32_XFD_ERR,
Carsten Otte043405e2007-10-10 17:16:19 +02001468};
1469
Chenyi Qiang7a5ee6e2019-11-06 14:35:20 +08001470static u32 msrs_to_save[ARRAY_SIZE(msrs_to_save_all)];
Carsten Otte043405e2007-10-10 17:16:19 +02001471static unsigned num_msrs_to_save;
1472
Chenyi Qiang7a5ee6e2019-11-06 14:35:20 +08001473static const u32 emulated_msrs_all[] = {
Paolo Bonzini62ef68b2015-05-05 12:08:55 +02001474 MSR_KVM_SYSTEM_TIME, MSR_KVM_WALL_CLOCK,
1475 MSR_KVM_SYSTEM_TIME_NEW, MSR_KVM_WALL_CLOCK_NEW,
1476 HV_X64_MSR_GUEST_OS_ID, HV_X64_MSR_HYPERCALL,
1477 HV_X64_MSR_TIME_REF_COUNT, HV_X64_MSR_REFERENCE_TSC,
Ladi Prosek72c139b2017-07-26 13:32:59 +02001478 HV_X64_MSR_TSC_FREQUENCY, HV_X64_MSR_APIC_FREQUENCY,
Andrey Smetanine7d95132015-07-03 15:01:37 +03001479 HV_X64_MSR_CRASH_P0, HV_X64_MSR_CRASH_P1, HV_X64_MSR_CRASH_P2,
1480 HV_X64_MSR_CRASH_P3, HV_X64_MSR_CRASH_P4, HV_X64_MSR_CRASH_CTL,
Andrey Smetanine516ceb2015-09-16 12:29:48 +03001481 HV_X64_MSR_RESET,
Andrey Smetanin11c4b1c2015-09-16 12:29:49 +03001482 HV_X64_MSR_VP_INDEX,
Andrey Smetanin9eec50b2015-09-16 12:29:50 +03001483 HV_X64_MSR_VP_RUNTIME,
Andrey Smetanin5c9194122015-11-10 15:36:34 +03001484 HV_X64_MSR_SCONTROL,
Andrey Smetanin1f4b34f2015-11-30 19:22:21 +03001485 HV_X64_MSR_STIMER0_CONFIG,
Ladi Prosekd4abc572018-03-20 15:02:07 +01001486 HV_X64_MSR_VP_ASSIST_PAGE,
Vitaly Kuznetsova2e164e2018-03-01 15:15:12 +01001487 HV_X64_MSR_REENLIGHTENMENT_CONTROL, HV_X64_MSR_TSC_EMULATION_CONTROL,
1488 HV_X64_MSR_TSC_EMULATION_STATUS,
Jon Doronf97f5a52020-05-29 16:45:40 +03001489 HV_X64_MSR_SYNDBG_OPTIONS,
1490 HV_X64_MSR_SYNDBG_CONTROL, HV_X64_MSR_SYNDBG_STATUS,
1491 HV_X64_MSR_SYNDBG_SEND_BUFFER, HV_X64_MSR_SYNDBG_RECV_BUFFER,
1492 HV_X64_MSR_SYNDBG_PENDING_BUFFER,
Vitaly Kuznetsova2e164e2018-03-01 15:15:12 +01001493
1494 MSR_KVM_ASYNC_PF_EN, MSR_KVM_STEAL_TIME,
Vitaly Kuznetsov557a9612020-05-25 16:41:21 +02001495 MSR_KVM_PV_EOI_EN, MSR_KVM_ASYNC_PF_INT, MSR_KVM_ASYNC_PF_ACK,
Paolo Bonzini62ef68b2015-05-05 12:08:55 +02001496
Will Auldba904632012-11-29 12:42:50 -08001497 MSR_IA32_TSC_ADJUST,
Dave Hansen09141ec2020-03-05 09:47:06 -08001498 MSR_IA32_TSC_DEADLINE,
Xiaoyao Li2bdb76c2019-03-08 15:57:20 +08001499 MSR_IA32_ARCH_CAPABILITIES,
Like Xu27461da32020-05-29 15:43:45 +08001500 MSR_IA32_PERF_CAPABILITIES,
Carsten Otte043405e2007-10-10 17:16:19 +02001501 MSR_IA32_MISC_ENABLE,
Avi Kivity908e75f2010-07-07 14:09:38 +03001502 MSR_IA32_MCG_STATUS,
1503 MSR_IA32_MCG_CTL,
Ashok Rajc45dcc72016-06-22 14:59:56 +08001504 MSR_IA32_MCG_EXT_CTL,
Paolo Bonzini64d60672015-05-07 11:36:11 +02001505 MSR_IA32_SMBASE,
Liran Alon52797bf2017-11-15 13:43:14 +02001506 MSR_SMI_COUNT,
Kyle Hueydb2336a2017-03-20 01:16:28 -07001507 MSR_PLATFORM_INFO,
1508 MSR_MISC_FEATURES_ENABLES,
Tom Lendackybc226f02018-05-10 22:06:39 +02001509 MSR_AMD64_VIRT_SPEC_CTRL,
Maxim Levitsky5228eb92021-09-14 18:48:24 +03001510 MSR_AMD64_TSC_RATIO,
Liran Alon6c6a2ab2019-04-15 18:45:26 +03001511 MSR_IA32_POWER_CTL,
Paolo Bonzini99634e32020-01-20 14:22:55 +01001512 MSR_IA32_UCODE_REV,
Borislav Petkov191c8132019-04-18 18:32:50 +02001513
Paolo Bonzini95c5c7c2019-07-02 14:45:24 +02001514 /*
1515 * The following list leaves out MSRs whose values are determined
1516 * by arch/x86/kvm/vmx/nested.c based on CPUID or other MSRs.
1517 * We always support the "true" VMX control MSRs, even if the host
1518 * processor does not, so I am putting these registers here rather
Chenyi Qiang7a5ee6e2019-11-06 14:35:20 +08001519 * than in msrs_to_save_all.
Paolo Bonzini95c5c7c2019-07-02 14:45:24 +02001520 */
1521 MSR_IA32_VMX_BASIC,
1522 MSR_IA32_VMX_TRUE_PINBASED_CTLS,
1523 MSR_IA32_VMX_TRUE_PROCBASED_CTLS,
1524 MSR_IA32_VMX_TRUE_EXIT_CTLS,
1525 MSR_IA32_VMX_TRUE_ENTRY_CTLS,
1526 MSR_IA32_VMX_MISC,
1527 MSR_IA32_VMX_CR0_FIXED0,
1528 MSR_IA32_VMX_CR4_FIXED0,
1529 MSR_IA32_VMX_VMCS_ENUM,
1530 MSR_IA32_VMX_PROCBASED_CTLS2,
1531 MSR_IA32_VMX_EPT_VPID_CAP,
1532 MSR_IA32_VMX_VMFUNC,
1533
Borislav Petkov191c8132019-04-18 18:32:50 +02001534 MSR_K7_HWCR,
Marcelo Tosatti2d5ba192019-06-03 19:52:44 -03001535 MSR_KVM_POLL_CONTROL,
Carsten Otte043405e2007-10-10 17:16:19 +02001536};
1537
Chenyi Qiang7a5ee6e2019-11-06 14:35:20 +08001538static u32 emulated_msrs[ARRAY_SIZE(emulated_msrs_all)];
Paolo Bonzini62ef68b2015-05-05 12:08:55 +02001539static unsigned num_emulated_msrs;
1540
Tom Lendacky801e4592018-02-21 13:39:51 -06001541/*
1542 * List of msr numbers which are used to expose MSR-based features that
1543 * can be used by a hypervisor to validate requested CPU features.
1544 */
Chenyi Qiang7a5ee6e2019-11-06 14:35:20 +08001545static const u32 msr_based_features_all[] = {
Paolo Bonzini13893092018-02-26 13:40:09 +01001546 MSR_IA32_VMX_BASIC,
1547 MSR_IA32_VMX_TRUE_PINBASED_CTLS,
1548 MSR_IA32_VMX_PINBASED_CTLS,
1549 MSR_IA32_VMX_TRUE_PROCBASED_CTLS,
1550 MSR_IA32_VMX_PROCBASED_CTLS,
1551 MSR_IA32_VMX_TRUE_EXIT_CTLS,
1552 MSR_IA32_VMX_EXIT_CTLS,
1553 MSR_IA32_VMX_TRUE_ENTRY_CTLS,
1554 MSR_IA32_VMX_ENTRY_CTLS,
1555 MSR_IA32_VMX_MISC,
1556 MSR_IA32_VMX_CR0_FIXED0,
1557 MSR_IA32_VMX_CR0_FIXED1,
1558 MSR_IA32_VMX_CR4_FIXED0,
1559 MSR_IA32_VMX_CR4_FIXED1,
1560 MSR_IA32_VMX_VMCS_ENUM,
1561 MSR_IA32_VMX_PROCBASED_CTLS2,
1562 MSR_IA32_VMX_EPT_VPID_CAP,
1563 MSR_IA32_VMX_VMFUNC,
1564
Tom Lendackyd1d93fa2018-02-24 00:18:20 +01001565 MSR_F10H_DECFG,
Wanpeng Li518e7b92018-02-28 14:03:31 +08001566 MSR_IA32_UCODE_REV,
Paolo Bonzinicd283252018-06-25 14:04:37 +02001567 MSR_IA32_ARCH_CAPABILITIES,
Like Xu27461da32020-05-29 15:43:45 +08001568 MSR_IA32_PERF_CAPABILITIES,
Tom Lendacky801e4592018-02-21 13:39:51 -06001569};
1570
Chenyi Qiang7a5ee6e2019-11-06 14:35:20 +08001571static u32 msr_based_features[ARRAY_SIZE(msr_based_features_all)];
Tom Lendacky801e4592018-02-21 13:39:51 -06001572static unsigned int num_msr_based_features;
1573
Jim Mattson02047502022-08-30 10:49:47 -07001574/*
1575 * Some IA32_ARCH_CAPABILITIES bits have dependencies on MSRs that KVM
1576 * does not yet virtualize. These include:
1577 * 10 - MISC_PACKAGE_CTRLS
1578 * 11 - ENERGY_FILTERING_CTL
1579 * 12 - DOITM
1580 * 18 - FB_CLEAR_CTRL
1581 * 21 - XAPIC_DISABLE_STATUS
1582 * 23 - OVERCLOCKING_STATUS
1583 */
1584
1585#define KVM_SUPPORTED_ARCH_CAP \
1586 (ARCH_CAP_RDCL_NO | ARCH_CAP_IBRS_ALL | ARCH_CAP_RSBA | \
1587 ARCH_CAP_SKIP_VMENTRY_L1DFLUSH | ARCH_CAP_SSB_NO | ARCH_CAP_MDS_NO | \
1588 ARCH_CAP_PSCHANGE_MC_NO | ARCH_CAP_TSX_CTRL_MSR | ARCH_CAP_TAA_NO | \
1589 ARCH_CAP_SBDR_SSDP_NO | ARCH_CAP_FBSDP_NO | ARCH_CAP_PSDP_NO | \
1590 ARCH_CAP_FB_CLEAR | ARCH_CAP_RRSBA | ARCH_CAP_PBRSB_NO)
1591
Xiaoyao Li4d22c172019-04-19 10:16:24 +08001592static u64 kvm_get_arch_capabilities(void)
Paolo Bonzini5b76a3c2018-08-05 16:07:47 +02001593{
Xiaoyao Li4d22c172019-04-19 10:16:24 +08001594 u64 data = 0;
Paolo Bonzini5b76a3c2018-08-05 16:07:47 +02001595
Jim Mattson02047502022-08-30 10:49:47 -07001596 if (boot_cpu_has(X86_FEATURE_ARCH_CAPABILITIES)) {
Xiaoyao Li4d22c172019-04-19 10:16:24 +08001597 rdmsrl(MSR_IA32_ARCH_CAPABILITIES, data);
Jim Mattson02047502022-08-30 10:49:47 -07001598 data &= KVM_SUPPORTED_ARCH_CAP;
1599 }
Paolo Bonzini5b76a3c2018-08-05 16:07:47 +02001600
1601 /*
Paolo Bonzinib8e8c832019-11-04 12:22:02 +01001602 * If nx_huge_pages is enabled, KVM's shadow paging will ensure that
1603 * the nested hypervisor runs with NX huge pages. If it is not,
Ingo Molnard9f6e122021-03-18 15:28:01 +01001604 * L1 is anyway vulnerable to ITLB_MULTIHIT exploits from other
Paolo Bonzinib8e8c832019-11-04 12:22:02 +01001605 * L1 guests, so it need not worry about its own (L2) guests.
1606 */
1607 data |= ARCH_CAP_PSCHANGE_MC_NO;
1608
1609 /*
Paolo Bonzini5b76a3c2018-08-05 16:07:47 +02001610 * If we're doing cache flushes (either "always" or "cond")
1611 * we will do one whenever the guest does a vmlaunch/vmresume.
1612 * If an outer hypervisor is doing the cache flush for us
1613 * (VMENTER_L1D_FLUSH_NESTED_VM), we can safely pass that
1614 * capability to the guest too, and if EPT is disabled we're not
1615 * vulnerable. Overall, only VMENTER_L1D_FLUSH_NEVER will
1616 * require a nested hypervisor to do a flush of its own.
1617 */
1618 if (l1tf_vmx_mitigation != VMENTER_L1D_FLUSH_NEVER)
1619 data |= ARCH_CAP_SKIP_VMENTRY_L1DFLUSH;
1620
Paolo Bonzini0c549142019-08-19 17:24:07 +02001621 if (!boot_cpu_has_bug(X86_BUG_CPU_MELTDOWN))
1622 data |= ARCH_CAP_RDCL_NO;
1623 if (!boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS))
1624 data |= ARCH_CAP_SSB_NO;
1625 if (!boot_cpu_has_bug(X86_BUG_MDS))
1626 data |= ARCH_CAP_MDS_NO;
1627
Paolo Bonzini71316362021-01-28 11:45:00 -05001628 if (!boot_cpu_has(X86_FEATURE_RTM)) {
1629 /*
1630 * If RTM=0 because the kernel has disabled TSX, the host might
1631 * have TAA_NO or TSX_CTRL. Clear TAA_NO (the guest sees RTM=0
1632 * and therefore knows that there cannot be TAA) but keep
1633 * TSX_CTRL: some buggy userspaces leave it set on tsx=on hosts,
1634 * and we want to allow migrating those guests to tsx=off hosts.
1635 */
1636 data &= ~ARCH_CAP_TAA_NO;
1637 } else if (!boot_cpu_has_bug(X86_BUG_TAA)) {
Paolo Bonzinicbbaa272019-11-18 18:58:26 +01001638 data |= ARCH_CAP_TAA_NO;
Paolo Bonzini71316362021-01-28 11:45:00 -05001639 } else {
1640 /*
1641 * Nothing to do here; we emulate TSX_CTRL if present on the
1642 * host so the guest can choose between disabling TSX or
1643 * using VERW to clear CPU buffers.
1644 */
1645 }
Pawan Guptae1d38b62019-10-23 12:23:33 +02001646
Paolo Bonzini5b76a3c2018-08-05 16:07:47 +02001647 return data;
1648}
Paolo Bonzini5b76a3c2018-08-05 16:07:47 +02001649
Wanpeng Li66421c12018-02-28 14:03:30 +08001650static int kvm_get_msr_feature(struct kvm_msr_entry *msr)
1651{
1652 switch (msr->index) {
Paolo Bonzinicd283252018-06-25 14:04:37 +02001653 case MSR_IA32_ARCH_CAPABILITIES:
Paolo Bonzini5b76a3c2018-08-05 16:07:47 +02001654 msr->data = kvm_get_arch_capabilities();
1655 break;
1656 case MSR_IA32_UCODE_REV:
Paolo Bonzinicd283252018-06-25 14:04:37 +02001657 rdmsrl_safe(msr->index, &msr->data);
Wanpeng Li518e7b92018-02-28 14:03:31 +08001658 break;
Wanpeng Li66421c12018-02-28 14:03:30 +08001659 default:
Jason Baronb36464772021-01-14 22:27:56 -05001660 return static_call(kvm_x86_get_msr_feature)(msr);
Wanpeng Li66421c12018-02-28 14:03:30 +08001661 }
1662 return 0;
1663}
1664
Tom Lendacky801e4592018-02-21 13:39:51 -06001665static int do_get_msr_feature(struct kvm_vcpu *vcpu, unsigned index, u64 *data)
1666{
1667 struct kvm_msr_entry msr;
Wanpeng Li66421c12018-02-28 14:03:30 +08001668 int r;
Tom Lendacky801e4592018-02-21 13:39:51 -06001669
1670 msr.index = index;
Wanpeng Li66421c12018-02-28 14:03:30 +08001671 r = kvm_get_msr_feature(&msr);
Peter Xu12bc2132020-06-22 18:04:42 -04001672
1673 if (r == KVM_MSR_RET_INVALID) {
1674 /* Unconditionally clear the output for simplicity */
1675 *data = 0;
Haiwei Lid6328262021-03-13 13:10:32 +08001676 if (kvm_msr_ignored_check(index, 0, false))
Maxim Levitskycc4cb012020-11-01 13:55:23 +02001677 r = 0;
Peter Xu12bc2132020-06-22 18:04:42 -04001678 }
1679
Wanpeng Li66421c12018-02-28 14:03:30 +08001680 if (r)
1681 return r;
Tom Lendacky801e4592018-02-21 13:39:51 -06001682
1683 *data = msr.data;
1684
1685 return 0;
1686}
1687
Sean Christopherson11988492019-04-02 08:19:15 -07001688static bool __kvm_valid_efer(struct kvm_vcpu *vcpu, u64 efer)
1689{
1690 if (efer & EFER_FFXSR && !guest_cpuid_has(vcpu, X86_FEATURE_FXSR_OPT))
1691 return false;
1692
1693 if (efer & EFER_SVME && !guest_cpuid_has(vcpu, X86_FEATURE_SVM))
1694 return false;
1695
Sean Christopherson0a629562019-04-02 08:19:16 -07001696 if (efer & (EFER_LME | EFER_LMA) &&
1697 !guest_cpuid_has(vcpu, X86_FEATURE_LM))
1698 return false;
1699
1700 if (efer & EFER_NX && !guest_cpuid_has(vcpu, X86_FEATURE_NX))
1701 return false;
1702
Sean Christopherson11988492019-04-02 08:19:15 -07001703 return true;
1704
1705}
Jan Kiszka384bb782013-04-20 10:52:36 +02001706bool kvm_valid_efer(struct kvm_vcpu *vcpu, u64 efer)
Carsten Otte15c4a642007-10-30 18:44:17 +01001707{
Roedel, Joergb69e8ca2010-05-06 11:38:43 +02001708 if (efer & efer_reserved_bits)
Jan Kiszka384bb782013-04-20 10:52:36 +02001709 return false;
Carsten Otte15c4a642007-10-30 18:44:17 +01001710
Sean Christopherson11988492019-04-02 08:19:15 -07001711 return __kvm_valid_efer(vcpu, efer);
Jan Kiszka384bb782013-04-20 10:52:36 +02001712}
1713EXPORT_SYMBOL_GPL(kvm_valid_efer);
1714
Sean Christopherson11988492019-04-02 08:19:15 -07001715static int set_efer(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
Jan Kiszka384bb782013-04-20 10:52:36 +02001716{
1717 u64 old_efer = vcpu->arch.efer;
Sean Christopherson11988492019-04-02 08:19:15 -07001718 u64 efer = msr_info->data;
Maxim Levitsky72f211ec2020-10-01 14:29:53 +03001719 int r;
Jan Kiszka384bb782013-04-20 10:52:36 +02001720
Sean Christopherson11988492019-04-02 08:19:15 -07001721 if (efer & efer_reserved_bits)
Paolo Bonzini66f61c92019-05-24 21:52:46 +02001722 return 1;
Jan Kiszka384bb782013-04-20 10:52:36 +02001723
Sean Christopherson11988492019-04-02 08:19:15 -07001724 if (!msr_info->host_initiated) {
1725 if (!__kvm_valid_efer(vcpu, efer))
1726 return 1;
1727
1728 if (is_paging(vcpu) &&
1729 (vcpu->arch.efer & EFER_LME) != (efer & EFER_LME))
1730 return 1;
1731 }
Jan Kiszka384bb782013-04-20 10:52:36 +02001732
Carsten Otte15c4a642007-10-30 18:44:17 +01001733 efer &= ~EFER_LMA;
Avi Kivityf6801df2010-01-21 15:31:50 +02001734 efer |= vcpu->arch.efer & EFER_LMA;
Carsten Otte15c4a642007-10-30 18:44:17 +01001735
Jason Baronb36464772021-01-14 22:27:56 -05001736 r = static_call(kvm_x86_set_efer)(vcpu, efer);
Maxim Levitsky72f211ec2020-10-01 14:29:53 +03001737 if (r) {
1738 WARN_ON(r > 0);
1739 return r;
1740 }
Sheng Yanga3d204e2010-05-12 16:40:40 +08001741
Paolo Bonzinid6174292022-02-09 04:56:05 -05001742 if ((efer ^ old_efer) & KVM_MMU_EFER_ROLE_BITS)
Sheng Yangaad82702010-05-12 16:40:42 +08001743 kvm_mmu_reset_context(vcpu);
1744
Roedel, Joergb69e8ca2010-05-06 11:38:43 +02001745 return 0;
Carsten Otte15c4a642007-10-30 18:44:17 +01001746}
1747
Joerg Roedelf2b4b7d2008-01-31 14:57:37 +01001748void kvm_enable_efer_bits(u64 mask)
1749{
1750 efer_reserved_bits &= ~mask;
1751}
1752EXPORT_SYMBOL_GPL(kvm_enable_efer_bits);
1753
Alexander Graf51de8152020-09-25 16:34:17 +02001754bool kvm_msr_allowed(struct kvm_vcpu *vcpu, u32 index, u32 type)
1755{
Sean Christophersonb318e8d2021-03-16 11:44:33 -07001756 struct kvm_x86_msr_filter *msr_filter;
1757 struct msr_bitmap_range *ranges;
Alexander Graf1a1552542020-09-25 16:34:21 +02001758 struct kvm *kvm = vcpu->kvm;
Sean Christophersonb318e8d2021-03-16 11:44:33 -07001759 bool allowed;
Alexander Graf1a1552542020-09-25 16:34:21 +02001760 int idx;
Sean Christophersonb318e8d2021-03-16 11:44:33 -07001761 u32 i;
Alexander Graf1a1552542020-09-25 16:34:21 +02001762
Sean Christophersonb318e8d2021-03-16 11:44:33 -07001763 /* x2APIC MSRs do not support filtering. */
1764 if (index >= 0x800 && index <= 0x8ff)
Alexander Graf1a1552542020-09-25 16:34:21 +02001765 return true;
1766
Alexander Graf1a1552542020-09-25 16:34:21 +02001767 idx = srcu_read_lock(&kvm->srcu);
1768
Sean Christophersonb318e8d2021-03-16 11:44:33 -07001769 msr_filter = srcu_dereference(kvm->arch.msr_filter, &kvm->srcu);
1770 if (!msr_filter) {
1771 allowed = true;
1772 goto out;
1773 }
1774
1775 allowed = msr_filter->default_allow;
1776 ranges = msr_filter->ranges;
1777
1778 for (i = 0; i < msr_filter->count; i++) {
Alexander Graf1a1552542020-09-25 16:34:21 +02001779 u32 start = ranges[i].base;
1780 u32 end = start + ranges[i].nmsrs;
1781 u32 flags = ranges[i].flags;
1782 unsigned long *bitmap = ranges[i].bitmap;
1783
1784 if ((index >= start) && (index < end) && (flags & type)) {
Sean Christophersonb318e8d2021-03-16 11:44:33 -07001785 allowed = !!test_bit(index - start, bitmap);
Alexander Graf1a1552542020-09-25 16:34:21 +02001786 break;
1787 }
1788 }
1789
Sean Christophersonb318e8d2021-03-16 11:44:33 -07001790out:
Alexander Graf1a1552542020-09-25 16:34:21 +02001791 srcu_read_unlock(&kvm->srcu, idx);
1792
Sean Christophersonb318e8d2021-03-16 11:44:33 -07001793 return allowed;
Alexander Graf51de8152020-09-25 16:34:17 +02001794}
1795EXPORT_SYMBOL_GPL(kvm_msr_allowed);
1796
Carsten Otte15c4a642007-10-30 18:44:17 +01001797/*
Sean Christophersonf20935d2019-09-05 14:22:54 -07001798 * Write @data into the MSR specified by @index. Select MSR specific fault
1799 * checks are bypassed if @host_initiated is %true.
Carsten Otte15c4a642007-10-30 18:44:17 +01001800 * Returns 0 on success, non-0 otherwise.
1801 * Assumes vcpu_load() was already called.
1802 */
Sean Christophersonf20935d2019-09-05 14:22:54 -07001803static int __kvm_set_msr(struct kvm_vcpu *vcpu, u32 index, u64 data,
1804 bool host_initiated)
Carsten Otte15c4a642007-10-30 18:44:17 +01001805{
Sean Christophersonf20935d2019-09-05 14:22:54 -07001806 struct msr_data msr;
1807
1808 switch (index) {
Nadav Amit854e8bb2014-09-16 03:24:05 +03001809 case MSR_FS_BASE:
1810 case MSR_GS_BASE:
1811 case MSR_KERNEL_GS_BASE:
1812 case MSR_CSTAR:
1813 case MSR_LSTAR:
Sean Christophersonf20935d2019-09-05 14:22:54 -07001814 if (is_noncanonical_address(data, vcpu))
Nadav Amit854e8bb2014-09-16 03:24:05 +03001815 return 1;
1816 break;
1817 case MSR_IA32_SYSENTER_EIP:
1818 case MSR_IA32_SYSENTER_ESP:
1819 /*
1820 * IA32_SYSENTER_ESP and IA32_SYSENTER_EIP cause #GP if
1821 * non-canonical address is written on Intel but not on
1822 * AMD (which ignores the top 32-bits, because it does
1823 * not implement 64-bit SYSENTER).
1824 *
1825 * 64-bit code should hence be able to write a non-canonical
1826 * value on AMD. Making the address canonical ensures that
1827 * vmentry does not fail on Intel after writing a non-canonical
1828 * value, and that something deterministic happens if the guest
1829 * invokes 64-bit SYSENTER.
1830 */
Adrian Hunter1fb85d02022-01-31 09:24:50 +02001831 data = __canonical_address(data, vcpu_virt_addr_bits(vcpu));
Sean Christopherson61a05d42021-05-04 10:17:33 -07001832 break;
1833 case MSR_TSC_AUX:
1834 if (!kvm_is_supported_user_return_msr(MSR_TSC_AUX))
1835 return 1;
1836
1837 if (!host_initiated &&
1838 !guest_cpuid_has(vcpu, X86_FEATURE_RDTSCP) &&
1839 !guest_cpuid_has(vcpu, X86_FEATURE_RDPID))
1840 return 1;
1841
1842 /*
1843 * Per Intel's SDM, bits 63:32 are reserved, but AMD's APM has
1844 * incomplete and conflicting architectural behavior. Current
1845 * AMD CPUs completely ignore bits 63:32, i.e. they aren't
1846 * reserved and always read as zeros. Enforce Intel's reserved
1847 * bits check if and only if the guest CPU is Intel, and clear
1848 * the bits in all other cases. This ensures cross-vendor
1849 * migration will provide consistent behavior for the guest.
1850 */
1851 if (guest_cpuid_is_intel(vcpu) && (data >> 32) != 0)
1852 return 1;
1853
1854 data = (u32)data;
1855 break;
Nadav Amit854e8bb2014-09-16 03:24:05 +03001856 }
Sean Christophersonf20935d2019-09-05 14:22:54 -07001857
1858 msr.data = data;
1859 msr.index = index;
1860 msr.host_initiated = host_initiated;
1861
Jason Baronb36464772021-01-14 22:27:56 -05001862 return static_call(kvm_x86_set_msr)(vcpu, &msr);
Sean Christophersonf20935d2019-09-05 14:22:54 -07001863}
1864
Peter Xu6abe9c12020-06-22 18:04:41 -04001865static int kvm_set_msr_ignored_check(struct kvm_vcpu *vcpu,
1866 u32 index, u64 data, bool host_initiated)
1867{
1868 int ret = __kvm_set_msr(vcpu, index, data, host_initiated);
1869
1870 if (ret == KVM_MSR_RET_INVALID)
Haiwei Lid6328262021-03-13 13:10:32 +08001871 if (kvm_msr_ignored_check(index, data, true))
Maxim Levitskycc4cb012020-11-01 13:55:23 +02001872 ret = 0;
Peter Xu6abe9c12020-06-22 18:04:41 -04001873
1874 return ret;
1875}
1876
Sean Christophersonf20935d2019-09-05 14:22:54 -07001877/*
1878 * Read the MSR specified by @index into @data. Select MSR specific fault
1879 * checks are bypassed if @host_initiated is %true.
1880 * Returns 0 on success, non-0 otherwise.
1881 * Assumes vcpu_load() was already called.
1882 */
Paolo Bonziniedef5c32019-11-18 12:23:00 -05001883int __kvm_get_msr(struct kvm_vcpu *vcpu, u32 index, u64 *data,
1884 bool host_initiated)
Sean Christophersonf20935d2019-09-05 14:22:54 -07001885{
1886 struct msr_data msr;
1887 int ret;
1888
Sean Christopherson61a05d42021-05-04 10:17:33 -07001889 switch (index) {
1890 case MSR_TSC_AUX:
1891 if (!kvm_is_supported_user_return_msr(MSR_TSC_AUX))
1892 return 1;
1893
1894 if (!host_initiated &&
1895 !guest_cpuid_has(vcpu, X86_FEATURE_RDTSCP) &&
1896 !guest_cpuid_has(vcpu, X86_FEATURE_RDPID))
1897 return 1;
1898 break;
1899 }
1900
Sean Christophersonf20935d2019-09-05 14:22:54 -07001901 msr.index = index;
1902 msr.host_initiated = host_initiated;
1903
Jason Baronb36464772021-01-14 22:27:56 -05001904 ret = static_call(kvm_x86_get_msr)(vcpu, &msr);
Sean Christophersonf20935d2019-09-05 14:22:54 -07001905 if (!ret)
1906 *data = msr.data;
1907 return ret;
1908}
1909
Peter Xu6abe9c12020-06-22 18:04:41 -04001910static int kvm_get_msr_ignored_check(struct kvm_vcpu *vcpu,
1911 u32 index, u64 *data, bool host_initiated)
1912{
1913 int ret = __kvm_get_msr(vcpu, index, data, host_initiated);
1914
1915 if (ret == KVM_MSR_RET_INVALID) {
1916 /* Unconditionally clear *data for simplicity */
1917 *data = 0;
Haiwei Lid6328262021-03-13 13:10:32 +08001918 if (kvm_msr_ignored_check(index, 0, false))
Maxim Levitskycc4cb012020-11-01 13:55:23 +02001919 ret = 0;
Peter Xu6abe9c12020-06-22 18:04:41 -04001920 }
1921
1922 return ret;
1923}
1924
Hou Wenlongac8d6ca2022-03-07 20:26:33 +08001925static int kvm_get_msr_with_filter(struct kvm_vcpu *vcpu, u32 index, u64 *data)
1926{
1927 if (!kvm_msr_allowed(vcpu, index, KVM_MSR_FILTER_READ))
1928 return KVM_MSR_RET_FILTERED;
1929 return kvm_get_msr_ignored_check(vcpu, index, data, false);
1930}
1931
1932static int kvm_set_msr_with_filter(struct kvm_vcpu *vcpu, u32 index, u64 data)
1933{
1934 if (!kvm_msr_allowed(vcpu, index, KVM_MSR_FILTER_WRITE))
1935 return KVM_MSR_RET_FILTERED;
1936 return kvm_set_msr_ignored_check(vcpu, index, data, false);
1937}
1938
Sean Christophersonf20935d2019-09-05 14:22:54 -07001939int kvm_get_msr(struct kvm_vcpu *vcpu, u32 index, u64 *data)
1940{
Peter Xu6abe9c12020-06-22 18:04:41 -04001941 return kvm_get_msr_ignored_check(vcpu, index, data, false);
Sean Christophersonf20935d2019-09-05 14:22:54 -07001942}
1943EXPORT_SYMBOL_GPL(kvm_get_msr);
1944
1945int kvm_set_msr(struct kvm_vcpu *vcpu, u32 index, u64 data)
1946{
Peter Xu6abe9c12020-06-22 18:04:41 -04001947 return kvm_set_msr_ignored_check(vcpu, index, data, false);
Carsten Otte15c4a642007-10-30 18:44:17 +01001948}
Nadav Amit854e8bb2014-09-16 03:24:05 +03001949EXPORT_SYMBOL_GPL(kvm_set_msr);
Carsten Otte15c4a642007-10-30 18:44:17 +01001950
Hou Wenlongd2f7d492021-11-02 17:15:31 +08001951static void complete_userspace_rdmsr(struct kvm_vcpu *vcpu)
Alexander Graf1ae09952020-09-25 16:34:16 +02001952{
Hou Wenlongd2f7d492021-11-02 17:15:31 +08001953 if (!vcpu->run->msr.error) {
Alexander Graf1ae09952020-09-25 16:34:16 +02001954 kvm_rax_write(vcpu, (u32)vcpu->run->msr.data);
1955 kvm_rdx_write(vcpu, vcpu->run->msr.data >> 32);
1956 }
Alexander Graf1ae09952020-09-25 16:34:16 +02001957}
1958
Hou Wenlongd2f7d492021-11-02 17:15:31 +08001959static int complete_emulated_msr_access(struct kvm_vcpu *vcpu)
1960{
1961 return complete_emulated_insn_gp(vcpu, vcpu->run->msr.error);
1962}
1963
1964static int complete_emulated_rdmsr(struct kvm_vcpu *vcpu)
1965{
1966 complete_userspace_rdmsr(vcpu);
1967 return complete_emulated_msr_access(vcpu);
1968}
1969
1970static int complete_fast_msr_access(struct kvm_vcpu *vcpu)
Alexander Graf1ae09952020-09-25 16:34:16 +02001971{
Jason Baronb36464772021-01-14 22:27:56 -05001972 return static_call(kvm_x86_complete_emulated_msr)(vcpu, vcpu->run->msr.error);
Alexander Graf1ae09952020-09-25 16:34:16 +02001973}
1974
Hou Wenlongd2f7d492021-11-02 17:15:31 +08001975static int complete_fast_rdmsr(struct kvm_vcpu *vcpu)
1976{
1977 complete_userspace_rdmsr(vcpu);
1978 return complete_fast_msr_access(vcpu);
1979}
1980
Alexander Graf1ae09952020-09-25 16:34:16 +02001981static u64 kvm_msr_reason(int r)
1982{
1983 switch (r) {
Maxim Levitskycc4cb012020-11-01 13:55:23 +02001984 case KVM_MSR_RET_INVALID:
Alexander Graf1ae09952020-09-25 16:34:16 +02001985 return KVM_MSR_EXIT_REASON_UNKNOWN;
Maxim Levitskycc4cb012020-11-01 13:55:23 +02001986 case KVM_MSR_RET_FILTERED:
Alexander Graf1a1552542020-09-25 16:34:21 +02001987 return KVM_MSR_EXIT_REASON_FILTER;
Alexander Graf1ae09952020-09-25 16:34:16 +02001988 default:
1989 return KVM_MSR_EXIT_REASON_INVAL;
1990 }
1991}
1992
1993static int kvm_msr_user_space(struct kvm_vcpu *vcpu, u32 index,
1994 u32 exit_reason, u64 data,
1995 int (*completion)(struct kvm_vcpu *vcpu),
1996 int r)
1997{
1998 u64 msr_reason = kvm_msr_reason(r);
1999
2000 /* Check if the user wanted to know about this MSR fault */
2001 if (!(vcpu->kvm->arch.user_space_msr_mask & msr_reason))
2002 return 0;
2003
2004 vcpu->run->exit_reason = exit_reason;
2005 vcpu->run->msr.error = 0;
2006 memset(vcpu->run->msr.pad, 0, sizeof(vcpu->run->msr.pad));
2007 vcpu->run->msr.reason = msr_reason;
2008 vcpu->run->msr.index = index;
2009 vcpu->run->msr.data = data;
2010 vcpu->arch.complete_userspace_io = completion;
2011
2012 return 1;
2013}
2014
Sean Christopherson1edce0a2019-09-05 14:22:55 -07002015int kvm_emulate_rdmsr(struct kvm_vcpu *vcpu)
2016{
2017 u32 ecx = kvm_rcx_read(vcpu);
2018 u64 data;
Alexander Graf1ae09952020-09-25 16:34:16 +02002019 int r;
Sean Christopherson1edce0a2019-09-05 14:22:55 -07002020
Hou Wenlongac8d6ca2022-03-07 20:26:33 +08002021 r = kvm_get_msr_with_filter(vcpu, ecx, &data);
Alexander Graf1ae09952020-09-25 16:34:16 +02002022
Paolo Bonzini8b474422020-12-14 07:44:46 -05002023 if (!r) {
2024 trace_kvm_msr_read(ecx, data);
2025
2026 kvm_rax_write(vcpu, data & -1u);
2027 kvm_rdx_write(vcpu, (data >> 32) & -1u);
2028 } else {
Hou Wenlongd2f7d492021-11-02 17:15:31 +08002029 /* MSR read failed? See if we should ask user space */
2030 if (kvm_msr_user_space(vcpu, ecx, KVM_EXIT_X86_RDMSR, 0,
2031 complete_fast_rdmsr, r))
2032 return 0;
Sean Christopherson1edce0a2019-09-05 14:22:55 -07002033 trace_kvm_msr_read_ex(ecx);
Sean Christopherson1edce0a2019-09-05 14:22:55 -07002034 }
2035
Jason Baronb36464772021-01-14 22:27:56 -05002036 return static_call(kvm_x86_complete_emulated_msr)(vcpu, r);
Sean Christopherson1edce0a2019-09-05 14:22:55 -07002037}
2038EXPORT_SYMBOL_GPL(kvm_emulate_rdmsr);
2039
2040int kvm_emulate_wrmsr(struct kvm_vcpu *vcpu)
2041{
2042 u32 ecx = kvm_rcx_read(vcpu);
2043 u64 data = kvm_read_edx_eax(vcpu);
Alexander Graf1ae09952020-09-25 16:34:16 +02002044 int r;
Sean Christopherson1edce0a2019-09-05 14:22:55 -07002045
Hou Wenlongac8d6ca2022-03-07 20:26:33 +08002046 r = kvm_set_msr_with_filter(vcpu, ecx, data);
Alexander Graf1ae09952020-09-25 16:34:16 +02002047
Hou Wenlongd2f7d492021-11-02 17:15:31 +08002048 if (!r) {
Paolo Bonzini8b474422020-12-14 07:44:46 -05002049 trace_kvm_msr_write(ecx, data);
Hou Wenlongd2f7d492021-11-02 17:15:31 +08002050 } else {
2051 /* MSR write failed? See if we should ask user space */
2052 if (kvm_msr_user_space(vcpu, ecx, KVM_EXIT_X86_WRMSR, data,
2053 complete_fast_msr_access, r))
2054 return 0;
2055 /* Signal all other negative errors to userspace */
2056 if (r < 0)
2057 return r;
Sean Christopherson1edce0a2019-09-05 14:22:55 -07002058 trace_kvm_msr_write_ex(ecx, data);
Hou Wenlongd2f7d492021-11-02 17:15:31 +08002059 }
Sean Christopherson1edce0a2019-09-05 14:22:55 -07002060
Jason Baronb36464772021-01-14 22:27:56 -05002061 return static_call(kvm_x86_complete_emulated_msr)(vcpu, r);
Sean Christopherson1edce0a2019-09-05 14:22:55 -07002062}
2063EXPORT_SYMBOL_GPL(kvm_emulate_wrmsr);
2064
Sean Christopherson5ff3a352021-02-04 16:57:47 -08002065int kvm_emulate_as_nop(struct kvm_vcpu *vcpu)
2066{
2067 return kvm_skip_emulated_instruction(vcpu);
2068}
2069EXPORT_SYMBOL_GPL(kvm_emulate_as_nop);
2070
2071int kvm_emulate_invd(struct kvm_vcpu *vcpu)
2072{
2073 /* Treat an INVD instruction as a NOP and just skip it. */
2074 return kvm_emulate_as_nop(vcpu);
2075}
2076EXPORT_SYMBOL_GPL(kvm_emulate_invd);
2077
Sean Christopherson5ff3a352021-02-04 16:57:47 -08002078int kvm_handle_invalid_op(struct kvm_vcpu *vcpu)
2079{
2080 kvm_queue_exception(vcpu, UD_VECTOR);
2081 return 1;
2082}
2083EXPORT_SYMBOL_GPL(kvm_handle_invalid_op);
2084
Sean Christophersonbfbcc812022-06-08 22:45:12 +00002085
2086static int kvm_emulate_monitor_mwait(struct kvm_vcpu *vcpu, const char *insn)
2087{
Sean Christopherson43bb9e02022-07-11 22:57:53 +00002088 if (!kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_MWAIT_NEVER_UD_FAULTS) &&
Sean Christophersonbfbcc812022-06-08 22:45:12 +00002089 !guest_cpuid_has(vcpu, X86_FEATURE_MWAIT))
2090 return kvm_handle_invalid_op(vcpu);
2091
2092 pr_warn_once("kvm: %s instruction emulated as NOP!\n", insn);
2093 return kvm_emulate_as_nop(vcpu);
2094}
2095int kvm_emulate_mwait(struct kvm_vcpu *vcpu)
2096{
2097 return kvm_emulate_monitor_mwait(vcpu, "MWAIT");
2098}
2099EXPORT_SYMBOL_GPL(kvm_emulate_mwait);
2100
Sean Christopherson5ff3a352021-02-04 16:57:47 -08002101int kvm_emulate_monitor(struct kvm_vcpu *vcpu)
2102{
Sean Christophersonbfbcc812022-06-08 22:45:12 +00002103 return kvm_emulate_monitor_mwait(vcpu, "MONITOR");
Sean Christopherson5ff3a352021-02-04 16:57:47 -08002104}
2105EXPORT_SYMBOL_GPL(kvm_emulate_monitor);
2106
Paolo Bonzinid89d04a2021-02-02 10:44:23 -05002107static inline bool kvm_vcpu_exit_request(struct kvm_vcpu *vcpu)
Wanpeng Li5a9f5442020-04-28 14:23:26 +08002108{
Frederic Weisbecker4ae7dc92021-02-01 00:05:48 +01002109 xfer_to_guest_mode_prepare();
Wanpeng Li5a9f5442020-04-28 14:23:26 +08002110 return vcpu->mode == EXITING_GUEST_MODE || kvm_request_pending(vcpu) ||
Thomas Gleixner72c3c0f2020-07-23 00:00:09 +02002111 xfer_to_guest_mode_work_pending();
Wanpeng Li5a9f5442020-04-28 14:23:26 +08002112}
Wanpeng Li5a9f5442020-04-28 14:23:26 +08002113
Carsten Otte313a3dc2007-10-11 19:16:52 +02002114/*
Wanpeng Li1e9e2622019-11-21 11:17:11 +08002115 * The fast path for frequent and performance sensitive wrmsr emulation,
2116 * i.e. the sending of IPI, sending IPI early in the VM-Exit flow reduces
2117 * the latency of virtual IPI by avoiding the expensive bits of transitioning
2118 * from guest to host, e.g. reacquiring KVM's SRCU lock. In contrast to the
2119 * other cases which must be called after interrupts are enabled on the host.
2120 */
2121static int handle_fastpath_set_x2apic_icr_irqoff(struct kvm_vcpu *vcpu, u64 data)
2122{
Wanpeng Lie1be9ac2020-03-26 10:20:01 +08002123 if (!lapic_in_kernel(vcpu) || !apic_x2apic_mode(vcpu->arch.apic))
2124 return 1;
2125
2126 if (((data & APIC_SHORT_MASK) == APIC_DEST_NOSHORT) &&
Sean Christophersonbd17f412022-02-04 21:42:00 +00002127 ((data & APIC_DEST_MASK) == APIC_DEST_PHYSICAL) &&
2128 ((data & APIC_MODE_MASK) == APIC_DM_FIXED) &&
Sean Christophersonb9964ee2022-02-04 21:42:04 +00002129 ((u32)(data >> 32) != X2APIC_BROADCAST))
2130 return kvm_x2apic_icr_write(vcpu->arch.apic, data);
Wanpeng Li1e9e2622019-11-21 11:17:11 +08002131
2132 return 1;
2133}
2134
Wanpeng Liae95f562020-04-28 14:23:28 +08002135static int handle_fastpath_set_tscdeadline(struct kvm_vcpu *vcpu, u64 data)
2136{
2137 if (!kvm_can_use_hv_timer(vcpu))
2138 return 1;
2139
2140 kvm_set_lapic_tscdeadline_msr(vcpu, data);
2141 return 0;
2142}
2143
Wanpeng Li404d5d72020-04-28 14:23:25 +08002144fastpath_t handle_fastpath_set_msr_irqoff(struct kvm_vcpu *vcpu)
Wanpeng Li1e9e2622019-11-21 11:17:11 +08002145{
2146 u32 msr = kvm_rcx_read(vcpu);
Wanpeng Li8a1038d2020-03-26 10:20:00 +08002147 u64 data;
Wanpeng Li404d5d72020-04-28 14:23:25 +08002148 fastpath_t ret = EXIT_FASTPATH_NONE;
Wanpeng Li1e9e2622019-11-21 11:17:11 +08002149
2150 switch (msr) {
2151 case APIC_BASE_MSR + (APIC_ICR >> 4):
Wanpeng Li8a1038d2020-03-26 10:20:00 +08002152 data = kvm_read_edx_eax(vcpu);
Wanpeng Li404d5d72020-04-28 14:23:25 +08002153 if (!handle_fastpath_set_x2apic_icr_irqoff(vcpu, data)) {
2154 kvm_skip_emulated_instruction(vcpu);
2155 ret = EXIT_FASTPATH_EXIT_HANDLED;
Haiwei Li80bc97f2020-05-18 09:31:38 +08002156 }
Wanpeng Li1e9e2622019-11-21 11:17:11 +08002157 break;
Dave Hansen09141ec2020-03-05 09:47:06 -08002158 case MSR_IA32_TSC_DEADLINE:
Wanpeng Liae95f562020-04-28 14:23:28 +08002159 data = kvm_read_edx_eax(vcpu);
2160 if (!handle_fastpath_set_tscdeadline(vcpu, data)) {
2161 kvm_skip_emulated_instruction(vcpu);
2162 ret = EXIT_FASTPATH_REENTER_GUEST;
2163 }
2164 break;
Wanpeng Li1e9e2622019-11-21 11:17:11 +08002165 default:
Wanpeng Li404d5d72020-04-28 14:23:25 +08002166 break;
Wanpeng Li1e9e2622019-11-21 11:17:11 +08002167 }
2168
Wanpeng Li404d5d72020-04-28 14:23:25 +08002169 if (ret != EXIT_FASTPATH_NONE)
Wanpeng Li1e9e2622019-11-21 11:17:11 +08002170 trace_kvm_msr_write(msr, data);
Wanpeng Li1e9e2622019-11-21 11:17:11 +08002171
Wanpeng Li404d5d72020-04-28 14:23:25 +08002172 return ret;
Wanpeng Li1e9e2622019-11-21 11:17:11 +08002173}
2174EXPORT_SYMBOL_GPL(handle_fastpath_set_msr_irqoff);
2175
2176/*
Carsten Otte313a3dc2007-10-11 19:16:52 +02002177 * Adapt set_msr() to msr_io()'s calling convention
2178 */
Paolo Bonzini609e36d2015-04-08 15:30:38 +02002179static int do_get_msr(struct kvm_vcpu *vcpu, unsigned index, u64 *data)
2180{
Peter Xu6abe9c12020-06-22 18:04:41 -04002181 return kvm_get_msr_ignored_check(vcpu, index, data, true);
Paolo Bonzini609e36d2015-04-08 15:30:38 +02002182}
2183
Carsten Otte313a3dc2007-10-11 19:16:52 +02002184static int do_set_msr(struct kvm_vcpu *vcpu, unsigned index, u64 *data)
2185{
Peter Xu6abe9c12020-06-22 18:04:41 -04002186 return kvm_set_msr_ignored_check(vcpu, index, *data, true);
Carsten Otte313a3dc2007-10-11 19:16:52 +02002187}
2188
Marcelo Tosatti16e8d742012-11-27 23:29:00 -02002189#ifdef CONFIG_X86_64
Marcelo Tosatti53fafdb2019-10-28 12:36:22 -02002190struct pvclock_clock {
2191 int vclock_mode;
2192 u64 cycle_last;
2193 u64 mask;
2194 u32 mult;
2195 u32 shift;
Paolo Bonzini917f9472020-01-22 14:32:20 +01002196 u64 base_cycles;
2197 u64 offset;
Marcelo Tosatti53fafdb2019-10-28 12:36:22 -02002198};
2199
Marcelo Tosatti16e8d742012-11-27 23:29:00 -02002200struct pvclock_gtod_data {
2201 seqcount_t seq;
2202
Marcelo Tosatti53fafdb2019-10-28 12:36:22 -02002203 struct pvclock_clock clock; /* extract of a clocksource struct */
2204 struct pvclock_clock raw_clock; /* extract of a clocksource struct */
Marcelo Tosatti16e8d742012-11-27 23:29:00 -02002205
Paolo Bonzini917f9472020-01-22 14:32:20 +01002206 ktime_t offs_boot;
Marcelo Tosatti55dd00a2017-01-24 15:09:39 -02002207 u64 wall_time_sec;
Marcelo Tosatti16e8d742012-11-27 23:29:00 -02002208};
2209
2210static struct pvclock_gtod_data pvclock_gtod_data;
2211
2212static void update_pvclock_gtod(struct timekeeper *tk)
2213{
2214 struct pvclock_gtod_data *vdata = &pvclock_gtod_data;
2215
2216 write_seqcount_begin(&vdata->seq);
2217
2218 /* copy pvclock gtod data */
Thomas Gleixnerb95a8a22020-02-07 13:38:56 +01002219 vdata->clock.vclock_mode = tk->tkr_mono.clock->vdso_clock_mode;
Peter Zijlstra876e7882015-03-19 10:09:06 +01002220 vdata->clock.cycle_last = tk->tkr_mono.cycle_last;
2221 vdata->clock.mask = tk->tkr_mono.mask;
2222 vdata->clock.mult = tk->tkr_mono.mult;
2223 vdata->clock.shift = tk->tkr_mono.shift;
Paolo Bonzini917f9472020-01-22 14:32:20 +01002224 vdata->clock.base_cycles = tk->tkr_mono.xtime_nsec;
2225 vdata->clock.offset = tk->tkr_mono.base;
Marcelo Tosatti16e8d742012-11-27 23:29:00 -02002226
Thomas Gleixnerb95a8a22020-02-07 13:38:56 +01002227 vdata->raw_clock.vclock_mode = tk->tkr_raw.clock->vdso_clock_mode;
Marcelo Tosatti53fafdb2019-10-28 12:36:22 -02002228 vdata->raw_clock.cycle_last = tk->tkr_raw.cycle_last;
2229 vdata->raw_clock.mask = tk->tkr_raw.mask;
2230 vdata->raw_clock.mult = tk->tkr_raw.mult;
2231 vdata->raw_clock.shift = tk->tkr_raw.shift;
Paolo Bonzini917f9472020-01-22 14:32:20 +01002232 vdata->raw_clock.base_cycles = tk->tkr_raw.xtime_nsec;
2233 vdata->raw_clock.offset = tk->tkr_raw.base;
Marcelo Tosatti16e8d742012-11-27 23:29:00 -02002234
Marcelo Tosatti55dd00a2017-01-24 15:09:39 -02002235 vdata->wall_time_sec = tk->xtime_sec;
2236
Paolo Bonzini917f9472020-01-22 14:32:20 +01002237 vdata->offs_boot = tk->offs_boot;
Marcelo Tosatti53fafdb2019-10-28 12:36:22 -02002238
Marcelo Tosatti16e8d742012-11-27 23:29:00 -02002239 write_seqcount_end(&vdata->seq);
2240}
Paolo Bonzini8171cd62020-01-22 14:36:09 +01002241
2242static s64 get_kvmclock_base_ns(void)
2243{
2244 /* Count up from boot time, but with the frequency of the raw clock. */
2245 return ktime_to_ns(ktime_add(ktime_get_raw(), pvclock_gtod_data.offs_boot));
2246}
2247#else
2248static s64 get_kvmclock_base_ns(void)
2249{
2250 /* Master clock not used, so we can just use CLOCK_BOOTTIME. */
2251 return ktime_get_boottime_ns();
2252}
Marcelo Tosatti16e8d742012-11-27 23:29:00 -02002253#endif
2254
David Woodhouse55749762021-12-10 16:36:24 +00002255static void kvm_write_wall_clock(struct kvm *kvm, gpa_t wall_clock, int sec_hi_ofs)
Glauber de Oliveira Costa18068522008-02-15 17:52:47 -02002256{
Avi Kivity9ed3c442010-05-04 15:00:37 +03002257 int version;
2258 int r;
Gerd Hoffmann50d0a0f2008-06-03 16:17:31 +02002259 struct pvclock_wall_clock wc;
Joao Martins629b5342018-06-28 15:06:43 -04002260 u32 wc_sec_hi;
Paolo Bonzini8171cd62020-01-22 14:36:09 +01002261 u64 wall_nsec;
Glauber de Oliveira Costa18068522008-02-15 17:52:47 -02002262
2263 if (!wall_clock)
2264 return;
2265
Avi Kivity9ed3c442010-05-04 15:00:37 +03002266 r = kvm_read_guest(kvm, wall_clock, &version, sizeof(version));
2267 if (r)
2268 return;
2269
2270 if (version & 1)
2271 ++version; /* first time write, random junk */
2272
2273 ++version;
Glauber de Oliveira Costa18068522008-02-15 17:52:47 -02002274
Nicholas Krause1dab1342015-12-30 13:08:46 -05002275 if (kvm_write_guest(kvm, wall_clock, &version, sizeof(version)))
2276 return;
Glauber de Oliveira Costa18068522008-02-15 17:52:47 -02002277
Gerd Hoffmann50d0a0f2008-06-03 16:17:31 +02002278 /*
2279 * The guest calculates current wall clock time by adding
Zachary Amsden34c238a2010-09-18 14:38:14 -10002280 * system time (updated by kvm_guest_time_update below) to the
Paolo Bonzini8171cd62020-01-22 14:36:09 +01002281 * wall clock specified here. We do the reverse here.
Gerd Hoffmann50d0a0f2008-06-03 16:17:31 +02002282 */
Paolo Bonzini8171cd62020-01-22 14:36:09 +01002283 wall_nsec = ktime_get_real_ns() - get_kvmclock_ns(kvm);
Gerd Hoffmann50d0a0f2008-06-03 16:17:31 +02002284
Paolo Bonzini8171cd62020-01-22 14:36:09 +01002285 wc.nsec = do_div(wall_nsec, 1000000000);
2286 wc.sec = (u32)wall_nsec; /* overflow in 2106 guest time */
Gerd Hoffmann50d0a0f2008-06-03 16:17:31 +02002287 wc.version = version;
Glauber de Oliveira Costa18068522008-02-15 17:52:47 -02002288
2289 kvm_write_guest(kvm, wall_clock, &wc, sizeof(wc));
2290
Joao Martins629b5342018-06-28 15:06:43 -04002291 if (sec_hi_ofs) {
2292 wc_sec_hi = wall_nsec >> 32;
2293 kvm_write_guest(kvm, wall_clock + sec_hi_ofs,
2294 &wc_sec_hi, sizeof(wc_sec_hi));
2295 }
2296
Glauber de Oliveira Costa18068522008-02-15 17:52:47 -02002297 version++;
2298 kvm_write_guest(kvm, wall_clock, &version, sizeof(version));
Glauber de Oliveira Costa18068522008-02-15 17:52:47 -02002299}
2300
Oliver Upton5b9bb0e2020-08-18 15:24:26 +00002301static void kvm_write_system_time(struct kvm_vcpu *vcpu, gpa_t system_time,
2302 bool old_msr, bool host_initiated)
2303{
2304 struct kvm_arch *ka = &vcpu->kvm->arch;
2305
2306 if (vcpu->vcpu_id == 0 && !host_initiated) {
Oliver Upton1e293d12020-10-27 16:10:43 -07002307 if (ka->boot_vcpu_runs_old_kvmclock != old_msr)
Oliver Upton5b9bb0e2020-08-18 15:24:26 +00002308 kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu);
2309
2310 ka->boot_vcpu_runs_old_kvmclock = old_msr;
2311 }
2312
2313 vcpu->arch.time = system_time;
2314 kvm_make_request(KVM_REQ_GLOBAL_CLOCK_UPDATE, vcpu);
2315
2316 /* we verify if the enable bit is set... */
David Woodhouse916d3602022-03-03 15:41:14 +00002317 if (system_time & 1) {
Michal Luczaj52491a382022-10-13 21:12:19 +00002318 kvm_gpc_activate(vcpu->kvm, &vcpu->arch.pv_time, vcpu,
2319 KVM_HOST_USES_PFN, system_time & ~1ULL,
2320 sizeof(struct pvclock_vcpu_time_info));
David Woodhouse916d3602022-03-03 15:41:14 +00002321 } else {
Michal Luczaj52491a382022-10-13 21:12:19 +00002322 kvm_gpc_deactivate(vcpu->kvm, &vcpu->arch.pv_time);
David Woodhouse916d3602022-03-03 15:41:14 +00002323 }
Oliver Upton5b9bb0e2020-08-18 15:24:26 +00002324
2325 return;
2326}
2327
Gerd Hoffmann50d0a0f2008-06-03 16:17:31 +02002328static uint32_t div_frac(uint32_t dividend, uint32_t divisor)
2329{
Paolo Bonzinib51012d2016-01-22 11:39:22 +01002330 do_shl32_div32(dividend, divisor);
2331 return dividend;
Gerd Hoffmann50d0a0f2008-06-03 16:17:31 +02002332}
2333
Paolo Bonzini3ae13fa2016-02-08 15:11:15 +01002334static void kvm_get_time_scale(uint64_t scaled_hz, uint64_t base_hz,
Zachary Amsden5f4e3f82010-09-18 14:38:13 -10002335 s8 *pshift, u32 *pmultiplier)
Gerd Hoffmann50d0a0f2008-06-03 16:17:31 +02002336{
Zachary Amsden5f4e3f82010-09-18 14:38:13 -10002337 uint64_t scaled64;
Gerd Hoffmann50d0a0f2008-06-03 16:17:31 +02002338 int32_t shift = 0;
2339 uint64_t tps64;
2340 uint32_t tps32;
2341
Paolo Bonzini3ae13fa2016-02-08 15:11:15 +01002342 tps64 = base_hz;
2343 scaled64 = scaled_hz;
Jan Kiszka50933622010-09-26 13:00:53 +02002344 while (tps64 > scaled64*2 || tps64 & 0xffffffff00000000ULL) {
Gerd Hoffmann50d0a0f2008-06-03 16:17:31 +02002345 tps64 >>= 1;
2346 shift--;
2347 }
2348
2349 tps32 = (uint32_t)tps64;
Jan Kiszka50933622010-09-26 13:00:53 +02002350 while (tps32 <= scaled64 || scaled64 & 0xffffffff00000000ULL) {
2351 if (scaled64 & 0xffffffff00000000ULL || tps32 & 0x80000000)
Zachary Amsden5f4e3f82010-09-18 14:38:13 -10002352 scaled64 >>= 1;
2353 else
2354 tps32 <<= 1;
Gerd Hoffmann50d0a0f2008-06-03 16:17:31 +02002355 shift++;
2356 }
2357
Zachary Amsden5f4e3f82010-09-18 14:38:13 -10002358 *pshift = shift;
2359 *pmultiplier = div_frac(scaled64, tps32);
Gerd Hoffmann50d0a0f2008-06-03 16:17:31 +02002360}
2361
Marcelo Tosattid8281992012-11-27 23:29:01 -02002362#ifdef CONFIG_X86_64
Marcelo Tosatti16e8d742012-11-27 23:29:00 -02002363static atomic_t kvm_guest_has_master_clock = ATOMIC_INIT(0);
Marcelo Tosattid8281992012-11-27 23:29:01 -02002364#endif
Marcelo Tosatti16e8d742012-11-27 23:29:00 -02002365
Gerd Hoffmannc8076602009-02-04 17:52:04 +01002366static DEFINE_PER_CPU(unsigned long, cpu_tsc_khz);
Fengguang Wu69b00492015-01-19 22:33:39 +08002367static unsigned long max_tsc_khz;
Gerd Hoffmannc8076602009-02-04 17:52:04 +01002368
Zachary Amsdencc578282012-02-03 15:43:50 -02002369static u32 adjust_tsc_khz(u32 khz, s32 ppm)
Zachary Amsdenc2855452010-09-18 14:38:15 -10002370{
Zachary Amsdencc578282012-02-03 15:43:50 -02002371 u64 v = (u64)khz * (1000000 + ppm);
2372 do_div(v, 1000000);
2373 return v;
2374}
2375
Ilias Stamatis1ab92872021-06-07 11:54:38 +01002376static void kvm_vcpu_write_tsc_multiplier(struct kvm_vcpu *vcpu, u64 l1_multiplier);
2377
Haozhong Zhang381d5852015-10-20 15:39:04 +08002378static int set_tsc_khz(struct kvm_vcpu *vcpu, u32 user_tsc_khz, bool scale)
2379{
2380 u64 ratio;
2381
2382 /* Guest TSC same frequency as host TSC? */
2383 if (!scale) {
Sean Christopherson938c8742022-05-24 21:56:23 +08002384 kvm_vcpu_write_tsc_multiplier(vcpu, kvm_caps.default_tsc_scaling_ratio);
Haozhong Zhang381d5852015-10-20 15:39:04 +08002385 return 0;
2386 }
2387
2388 /* TSC scaling supported? */
Sean Christopherson938c8742022-05-24 21:56:23 +08002389 if (!kvm_caps.has_tsc_control) {
Haozhong Zhang381d5852015-10-20 15:39:04 +08002390 if (user_tsc_khz > tsc_khz) {
2391 vcpu->arch.tsc_catchup = 1;
2392 vcpu->arch.tsc_always_catchup = 1;
2393 return 0;
2394 } else {
Paolo Bonzini3f16a5c2019-06-26 14:16:13 +02002395 pr_warn_ratelimited("user requested TSC rate below hardware speed\n");
Haozhong Zhang381d5852015-10-20 15:39:04 +08002396 return -1;
2397 }
2398 }
2399
2400 /* TSC scaling required - calculate ratio */
Sean Christopherson938c8742022-05-24 21:56:23 +08002401 ratio = mul_u64_u32_div(1ULL << kvm_caps.tsc_scaling_ratio_frac_bits,
Haozhong Zhang381d5852015-10-20 15:39:04 +08002402 user_tsc_khz, tsc_khz);
2403
Sean Christopherson938c8742022-05-24 21:56:23 +08002404 if (ratio == 0 || ratio >= kvm_caps.max_tsc_scaling_ratio) {
Paolo Bonzini3f16a5c2019-06-26 14:16:13 +02002405 pr_warn_ratelimited("Invalid TSC scaling ratio - virtual-tsc-khz=%u\n",
2406 user_tsc_khz);
Haozhong Zhang381d5852015-10-20 15:39:04 +08002407 return -1;
2408 }
2409
Ilias Stamatis1ab92872021-06-07 11:54:38 +01002410 kvm_vcpu_write_tsc_multiplier(vcpu, ratio);
Haozhong Zhang381d5852015-10-20 15:39:04 +08002411 return 0;
2412}
2413
Paolo Bonzini4941b8c2016-02-08 14:51:12 +01002414static int kvm_set_tsc_khz(struct kvm_vcpu *vcpu, u32 user_tsc_khz)
Zachary Amsdencc578282012-02-03 15:43:50 -02002415{
2416 u32 thresh_lo, thresh_hi;
2417 int use_scaling = 0;
2418
Marcelo Tosatti03ba32c2013-03-11 23:10:24 -03002419 /* tsc_khz can be zero if TSC calibration fails */
Paolo Bonzini4941b8c2016-02-08 14:51:12 +01002420 if (user_tsc_khz == 0) {
Haozhong Zhangad7218832015-10-20 15:39:02 +08002421 /* set tsc_scaling_ratio to a safe value */
Sean Christopherson938c8742022-05-24 21:56:23 +08002422 kvm_vcpu_write_tsc_multiplier(vcpu, kvm_caps.default_tsc_scaling_ratio);
Haozhong Zhang381d5852015-10-20 15:39:04 +08002423 return -1;
Haozhong Zhangad7218832015-10-20 15:39:02 +08002424 }
Marcelo Tosatti03ba32c2013-03-11 23:10:24 -03002425
Zachary Amsdenc2855452010-09-18 14:38:15 -10002426 /* Compute a scale to convert nanoseconds in TSC cycles */
Paolo Bonzini3ae13fa2016-02-08 15:11:15 +01002427 kvm_get_time_scale(user_tsc_khz * 1000LL, NSEC_PER_SEC,
Zachary Amsdencc578282012-02-03 15:43:50 -02002428 &vcpu->arch.virtual_tsc_shift,
2429 &vcpu->arch.virtual_tsc_mult);
Paolo Bonzini4941b8c2016-02-08 14:51:12 +01002430 vcpu->arch.virtual_tsc_khz = user_tsc_khz;
Zachary Amsdencc578282012-02-03 15:43:50 -02002431
2432 /*
2433 * Compute the variation in TSC rate which is acceptable
2434 * within the range of tolerance and decide if the
2435 * rate being applied is within that bounds of the hardware
2436 * rate. If so, no scaling or compensation need be done.
2437 */
2438 thresh_lo = adjust_tsc_khz(tsc_khz, -tsc_tolerance_ppm);
2439 thresh_hi = adjust_tsc_khz(tsc_khz, tsc_tolerance_ppm);
Paolo Bonzini4941b8c2016-02-08 14:51:12 +01002440 if (user_tsc_khz < thresh_lo || user_tsc_khz > thresh_hi) {
2441 pr_debug("kvm: requested TSC rate %u falls outside tolerance [%u,%u]\n", user_tsc_khz, thresh_lo, thresh_hi);
Zachary Amsdencc578282012-02-03 15:43:50 -02002442 use_scaling = 1;
2443 }
Paolo Bonzini4941b8c2016-02-08 14:51:12 +01002444 return set_tsc_khz(vcpu, user_tsc_khz, use_scaling);
Zachary Amsdenc2855452010-09-18 14:38:15 -10002445}
2446
2447static u64 compute_guest_tsc(struct kvm_vcpu *vcpu, s64 kernel_ns)
2448{
Zachary Amsdene26101b2012-02-03 15:43:57 -02002449 u64 tsc = pvclock_scale_delta(kernel_ns-vcpu->arch.this_tsc_nsec,
Zachary Amsdencc578282012-02-03 15:43:50 -02002450 vcpu->arch.virtual_tsc_mult,
2451 vcpu->arch.virtual_tsc_shift);
Zachary Amsdene26101b2012-02-03 15:43:57 -02002452 tsc += vcpu->arch.this_tsc_write;
Zachary Amsdenc2855452010-09-18 14:38:15 -10002453 return tsc;
2454}
2455
Leonardo Brasba1f77c2022-02-18 00:41:00 -03002456#ifdef CONFIG_X86_64
Vitaly Kuznetsovb0c39dc2018-01-24 14:23:36 +01002457static inline int gtod_is_based_on_tsc(int mode)
2458{
Thomas Gleixnerb95a8a22020-02-07 13:38:56 +01002459 return mode == VDSO_CLOCKMODE_TSC || mode == VDSO_CLOCKMODE_HVCLOCK;
Vitaly Kuznetsovb0c39dc2018-01-24 14:23:36 +01002460}
Leonardo Brasba1f77c2022-02-18 00:41:00 -03002461#endif
Vitaly Kuznetsovb0c39dc2018-01-24 14:23:36 +01002462
Fengguang Wu69b00492015-01-19 22:33:39 +08002463static void kvm_track_tsc_matching(struct kvm_vcpu *vcpu)
Marcelo Tosattib48aa972012-11-27 23:29:03 -02002464{
2465#ifdef CONFIG_X86_64
2466 bool vcpus_matched;
Marcelo Tosattib48aa972012-11-27 23:29:03 -02002467 struct kvm_arch *ka = &vcpu->kvm->arch;
2468 struct pvclock_gtod_data *gtod = &pvclock_gtod_data;
2469
2470 vcpus_matched = (ka->nr_vcpus_matched_tsc + 1 ==
2471 atomic_read(&vcpu->kvm->online_vcpus));
2472
Marcelo Tosatti7f187922014-11-04 21:30:44 -02002473 /*
2474 * Once the masterclock is enabled, always perform request in
2475 * order to update it.
2476 *
2477 * In order to enable masterclock, the host clocksource must be TSC
2478 * and the vcpus need to have matched TSCs. When that happens,
2479 * perform request to enable masterclock.
2480 */
2481 if (ka->use_master_clock ||
Vitaly Kuznetsovb0c39dc2018-01-24 14:23:36 +01002482 (gtod_is_based_on_tsc(gtod->clock.vclock_mode) && vcpus_matched))
Marcelo Tosattib48aa972012-11-27 23:29:03 -02002483 kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu);
2484
2485 trace_kvm_track_tsc(vcpu->vcpu_id, ka->nr_vcpus_matched_tsc,
2486 atomic_read(&vcpu->kvm->online_vcpus),
2487 ka->use_master_clock, gtod->clock.vclock_mode);
2488#endif
2489}
2490
Haozhong Zhang35181e82015-10-20 15:39:03 +08002491/*
2492 * Multiply tsc by a fixed point number represented by ratio.
2493 *
2494 * The most significant 64-N bits (mult) of ratio represent the
2495 * integral part of the fixed point number; the remaining N bits
2496 * (frac) represent the fractional part, ie. ratio represents a fixed
2497 * point number (mult + frac * 2^(-N)).
2498 *
Sean Christopherson938c8742022-05-24 21:56:23 +08002499 * N equals to kvm_caps.tsc_scaling_ratio_frac_bits.
Haozhong Zhang35181e82015-10-20 15:39:03 +08002500 */
2501static inline u64 __scale_tsc(u64 ratio, u64 tsc)
2502{
Sean Christopherson938c8742022-05-24 21:56:23 +08002503 return mul_u64_u64_shr(tsc, ratio, kvm_caps.tsc_scaling_ratio_frac_bits);
Haozhong Zhang35181e82015-10-20 15:39:03 +08002504}
2505
Jinrong Liang62711e52022-01-25 17:59:07 +08002506u64 kvm_scale_tsc(u64 tsc, u64 ratio)
Haozhong Zhang35181e82015-10-20 15:39:03 +08002507{
2508 u64 _tsc = tsc;
Haozhong Zhang35181e82015-10-20 15:39:03 +08002509
Sean Christopherson938c8742022-05-24 21:56:23 +08002510 if (ratio != kvm_caps.default_tsc_scaling_ratio)
Haozhong Zhang35181e82015-10-20 15:39:03 +08002511 _tsc = __scale_tsc(ratio, tsc);
2512
2513 return _tsc;
2514}
2515EXPORT_SYMBOL_GPL(kvm_scale_tsc);
2516
Ilias Stamatis9b399df2021-05-26 19:44:10 +01002517static u64 kvm_compute_l1_tsc_offset(struct kvm_vcpu *vcpu, u64 target_tsc)
Haozhong Zhang07c14192015-10-20 15:39:05 +08002518{
2519 u64 tsc;
2520
Jinrong Liang62711e52022-01-25 17:59:07 +08002521 tsc = kvm_scale_tsc(rdtsc(), vcpu->arch.l1_tsc_scaling_ratio);
Haozhong Zhang07c14192015-10-20 15:39:05 +08002522
2523 return target_tsc - tsc;
2524}
2525
Haozhong Zhang4ba76532015-10-20 15:39:07 +08002526u64 kvm_read_l1_tsc(struct kvm_vcpu *vcpu, u64 host_tsc)
2527{
Ilias Stamatisfe3eb502021-05-26 19:44:11 +01002528 return vcpu->arch.l1_tsc_offset +
Jinrong Liang62711e52022-01-25 17:59:07 +08002529 kvm_scale_tsc(host_tsc, vcpu->arch.l1_tsc_scaling_ratio);
Haozhong Zhang4ba76532015-10-20 15:39:07 +08002530}
2531EXPORT_SYMBOL_GPL(kvm_read_l1_tsc);
2532
Ilias Stamatis83150f22021-05-26 19:44:14 +01002533u64 kvm_calc_nested_tsc_offset(u64 l1_offset, u64 l2_offset, u64 l2_multiplier)
2534{
2535 u64 nested_offset;
2536
Sean Christopherson938c8742022-05-24 21:56:23 +08002537 if (l2_multiplier == kvm_caps.default_tsc_scaling_ratio)
Ilias Stamatis83150f22021-05-26 19:44:14 +01002538 nested_offset = l1_offset;
2539 else
2540 nested_offset = mul_s64_u64_shr((s64) l1_offset, l2_multiplier,
Sean Christopherson938c8742022-05-24 21:56:23 +08002541 kvm_caps.tsc_scaling_ratio_frac_bits);
Ilias Stamatis83150f22021-05-26 19:44:14 +01002542
2543 nested_offset += l2_offset;
2544 return nested_offset;
2545}
2546EXPORT_SYMBOL_GPL(kvm_calc_nested_tsc_offset);
2547
2548u64 kvm_calc_nested_tsc_multiplier(u64 l1_multiplier, u64 l2_multiplier)
2549{
Sean Christopherson938c8742022-05-24 21:56:23 +08002550 if (l2_multiplier != kvm_caps.default_tsc_scaling_ratio)
Ilias Stamatis83150f22021-05-26 19:44:14 +01002551 return mul_u64_u64_shr(l1_multiplier, l2_multiplier,
Sean Christopherson938c8742022-05-24 21:56:23 +08002552 kvm_caps.tsc_scaling_ratio_frac_bits);
Ilias Stamatis83150f22021-05-26 19:44:14 +01002553
2554 return l1_multiplier;
2555}
2556EXPORT_SYMBOL_GPL(kvm_calc_nested_tsc_multiplier);
2557
Ilias Stamatisedcfe542021-05-26 19:44:15 +01002558static void kvm_vcpu_write_tsc_offset(struct kvm_vcpu *vcpu, u64 l1_offset)
Luiz Capitulinoa545ab62016-09-07 14:47:19 -04002559{
Ilias Stamatisedcfe542021-05-26 19:44:15 +01002560 trace_kvm_write_tsc_offset(vcpu->vcpu_id,
2561 vcpu->arch.l1_tsc_offset,
2562 l1_offset);
2563
2564 vcpu->arch.l1_tsc_offset = l1_offset;
2565
2566 /*
2567 * If we are here because L1 chose not to trap WRMSR to TSC then
2568 * according to the spec this should set L1's TSC (as opposed to
2569 * setting L1's offset for L2).
2570 */
2571 if (is_guest_mode(vcpu))
2572 vcpu->arch.tsc_offset = kvm_calc_nested_tsc_offset(
2573 l1_offset,
2574 static_call(kvm_x86_get_l2_tsc_offset)(vcpu),
2575 static_call(kvm_x86_get_l2_tsc_multiplier)(vcpu));
2576 else
2577 vcpu->arch.tsc_offset = l1_offset;
2578
2579 static_call(kvm_x86_write_tsc_offset)(vcpu, vcpu->arch.tsc_offset);
Luiz Capitulinoa545ab62016-09-07 14:47:19 -04002580}
2581
Ilias Stamatis1ab92872021-06-07 11:54:38 +01002582static void kvm_vcpu_write_tsc_multiplier(struct kvm_vcpu *vcpu, u64 l1_multiplier)
2583{
2584 vcpu->arch.l1_tsc_scaling_ratio = l1_multiplier;
2585
2586 /* Userspace is changing the multiplier while L2 is active */
2587 if (is_guest_mode(vcpu))
2588 vcpu->arch.tsc_scaling_ratio = kvm_calc_nested_tsc_multiplier(
2589 l1_multiplier,
2590 static_call(kvm_x86_get_l2_tsc_multiplier)(vcpu));
2591 else
2592 vcpu->arch.tsc_scaling_ratio = l1_multiplier;
2593
Sean Christopherson938c8742022-05-24 21:56:23 +08002594 if (kvm_caps.has_tsc_control)
Ilias Stamatis1ab92872021-06-07 11:54:38 +01002595 static_call(kvm_x86_write_tsc_multiplier)(
2596 vcpu, vcpu->arch.tsc_scaling_ratio);
2597}
2598
Vitaly Kuznetsovb0c39dc2018-01-24 14:23:36 +01002599static inline bool kvm_check_tsc_unstable(void)
2600{
2601#ifdef CONFIG_X86_64
2602 /*
2603 * TSC is marked unstable when we're running on Hyper-V,
2604 * 'TSC page' clocksource is good.
2605 */
Thomas Gleixnerb95a8a22020-02-07 13:38:56 +01002606 if (pvclock_gtod_data.clock.vclock_mode == VDSO_CLOCKMODE_HVCLOCK)
Vitaly Kuznetsovb0c39dc2018-01-24 14:23:36 +01002607 return false;
2608#endif
2609 return check_tsc_unstable();
2610}
2611
Oliver Upton58d42772021-09-16 18:15:37 +00002612/*
2613 * Infers attempts to synchronize the guest's tsc from host writes. Sets the
2614 * offset for the vcpu and tracks the TSC matching generation that the vcpu
2615 * participates in.
2616 */
2617static void __kvm_synchronize_tsc(struct kvm_vcpu *vcpu, u64 offset, u64 tsc,
2618 u64 ns, bool matched)
2619{
2620 struct kvm *kvm = vcpu->kvm;
2621
2622 lockdep_assert_held(&kvm->arch.tsc_write_lock);
2623
2624 /*
2625 * We also track th most recent recorded KHZ, write and time to
2626 * allow the matching interval to be extended at each write.
2627 */
2628 kvm->arch.last_tsc_nsec = ns;
2629 kvm->arch.last_tsc_write = tsc;
2630 kvm->arch.last_tsc_khz = vcpu->arch.virtual_tsc_khz;
Oliver Upton828ca892021-09-16 18:15:38 +00002631 kvm->arch.last_tsc_offset = offset;
Oliver Upton58d42772021-09-16 18:15:37 +00002632
2633 vcpu->arch.last_guest_tsc = tsc;
2634
2635 kvm_vcpu_write_tsc_offset(vcpu, offset);
2636
2637 if (!matched) {
2638 /*
2639 * We split periods of matched TSC writes into generations.
2640 * For each generation, we track the original measured
2641 * nanosecond time, offset, and write, so if TSCs are in
2642 * sync, we can match exact offset, and if not, we can match
2643 * exact software computation in compute_guest_tsc()
2644 *
2645 * These values are tracked in kvm->arch.cur_xxx variables.
2646 */
2647 kvm->arch.cur_tsc_generation++;
2648 kvm->arch.cur_tsc_nsec = ns;
2649 kvm->arch.cur_tsc_write = tsc;
2650 kvm->arch.cur_tsc_offset = offset;
2651 kvm->arch.nr_vcpus_matched_tsc = 0;
2652 } else if (vcpu->arch.this_tsc_generation != kvm->arch.cur_tsc_generation) {
2653 kvm->arch.nr_vcpus_matched_tsc++;
2654 }
2655
2656 /* Keep track of which generation this VCPU has synchronized to */
2657 vcpu->arch.this_tsc_generation = kvm->arch.cur_tsc_generation;
2658 vcpu->arch.this_tsc_nsec = kvm->arch.cur_tsc_nsec;
2659 vcpu->arch.this_tsc_write = kvm->arch.cur_tsc_write;
2660
2661 kvm_track_tsc_matching(vcpu);
2662}
2663
Paolo Bonzini0c899c22020-09-24 14:45:27 +02002664static void kvm_synchronize_tsc(struct kvm_vcpu *vcpu, u64 data)
Zachary Amsden99e3e302010-08-19 22:07:17 -10002665{
2666 struct kvm *kvm = vcpu->kvm;
Zachary Amsdenf38e0982010-08-19 22:07:20 -10002667 u64 offset, ns, elapsed;
Zachary Amsden99e3e302010-08-19 22:07:17 -10002668 unsigned long flags;
Oliver Upton58d42772021-09-16 18:15:37 +00002669 bool matched = false;
Denis Plotnikovc5e8ec82017-04-07 12:09:52 +03002670 bool synchronizing = false;
Zachary Amsden99e3e302010-08-19 22:07:17 -10002671
Jan Kiszka038f8c12011-02-04 10:49:11 +01002672 raw_spin_lock_irqsave(&kvm->arch.tsc_write_lock, flags);
Ilias Stamatis9b399df2021-05-26 19:44:10 +01002673 offset = kvm_compute_l1_tsc_offset(vcpu, data);
Paolo Bonzini8171cd62020-01-22 14:36:09 +01002674 ns = get_kvmclock_base_ns();
Zachary Amsdenf38e0982010-08-19 22:07:20 -10002675 elapsed = ns - kvm->arch.last_tsc_nsec;
Zachary Amsden5d3cb0f62012-02-03 15:43:51 -02002676
Marcelo Tosatti03ba32c2013-03-11 23:10:24 -03002677 if (vcpu->arch.virtual_tsc_khz) {
Paolo Bonzini0c899c22020-09-24 14:45:27 +02002678 if (data == 0) {
Denis Plotnikovbd8fab32017-04-07 12:09:53 +03002679 /*
2680 * detection of vcpu initialization -- need to sync
2681 * with other vCPUs. This particularly helps to keep
2682 * kvm_clock stable after CPU hotplug
2683 */
2684 synchronizing = true;
2685 } else {
2686 u64 tsc_exp = kvm->arch.last_tsc_write +
2687 nsec_to_cycles(vcpu, elapsed);
2688 u64 tsc_hz = vcpu->arch.virtual_tsc_khz * 1000LL;
2689 /*
2690 * Special case: TSC write with a small delta (1 second)
2691 * of virtual cycle time against real time is
2692 * interpreted as an attempt to synchronize the CPU.
2693 */
2694 synchronizing = data < tsc_exp + tsc_hz &&
2695 data + tsc_hz > tsc_exp;
2696 }
Denis Plotnikovc5e8ec82017-04-07 12:09:52 +03002697 }
Zachary Amsdenf38e0982010-08-19 22:07:20 -10002698
2699 /*
Zachary Amsden5d3cb0f62012-02-03 15:43:51 -02002700 * For a reliable TSC, we can match TSC offsets, and for an unstable
2701 * TSC, we add elapsed time in this computation. We could let the
2702 * compensation code attempt to catch up if we fall behind, but
2703 * it's better to try to match offsets from the beginning.
2704 */
Denis Plotnikovc5e8ec82017-04-07 12:09:52 +03002705 if (synchronizing &&
Zachary Amsden5d3cb0f62012-02-03 15:43:51 -02002706 vcpu->arch.virtual_tsc_khz == kvm->arch.last_tsc_khz) {
Vitaly Kuznetsovb0c39dc2018-01-24 14:23:36 +01002707 if (!kvm_check_tsc_unstable()) {
Zachary Amsdene26101b2012-02-03 15:43:57 -02002708 offset = kvm->arch.cur_tsc_offset;
Zachary Amsdenf38e0982010-08-19 22:07:20 -10002709 } else {
Joerg Roedel857e4092011-03-25 09:44:50 +01002710 u64 delta = nsec_to_cycles(vcpu, elapsed);
Zachary Amsden5d3cb0f62012-02-03 15:43:51 -02002711 data += delta;
Ilias Stamatis9b399df2021-05-26 19:44:10 +01002712 offset = kvm_compute_l1_tsc_offset(vcpu, data);
Zachary Amsdenf38e0982010-08-19 22:07:20 -10002713 }
Marcelo Tosattib48aa972012-11-27 23:29:03 -02002714 matched = true;
Zachary Amsdenf38e0982010-08-19 22:07:20 -10002715 }
Zachary Amsdene26101b2012-02-03 15:43:57 -02002716
Oliver Upton58d42772021-09-16 18:15:37 +00002717 __kvm_synchronize_tsc(vcpu, offset, data, ns, matched);
Zachary Amsdene26101b2012-02-03 15:43:57 -02002718 raw_spin_unlock_irqrestore(&kvm->arch.tsc_write_lock, flags);
Zachary Amsden99e3e302010-08-19 22:07:17 -10002719}
Zachary Amsdene26101b2012-02-03 15:43:57 -02002720
Haozhong Zhang58ea6762015-10-20 15:39:06 +08002721static inline void adjust_tsc_offset_guest(struct kvm_vcpu *vcpu,
2722 s64 adjustment)
2723{
Sean Christopherson56ba77a2020-05-01 21:32:25 -07002724 u64 tsc_offset = vcpu->arch.l1_tsc_offset;
Leonid Shatz326e7422018-11-06 12:14:25 +02002725 kvm_vcpu_write_tsc_offset(vcpu, tsc_offset + adjustment);
Haozhong Zhang58ea6762015-10-20 15:39:06 +08002726}
2727
2728static inline void adjust_tsc_offset_host(struct kvm_vcpu *vcpu, s64 adjustment)
2729{
Sean Christopherson938c8742022-05-24 21:56:23 +08002730 if (vcpu->arch.l1_tsc_scaling_ratio != kvm_caps.default_tsc_scaling_ratio)
Haozhong Zhang58ea6762015-10-20 15:39:06 +08002731 WARN_ON(adjustment < 0);
Jinrong Liang62711e52022-01-25 17:59:07 +08002732 adjustment = kvm_scale_tsc((u64) adjustment,
Ilias Stamatisfe3eb502021-05-26 19:44:11 +01002733 vcpu->arch.l1_tsc_scaling_ratio);
Paolo Bonziniea26e4e2016-11-01 00:39:48 +01002734 adjust_tsc_offset_guest(vcpu, adjustment);
Haozhong Zhang58ea6762015-10-20 15:39:06 +08002735}
2736
Marcelo Tosattid8281992012-11-27 23:29:01 -02002737#ifdef CONFIG_X86_64
2738
Thomas Gleixnera5a1d1c2016-12-21 20:32:01 +01002739static u64 read_tsc(void)
Marcelo Tosattid8281992012-11-27 23:29:01 -02002740{
Thomas Gleixnera5a1d1c2016-12-21 20:32:01 +01002741 u64 ret = (u64)rdtsc_ordered();
Andy Lutomirski03b97302015-06-25 18:44:08 +02002742 u64 last = pvclock_gtod_data.clock.cycle_last;
Marcelo Tosattid8281992012-11-27 23:29:01 -02002743
2744 if (likely(ret >= last))
2745 return ret;
2746
2747 /*
2748 * GCC likes to generate cmov here, but this branch is extremely
Adam Buchbinder6a6256f2016-02-23 15:34:30 -08002749 * predictable (it's just a function of time and the likely is
Marcelo Tosattid8281992012-11-27 23:29:01 -02002750 * very likely) and there's a data dependence, so force GCC
2751 * to generate a branch instead. I don't barrier() because
2752 * we don't actually need a barrier, and if this function
2753 * ever gets inlined it will generate worse code.
2754 */
2755 asm volatile ("");
2756 return last;
2757}
2758
Marcelo Tosatti53fafdb2019-10-28 12:36:22 -02002759static inline u64 vgettsc(struct pvclock_clock *clock, u64 *tsc_timestamp,
2760 int *mode)
Marcelo Tosattid8281992012-11-27 23:29:01 -02002761{
2762 long v;
Vitaly Kuznetsovb0c39dc2018-01-24 14:23:36 +01002763 u64 tsc_pg_val;
Marcelo Tosattid8281992012-11-27 23:29:01 -02002764
Marcelo Tosatti53fafdb2019-10-28 12:36:22 -02002765 switch (clock->vclock_mode) {
Thomas Gleixnerb95a8a22020-02-07 13:38:56 +01002766 case VDSO_CLOCKMODE_HVCLOCK:
Vitaly Kuznetsovb0c39dc2018-01-24 14:23:36 +01002767 tsc_pg_val = hv_read_tsc_page_tsc(hv_get_tsc_page(),
2768 tsc_timestamp);
2769 if (tsc_pg_val != U64_MAX) {
2770 /* TSC page valid */
Thomas Gleixnerb95a8a22020-02-07 13:38:56 +01002771 *mode = VDSO_CLOCKMODE_HVCLOCK;
Marcelo Tosatti53fafdb2019-10-28 12:36:22 -02002772 v = (tsc_pg_val - clock->cycle_last) &
2773 clock->mask;
Vitaly Kuznetsovb0c39dc2018-01-24 14:23:36 +01002774 } else {
2775 /* TSC page invalid */
Thomas Gleixnerb95a8a22020-02-07 13:38:56 +01002776 *mode = VDSO_CLOCKMODE_NONE;
Vitaly Kuznetsovb0c39dc2018-01-24 14:23:36 +01002777 }
2778 break;
Thomas Gleixnerb95a8a22020-02-07 13:38:56 +01002779 case VDSO_CLOCKMODE_TSC:
2780 *mode = VDSO_CLOCKMODE_TSC;
Vitaly Kuznetsovb0c39dc2018-01-24 14:23:36 +01002781 *tsc_timestamp = read_tsc();
Marcelo Tosatti53fafdb2019-10-28 12:36:22 -02002782 v = (*tsc_timestamp - clock->cycle_last) &
2783 clock->mask;
Vitaly Kuznetsovb0c39dc2018-01-24 14:23:36 +01002784 break;
2785 default:
Thomas Gleixnerb95a8a22020-02-07 13:38:56 +01002786 *mode = VDSO_CLOCKMODE_NONE;
Vitaly Kuznetsovb0c39dc2018-01-24 14:23:36 +01002787 }
Marcelo Tosattid8281992012-11-27 23:29:01 -02002788
Thomas Gleixnerb95a8a22020-02-07 13:38:56 +01002789 if (*mode == VDSO_CLOCKMODE_NONE)
Vitaly Kuznetsovb0c39dc2018-01-24 14:23:36 +01002790 *tsc_timestamp = v = 0;
2791
Marcelo Tosatti53fafdb2019-10-28 12:36:22 -02002792 return v * clock->mult;
Marcelo Tosattid8281992012-11-27 23:29:01 -02002793}
2794
Marcelo Tosatti53fafdb2019-10-28 12:36:22 -02002795static int do_monotonic_raw(s64 *t, u64 *tsc_timestamp)
Marcelo Tosattid8281992012-11-27 23:29:01 -02002796{
Marcelo Tosattid8281992012-11-27 23:29:01 -02002797 struct pvclock_gtod_data *gtod = &pvclock_gtod_data;
Thomas Gleixnercbcf2dd2014-07-16 21:04:54 +00002798 unsigned long seq;
2799 int mode;
2800 u64 ns;
Marcelo Tosattid8281992012-11-27 23:29:01 -02002801
Marcelo Tosattid8281992012-11-27 23:29:01 -02002802 do {
2803 seq = read_seqcount_begin(&gtod->seq);
Paolo Bonzini917f9472020-01-22 14:32:20 +01002804 ns = gtod->raw_clock.base_cycles;
Marcelo Tosatti53fafdb2019-10-28 12:36:22 -02002805 ns += vgettsc(&gtod->raw_clock, tsc_timestamp, &mode);
Paolo Bonzini917f9472020-01-22 14:32:20 +01002806 ns >>= gtod->raw_clock.shift;
2807 ns += ktime_to_ns(ktime_add(gtod->raw_clock.offset, gtod->offs_boot));
Marcelo Tosattid8281992012-11-27 23:29:01 -02002808 } while (unlikely(read_seqcount_retry(&gtod->seq, seq)));
Thomas Gleixnercbcf2dd2014-07-16 21:04:54 +00002809 *t = ns;
Marcelo Tosattid8281992012-11-27 23:29:01 -02002810
2811 return mode;
2812}
2813
Arnd Bergmann899a31f2018-04-23 10:04:26 +02002814static int do_realtime(struct timespec64 *ts, u64 *tsc_timestamp)
Marcelo Tosatti55dd00a2017-01-24 15:09:39 -02002815{
2816 struct pvclock_gtod_data *gtod = &pvclock_gtod_data;
2817 unsigned long seq;
2818 int mode;
2819 u64 ns;
2820
2821 do {
2822 seq = read_seqcount_begin(&gtod->seq);
Marcelo Tosatti55dd00a2017-01-24 15:09:39 -02002823 ts->tv_sec = gtod->wall_time_sec;
Paolo Bonzini917f9472020-01-22 14:32:20 +01002824 ns = gtod->clock.base_cycles;
Marcelo Tosatti53fafdb2019-10-28 12:36:22 -02002825 ns += vgettsc(&gtod->clock, tsc_timestamp, &mode);
Marcelo Tosatti55dd00a2017-01-24 15:09:39 -02002826 ns >>= gtod->clock.shift;
2827 } while (unlikely(read_seqcount_retry(&gtod->seq, seq)));
2828
2829 ts->tv_sec += __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns);
2830 ts->tv_nsec = ns;
2831
2832 return mode;
2833}
2834
Vitaly Kuznetsovb0c39dc2018-01-24 14:23:36 +01002835/* returns true if host is using TSC based clocksource */
2836static bool kvm_get_time_and_clockread(s64 *kernel_ns, u64 *tsc_timestamp)
Marcelo Tosattid8281992012-11-27 23:29:01 -02002837{
Marcelo Tosattid8281992012-11-27 23:29:01 -02002838 /* checked again under seqlock below */
Vitaly Kuznetsovb0c39dc2018-01-24 14:23:36 +01002839 if (!gtod_is_based_on_tsc(pvclock_gtod_data.clock.vclock_mode))
Marcelo Tosattid8281992012-11-27 23:29:01 -02002840 return false;
2841
Marcelo Tosatti53fafdb2019-10-28 12:36:22 -02002842 return gtod_is_based_on_tsc(do_monotonic_raw(kernel_ns,
Vitaly Kuznetsovb0c39dc2018-01-24 14:23:36 +01002843 tsc_timestamp));
Marcelo Tosattid8281992012-11-27 23:29:01 -02002844}
Marcelo Tosatti55dd00a2017-01-24 15:09:39 -02002845
Vitaly Kuznetsovb0c39dc2018-01-24 14:23:36 +01002846/* returns true if host is using TSC based clocksource */
Arnd Bergmann899a31f2018-04-23 10:04:26 +02002847static bool kvm_get_walltime_and_clockread(struct timespec64 *ts,
Vitaly Kuznetsovb0c39dc2018-01-24 14:23:36 +01002848 u64 *tsc_timestamp)
Marcelo Tosatti55dd00a2017-01-24 15:09:39 -02002849{
2850 /* checked again under seqlock below */
Vitaly Kuznetsovb0c39dc2018-01-24 14:23:36 +01002851 if (!gtod_is_based_on_tsc(pvclock_gtod_data.clock.vclock_mode))
Marcelo Tosatti55dd00a2017-01-24 15:09:39 -02002852 return false;
2853
Vitaly Kuznetsovb0c39dc2018-01-24 14:23:36 +01002854 return gtod_is_based_on_tsc(do_realtime(ts, tsc_timestamp));
Marcelo Tosatti55dd00a2017-01-24 15:09:39 -02002855}
Marcelo Tosattid8281992012-11-27 23:29:01 -02002856#endif
2857
2858/*
2859 *
Marcelo Tosattib48aa972012-11-27 23:29:03 -02002860 * Assuming a stable TSC across physical CPUS, and a stable TSC
2861 * across virtual CPUs, the following condition is possible.
2862 * Each numbered line represents an event visible to both
Marcelo Tosattid8281992012-11-27 23:29:01 -02002863 * CPUs at the next numbered event.
2864 *
2865 * "timespecX" represents host monotonic time. "tscX" represents
2866 * RDTSC value.
2867 *
2868 * VCPU0 on CPU0 | VCPU1 on CPU1
2869 *
2870 * 1. read timespec0,tsc0
2871 * 2. | timespec1 = timespec0 + N
2872 * | tsc1 = tsc0 + M
2873 * 3. transition to guest | transition to guest
2874 * 4. ret0 = timespec0 + (rdtsc - tsc0) |
2875 * 5. | ret1 = timespec1 + (rdtsc - tsc1)
2876 * | ret1 = timespec0 + N + (rdtsc - (tsc0 + M))
2877 *
2878 * Since ret0 update is visible to VCPU1 at time 5, to obey monotonicity:
2879 *
2880 * - ret0 < ret1
2881 * - timespec0 + (rdtsc - tsc0) < timespec0 + N + (rdtsc - (tsc0 + M))
2882 * ...
2883 * - 0 < N - M => M < N
2884 *
2885 * That is, when timespec0 != timespec1, M < N. Unfortunately that is not
2886 * always the case (the difference between two distinct xtime instances
2887 * might be smaller then the difference between corresponding TSC reads,
2888 * when updating guest vcpus pvclock areas).
2889 *
2890 * To avoid that problem, do not allow visibility of distinct
2891 * system_timestamp/tsc_timestamp values simultaneously: use a master
2892 * copy of host monotonic time values. Update that master copy
2893 * in lockstep.
2894 *
Marcelo Tosattib48aa972012-11-27 23:29:03 -02002895 * Rely on synchronization of host TSCs and guest TSCs for monotonicity.
Marcelo Tosattid8281992012-11-27 23:29:01 -02002896 *
2897 */
2898
2899static void pvclock_update_vm_gtod_copy(struct kvm *kvm)
2900{
2901#ifdef CONFIG_X86_64
2902 struct kvm_arch *ka = &kvm->arch;
2903 int vclock_mode;
Marcelo Tosattib48aa972012-11-27 23:29:03 -02002904 bool host_tsc_clocksource, vcpus_matched;
2905
Paolo Bonzini869b4422021-09-16 18:15:36 +00002906 lockdep_assert_held(&kvm->arch.tsc_write_lock);
Marcelo Tosattib48aa972012-11-27 23:29:03 -02002907 vcpus_matched = (ka->nr_vcpus_matched_tsc + 1 ==
2908 atomic_read(&kvm->online_vcpus));
Marcelo Tosattid8281992012-11-27 23:29:01 -02002909
2910 /*
2911 * If the host uses TSC clock, then passthrough TSC as stable
2912 * to the guest.
2913 */
Marcelo Tosattib48aa972012-11-27 23:29:03 -02002914 host_tsc_clocksource = kvm_get_time_and_clockread(
Marcelo Tosattid8281992012-11-27 23:29:01 -02002915 &ka->master_kernel_ns,
2916 &ka->master_cycle_now);
2917
Marcelo Tosatti16a96022014-05-14 12:43:24 -03002918 ka->use_master_clock = host_tsc_clocksource && vcpus_matched
Ladi Proseka826faf12017-06-26 09:56:43 +02002919 && !ka->backwards_tsc_observed
Marcelo Tosatti54750f22015-01-20 15:54:52 -02002920 && !ka->boot_vcpu_runs_old_kvmclock;
Marcelo Tosattib48aa972012-11-27 23:29:03 -02002921
Marcelo Tosattid8281992012-11-27 23:29:01 -02002922 if (ka->use_master_clock)
2923 atomic_set(&kvm_guest_has_master_clock, 1);
2924
2925 vclock_mode = pvclock_gtod_data.clock.vclock_mode;
Marcelo Tosattib48aa972012-11-27 23:29:03 -02002926 trace_kvm_update_master_clock(ka->use_master_clock, vclock_mode,
2927 vcpus_matched);
Marcelo Tosattid8281992012-11-27 23:29:01 -02002928#endif
2929}
2930
Paolo Bonzini6b6fcd22021-09-16 18:15:32 +00002931static void kvm_make_mclock_inprogress_request(struct kvm *kvm)
Paolo Bonzini2860c4b2016-01-07 15:05:10 +01002932{
2933 kvm_make_all_cpus_request(kvm, KVM_REQ_MCLOCK_INPROGRESS);
2934}
2935
Paolo Bonzini869b4422021-09-16 18:15:36 +00002936static void __kvm_start_pvclock_update(struct kvm *kvm)
Marcelo Tosatti2e762ff2013-08-27 23:55:29 -03002937{
Paolo Bonzini869b4422021-09-16 18:15:36 +00002938 raw_spin_lock_irq(&kvm->arch.tsc_write_lock);
2939 write_seqcount_begin(&kvm->arch.pvclock_sc);
2940}
Marcelo Tosatti2e762ff2013-08-27 23:55:29 -03002941
Paolo Bonzini6b6fcd22021-09-16 18:15:32 +00002942static void kvm_start_pvclock_update(struct kvm *kvm)
Marcelo Tosatti2e762ff2013-08-27 23:55:29 -03002943{
Marcelo Tosatti2e762ff2013-08-27 23:55:29 -03002944 kvm_make_mclock_inprogress_request(kvm);
Paolo Bonzinic2c647f2021-03-25 14:05:11 -04002945
Marcelo Tosatti2e762ff2013-08-27 23:55:29 -03002946 /* no guest entries from this point */
Paolo Bonzini869b4422021-09-16 18:15:36 +00002947 __kvm_start_pvclock_update(kvm);
Paolo Bonzini6b6fcd22021-09-16 18:15:32 +00002948}
Marcelo Tosatti2e762ff2013-08-27 23:55:29 -03002949
Paolo Bonzini6b6fcd22021-09-16 18:15:32 +00002950static void kvm_end_pvclock_update(struct kvm *kvm)
2951{
2952 struct kvm_arch *ka = &kvm->arch;
2953 struct kvm_vcpu *vcpu;
Marc Zyngier46808a42021-11-16 16:04:02 +00002954 unsigned long i;
Paolo Bonzini6b6fcd22021-09-16 18:15:32 +00002955
Paolo Bonzini869b4422021-09-16 18:15:36 +00002956 write_seqcount_end(&ka->pvclock_sc);
2957 raw_spin_unlock_irq(&ka->tsc_write_lock);
Marcelo Tosatti2e762ff2013-08-27 23:55:29 -03002958 kvm_for_each_vcpu(i, vcpu, kvm)
Guo Hui Liu105b21b2014-09-12 13:43:19 +08002959 kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
Marcelo Tosatti2e762ff2013-08-27 23:55:29 -03002960
2961 /* guest entries allowed */
2962 kvm_for_each_vcpu(i, vcpu, kvm)
Radim Krčmář72875d82017-04-26 22:32:19 +02002963 kvm_clear_request(KVM_REQ_MCLOCK_INPROGRESS, vcpu);
Marcelo Tosatti2e762ff2013-08-27 23:55:29 -03002964}
2965
Paolo Bonzini6b6fcd22021-09-16 18:15:32 +00002966static void kvm_update_masterclock(struct kvm *kvm)
2967{
Vitaly Kuznetsov42dcbe72022-04-07 22:10:13 +02002968 kvm_hv_request_tsc_page_update(kvm);
Paolo Bonzini6b6fcd22021-09-16 18:15:32 +00002969 kvm_start_pvclock_update(kvm);
2970 pvclock_update_vm_gtod_copy(kvm);
2971 kvm_end_pvclock_update(kvm);
Zachary Amsden99e3e302010-08-19 22:07:17 -10002972}
2973
Paolo Bonzini869b4422021-09-16 18:15:36 +00002974/* Called within read_seqcount_begin/retry for kvm->pvclock_sc. */
2975static void __get_kvmclock(struct kvm *kvm, struct kvm_clock_data *data)
Paolo Bonzini108b2492016-09-01 14:21:03 +02002976{
Paolo Bonzini108b2492016-09-01 14:21:03 +02002977 struct kvm_arch *ka = &kvm->arch;
Paolo Bonzini8b953442016-11-16 18:31:30 +01002978 struct pvclock_vcpu_time_info hv_clock;
Paolo Bonzini8b953442016-11-16 18:31:30 +01002979
Wanpeng Lie2c22062017-05-11 18:12:05 -07002980 /* both __this_cpu_read() and rdtsc() should be on the same cpu */
2981 get_cpu();
2982
Paolo Bonzini869b4422021-09-16 18:15:36 +00002983 data->flags = 0;
2984 if (ka->use_master_clock && __this_cpu_read(cpu_tsc_khz)) {
Oliver Uptonc68dc1b2021-09-16 18:15:35 +00002985#ifdef CONFIG_X86_64
2986 struct timespec64 ts;
2987
2988 if (kvm_get_walltime_and_clockread(&ts, &data->host_tsc)) {
2989 data->realtime = ts.tv_nsec + NSEC_PER_SEC * ts.tv_sec;
2990 data->flags |= KVM_CLOCK_REALTIME | KVM_CLOCK_HOST_TSC;
2991 } else
2992#endif
2993 data->host_tsc = rdtsc();
2994
Paolo Bonzini869b4422021-09-16 18:15:36 +00002995 data->flags |= KVM_CLOCK_TSC_STABLE;
2996 hv_clock.tsc_timestamp = ka->master_cycle_now;
2997 hv_clock.system_time = ka->master_kernel_ns + ka->kvmclock_offset;
Wanpeng Lie70b57a62017-11-20 14:55:05 -08002998 kvm_get_time_scale(NSEC_PER_SEC, __this_cpu_read(cpu_tsc_khz) * 1000LL,
2999 &hv_clock.tsc_shift,
3000 &hv_clock.tsc_to_system_mul);
Oliver Uptonc68dc1b2021-09-16 18:15:35 +00003001 data->clock = __pvclock_read_cycles(&hv_clock, data->host_tsc);
Oliver Upton55c0cef2021-09-16 18:15:34 +00003002 } else {
3003 data->clock = get_kvmclock_base_ns() + ka->kvmclock_offset;
3004 }
Wanpeng Lie2c22062017-05-11 18:12:05 -07003005
3006 put_cpu();
Oliver Upton55c0cef2021-09-16 18:15:34 +00003007}
Wanpeng Lie2c22062017-05-11 18:12:05 -07003008
Paolo Bonzini869b4422021-09-16 18:15:36 +00003009static void get_kvmclock(struct kvm *kvm, struct kvm_clock_data *data)
3010{
3011 struct kvm_arch *ka = &kvm->arch;
3012 unsigned seq;
3013
3014 do {
3015 seq = read_seqcount_begin(&ka->pvclock_sc);
3016 __get_kvmclock(kvm, data);
3017 } while (read_seqcount_retry(&ka->pvclock_sc, seq));
3018}
3019
Oliver Upton55c0cef2021-09-16 18:15:34 +00003020u64 get_kvmclock_ns(struct kvm *kvm)
3021{
3022 struct kvm_clock_data data;
3023
Oliver Upton55c0cef2021-09-16 18:15:34 +00003024 get_kvmclock(kvm, &data);
3025 return data.clock;
Paolo Bonzini108b2492016-09-01 14:21:03 +02003026}
3027
David Woodhouse916d3602022-03-03 15:41:14 +00003028static void kvm_setup_guest_pvclock(struct kvm_vcpu *v,
3029 struct gfn_to_pfn_cache *gpc,
3030 unsigned int offset)
Paolo Bonzini0d6dd2f2016-09-01 14:20:09 +02003031{
3032 struct kvm_vcpu_arch *vcpu = &v->arch;
David Woodhouse916d3602022-03-03 15:41:14 +00003033 struct pvclock_vcpu_time_info *guest_hv_clock;
3034 unsigned long flags;
Paolo Bonzini0d6dd2f2016-09-01 14:20:09 +02003035
David Woodhouse916d3602022-03-03 15:41:14 +00003036 read_lock_irqsave(&gpc->lock, flags);
3037 while (!kvm_gfn_to_pfn_cache_check(v->kvm, gpc, gpc->gpa,
3038 offset + sizeof(*guest_hv_clock))) {
3039 read_unlock_irqrestore(&gpc->lock, flags);
Paolo Bonzini0d6dd2f2016-09-01 14:20:09 +02003040
David Woodhouse916d3602022-03-03 15:41:14 +00003041 if (kvm_gfn_to_pfn_cache_refresh(v->kvm, gpc, gpc->gpa,
3042 offset + sizeof(*guest_hv_clock)))
3043 return;
3044
3045 read_lock_irqsave(&gpc->lock, flags);
3046 }
3047
3048 guest_hv_clock = (void *)(gpc->khva + offset);
3049
3050 /*
3051 * This VCPU is paused, but it's legal for a guest to read another
Paolo Bonzini0d6dd2f2016-09-01 14:20:09 +02003052 * VCPU's kvmclock, so we really have to follow the specification where
3053 * it says that version is odd if data is being modified, and even after
3054 * it is consistent.
Paolo Bonzini0d6dd2f2016-09-01 14:20:09 +02003055 */
Paolo Bonzini0d6dd2f2016-09-01 14:20:09 +02003056
David Woodhouse916d3602022-03-03 15:41:14 +00003057 guest_hv_clock->version = vcpu->hv_clock.version = (guest_hv_clock->version + 1) | 1;
Paolo Bonzini0d6dd2f2016-09-01 14:20:09 +02003058 smp_wmb();
3059
3060 /* retain PVCLOCK_GUEST_STOPPED if set in guest copy */
David Woodhouse916d3602022-03-03 15:41:14 +00003061 vcpu->hv_clock.flags |= (guest_hv_clock->flags & PVCLOCK_GUEST_STOPPED);
Paolo Bonzini0d6dd2f2016-09-01 14:20:09 +02003062
3063 if (vcpu->pvclock_set_guest_stopped_request) {
3064 vcpu->hv_clock.flags |= PVCLOCK_GUEST_STOPPED;
3065 vcpu->pvclock_set_guest_stopped_request = false;
3066 }
3067
David Woodhouse916d3602022-03-03 15:41:14 +00003068 memcpy(guest_hv_clock, &vcpu->hv_clock, sizeof(*guest_hv_clock));
Paolo Bonzini0d6dd2f2016-09-01 14:20:09 +02003069 smp_wmb();
3070
David Woodhouse916d3602022-03-03 15:41:14 +00003071 guest_hv_clock->version = ++vcpu->hv_clock.version;
3072
3073 mark_page_dirty_in_slot(v->kvm, gpc->memslot, gpc->gpa >> PAGE_SHIFT);
3074 read_unlock_irqrestore(&gpc->lock, flags);
3075
3076 trace_kvm_pvclock_update(v->vcpu_id, &vcpu->hv_clock);
Paolo Bonzini0d6dd2f2016-09-01 14:20:09 +02003077}
3078
Zachary Amsden34c238a2010-09-18 14:38:14 -10003079static int kvm_guest_time_update(struct kvm_vcpu *v)
Glauber de Oliveira Costa18068522008-02-15 17:52:47 -02003080{
Paolo Bonzini78db6a52016-02-08 14:51:40 +01003081 unsigned long flags, tgt_tsc_khz;
Paolo Bonzini869b4422021-09-16 18:15:36 +00003082 unsigned seq;
Glauber de Oliveira Costa18068522008-02-15 17:52:47 -02003083 struct kvm_vcpu_arch *vcpu = &v->arch;
Marcelo Tosattid8281992012-11-27 23:29:01 -02003084 struct kvm_arch *ka = &v->kvm->arch;
Marcelo Tosattif25e6562014-01-06 12:18:59 -02003085 s64 kernel_ns;
Marcelo Tosattid8281992012-11-27 23:29:01 -02003086 u64 tsc_timestamp, host_tsc;
Marcelo Tosatti51d59c62012-08-03 15:57:49 -03003087 u8 pvclock_flags;
Marcelo Tosattid8281992012-11-27 23:29:01 -02003088 bool use_master_clock;
3089
3090 kernel_ns = 0;
3091 host_tsc = 0;
Gerd Hoffmann50d0a0f2008-06-03 16:17:31 +02003092
Zachary Amsden1d5f0662010-08-19 22:07:30 -10003093 /*
Marcelo Tosattid8281992012-11-27 23:29:01 -02003094 * If the host uses TSC clock, then passthrough TSC as stable
3095 * to the guest.
3096 */
Paolo Bonzini869b4422021-09-16 18:15:36 +00003097 do {
3098 seq = read_seqcount_begin(&ka->pvclock_sc);
3099 use_master_clock = ka->use_master_clock;
3100 if (use_master_clock) {
3101 host_tsc = ka->master_cycle_now;
3102 kernel_ns = ka->master_kernel_ns;
3103 }
3104 } while (read_seqcount_retry(&ka->pvclock_sc, seq));
Marcelo Tosattic09664b2013-03-18 13:54:32 -03003105
3106 /* Keep irq disabled to prevent changes to the clock */
3107 local_irq_save(flags);
Paolo Bonzini78db6a52016-02-08 14:51:40 +01003108 tgt_tsc_khz = __this_cpu_read(cpu_tsc_khz);
3109 if (unlikely(tgt_tsc_khz == 0)) {
Marcelo Tosattic09664b2013-03-18 13:54:32 -03003110 local_irq_restore(flags);
3111 kvm_make_request(KVM_REQ_CLOCK_UPDATE, v);
3112 return 1;
3113 }
Marcelo Tosattid8281992012-11-27 23:29:01 -02003114 if (!use_master_clock) {
Andy Lutomirski4ea16362015-06-25 18:44:07 +02003115 host_tsc = rdtsc();
Paolo Bonzini8171cd62020-01-22 14:36:09 +01003116 kernel_ns = get_kvmclock_base_ns();
Marcelo Tosattid8281992012-11-27 23:29:01 -02003117 }
3118
Haozhong Zhang4ba76532015-10-20 15:39:07 +08003119 tsc_timestamp = kvm_read_l1_tsc(v, host_tsc);
Marcelo Tosattid8281992012-11-27 23:29:01 -02003120
3121 /*
Zachary Amsdenc2855452010-09-18 14:38:15 -10003122 * We may have to catch up the TSC to match elapsed wall clock
3123 * time for two reasons, even if kvmclock is used.
3124 * 1) CPU could have been running below the maximum TSC rate
3125 * 2) Broken TSC compensation resets the base at each VCPU
3126 * entry to avoid unknown leaps of TSC even when running
3127 * again on the same CPU. This may cause apparent elapsed
3128 * time to disappear, and the guest to stand still or run
3129 * very slowly.
3130 */
3131 if (vcpu->tsc_catchup) {
3132 u64 tsc = compute_guest_tsc(v, kernel_ns);
3133 if (tsc > tsc_timestamp) {
Marcelo Tosattif1e2b262012-02-03 15:43:55 -02003134 adjust_tsc_offset_guest(v, tsc - tsc_timestamp);
Zachary Amsdenc2855452010-09-18 14:38:15 -10003135 tsc_timestamp = tsc;
3136 }
3137 }
3138
Glauber de Oliveira Costa18068522008-02-15 17:52:47 -02003139 local_irq_restore(flags);
3140
Paolo Bonzini0d6dd2f2016-09-01 14:20:09 +02003141 /* With all the info we got, fill in the values */
Zachary Amsdenc2855452010-09-18 14:38:15 -10003142
Sean Christopherson938c8742022-05-24 21:56:23 +08003143 if (kvm_caps.has_tsc_control)
Jinrong Liang62711e52022-01-25 17:59:07 +08003144 tgt_tsc_khz = kvm_scale_tsc(tgt_tsc_khz,
Ilias Stamatisfe3eb502021-05-26 19:44:11 +01003145 v->arch.l1_tsc_scaling_ratio);
Paolo Bonzini78db6a52016-02-08 14:51:40 +01003146
3147 if (unlikely(vcpu->hw_tsc_khz != tgt_tsc_khz)) {
Paolo Bonzini3ae13fa2016-02-08 15:11:15 +01003148 kvm_get_time_scale(NSEC_PER_SEC, tgt_tsc_khz * 1000LL,
Zachary Amsden5f4e3f82010-09-18 14:38:13 -10003149 &vcpu->hv_clock.tsc_shift,
3150 &vcpu->hv_clock.tsc_to_system_mul);
Paolo Bonzini78db6a52016-02-08 14:51:40 +01003151 vcpu->hw_tsc_khz = tgt_tsc_khz;
Zachary Amsden8cfdc002010-08-19 22:07:21 -10003152 }
3153
Zachary Amsden1d5f0662010-08-19 22:07:30 -10003154 vcpu->hv_clock.tsc_timestamp = tsc_timestamp;
Zachary Amsden759379d2010-08-19 22:07:25 -10003155 vcpu->hv_clock.system_time = kernel_ns + v->kvm->arch.kvmclock_offset;
Zachary Amsden28e46392010-09-18 14:38:12 -10003156 vcpu->last_guest_tsc = tsc_timestamp;
Marcelo Tosatti51d59c62012-08-03 15:57:49 -03003157
Marcelo Tosattid8281992012-11-27 23:29:01 -02003158 /* If the host uses TSC clocksource, then it is stable */
Paolo Bonzini0d6dd2f2016-09-01 14:20:09 +02003159 pvclock_flags = 0;
Marcelo Tosattid8281992012-11-27 23:29:01 -02003160 if (use_master_clock)
3161 pvclock_flags |= PVCLOCK_TSC_STABLE_BIT;
3162
Marcelo Tosatti78c03372012-11-27 23:28:47 -02003163 vcpu->hv_clock.flags = pvclock_flags;
3164
David Woodhouse916d3602022-03-03 15:41:14 +00003165 if (vcpu->pv_time.active)
3166 kvm_setup_guest_pvclock(v, &vcpu->pv_time, 0);
David Woodhouse7caf9572022-03-03 15:41:15 +00003167 if (vcpu->xen.vcpu_info_cache.active)
3168 kvm_setup_guest_pvclock(v, &vcpu->xen.vcpu_info_cache,
3169 offsetof(struct compat_vcpu_info, time));
David Woodhouse69d413c2022-03-03 15:41:16 +00003170 if (vcpu->xen.vcpu_time_info_cache.active)
3171 kvm_setup_guest_pvclock(v, &vcpu->xen.vcpu_time_info_cache, 0);
Vitaly Kuznetsov42dcbe72022-04-07 22:10:13 +02003172 kvm_hv_setup_tsc_page(v->kvm, &vcpu->hv_clock);
Zachary Amsden8cfdc002010-08-19 22:07:21 -10003173 return 0;
Gerd Hoffmannc8076602009-02-04 17:52:04 +01003174}
3175
Marcelo Tosatti0061d53d2013-05-09 20:21:41 -03003176/*
3177 * kvmclock updates which are isolated to a given vcpu, such as
3178 * vcpu->cpu migration, should not allow system_timestamp from
3179 * the rest of the vcpus to remain static. Otherwise ntp frequency
3180 * correction applies to one vcpu's system_timestamp but not
3181 * the others.
3182 *
3183 * So in those cases, request a kvmclock update for all vcpus.
Andrew Jones7e44e442014-02-28 12:52:54 +01003184 * We need to rate-limit these requests though, as they can
3185 * considerably slow guests that have a large number of vcpus.
3186 * The time for a remote vcpu to update its kvmclock is bound
3187 * by the delay we use to rate-limit the updates.
Marcelo Tosatti0061d53d2013-05-09 20:21:41 -03003188 */
3189
Andrew Jones7e44e442014-02-28 12:52:54 +01003190#define KVMCLOCK_UPDATE_DELAY msecs_to_jiffies(100)
3191
3192static void kvmclock_update_fn(struct work_struct *work)
Marcelo Tosatti0061d53d2013-05-09 20:21:41 -03003193{
Marc Zyngier46808a42021-11-16 16:04:02 +00003194 unsigned long i;
Andrew Jones7e44e442014-02-28 12:52:54 +01003195 struct delayed_work *dwork = to_delayed_work(work);
3196 struct kvm_arch *ka = container_of(dwork, struct kvm_arch,
3197 kvmclock_update_work);
3198 struct kvm *kvm = container_of(ka, struct kvm, arch);
Marcelo Tosatti0061d53d2013-05-09 20:21:41 -03003199 struct kvm_vcpu *vcpu;
3200
3201 kvm_for_each_vcpu(i, vcpu, kvm) {
Guo Hui Liu105b21b2014-09-12 13:43:19 +08003202 kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
Marcelo Tosatti0061d53d2013-05-09 20:21:41 -03003203 kvm_vcpu_kick(vcpu);
3204 }
3205}
3206
Andrew Jones7e44e442014-02-28 12:52:54 +01003207static void kvm_gen_kvmclock_update(struct kvm_vcpu *v)
3208{
3209 struct kvm *kvm = v->kvm;
3210
Guo Hui Liu105b21b2014-09-12 13:43:19 +08003211 kvm_make_request(KVM_REQ_CLOCK_UPDATE, v);
Andrew Jones7e44e442014-02-28 12:52:54 +01003212 schedule_delayed_work(&kvm->arch.kvmclock_update_work,
3213 KVMCLOCK_UPDATE_DELAY);
3214}
3215
Andrew Jones332967a2014-02-28 12:52:55 +01003216#define KVMCLOCK_SYNC_PERIOD (300 * HZ)
3217
3218static void kvmclock_sync_fn(struct work_struct *work)
3219{
3220 struct delayed_work *dwork = to_delayed_work(work);
3221 struct kvm_arch *ka = container_of(dwork, struct kvm_arch,
3222 kvmclock_sync_work);
3223 struct kvm *kvm = container_of(ka, struct kvm, arch);
3224
Marcelo Tosatti630994b2015-05-12 22:42:04 -03003225 if (!kvmclock_periodic_sync)
3226 return;
3227
Andrew Jones332967a2014-02-28 12:52:55 +01003228 schedule_delayed_work(&kvm->arch.kvmclock_update_work, 0);
3229 schedule_delayed_work(&kvm->arch.kvmclock_sync_work,
3230 KVMCLOCK_SYNC_PERIOD);
3231}
3232
Jue Wang281b5272022-06-10 10:11:32 -07003233/* These helpers are safe iff @msr is known to be an MCx bank MSR. */
3234static bool is_mci_control_msr(u32 msr)
3235{
3236 return (msr & 3) == 0;
3237}
3238static bool is_mci_status_msr(u32 msr)
3239{
3240 return (msr & 3) == 1;
3241}
3242
Borislav Petkov191c8132019-04-18 18:32:50 +02003243/*
3244 * On AMD, HWCR[McStatusWrEn] controls whether setting MCi_STATUS results in #GP.
3245 */
3246static bool can_set_mci_status(struct kvm_vcpu *vcpu)
3247{
3248 /* McStatusWrEn enabled? */
Sean Christopherson23493d02020-03-04 17:34:33 -08003249 if (guest_cpuid_is_amd_or_hygon(vcpu))
Borislav Petkov191c8132019-04-18 18:32:50 +02003250 return !!(vcpu->arch.msr_hwcr & BIT_ULL(18));
3251
3252 return false;
3253}
3254
Wanpeng Li9ffd9862017-10-19 06:47:56 -07003255static int set_msr_mce(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
Huang Ying890ca9a2009-05-11 16:48:15 +08003256{
3257 u64 mcg_cap = vcpu->arch.mcg_cap;
3258 unsigned bank_num = mcg_cap & 0xff;
Wanpeng Li9ffd9862017-10-19 06:47:56 -07003259 u32 msr = msr_info->index;
3260 u64 data = msr_info->data;
Jue Wang281b5272022-06-10 10:11:32 -07003261 u32 offset, last_msr;
Huang Ying890ca9a2009-05-11 16:48:15 +08003262
3263 switch (msr) {
3264 case MSR_IA32_MCG_STATUS:
3265 vcpu->arch.mcg_status = data;
3266 break;
3267 case MSR_IA32_MCG_CTL:
Paolo Bonzini44883f02018-07-26 13:01:52 +02003268 if (!(mcg_cap & MCG_CTL_P) &&
3269 (data || !msr_info->host_initiated))
Huang Ying890ca9a2009-05-11 16:48:15 +08003270 return 1;
3271 if (data != 0 && data != ~(u64)0)
Paolo Bonzini44883f02018-07-26 13:01:52 +02003272 return 1;
Huang Ying890ca9a2009-05-11 16:48:15 +08003273 vcpu->arch.mcg_ctl = data;
3274 break;
Jue Wang281b5272022-06-10 10:11:32 -07003275 case MSR_IA32_MC0_CTL2 ... MSR_IA32_MCx_CTL2(KVM_MAX_MCE_BANKS) - 1:
3276 last_msr = MSR_IA32_MCx_CTL2(bank_num) - 1;
3277 if (msr > last_msr)
3278 return 1;
Marios Pomonis6ec4c5e2019-12-11 12:47:49 -08003279
Jue Wang281b5272022-06-10 10:11:32 -07003280 if (!(mcg_cap & MCG_CMCI_P) && (data || !msr_info->host_initiated))
3281 return 1;
3282 /* An attempt to write a 1 to a reserved bit raises #GP */
3283 if (data & ~(MCI_CTL2_CMCI_EN | MCI_CTL2_CMCI_THRESHOLD_MASK))
3284 return 1;
3285 offset = array_index_nospec(msr - MSR_IA32_MC0_CTL2,
3286 last_msr + 1 - MSR_IA32_MC0_CTL2);
3287 vcpu->arch.mci_ctl2_banks[offset] = data;
3288 break;
3289 case MSR_IA32_MC0_CTL ... MSR_IA32_MCx_CTL(KVM_MAX_MCE_BANKS) - 1:
3290 last_msr = MSR_IA32_MCx_CTL(bank_num) - 1;
3291 if (msr > last_msr)
3292 return 1;
3293
3294 /*
3295 * Only 0 or all 1s can be written to IA32_MCi_CTL, all other
3296 * values are architecturally undefined. But, some Linux
3297 * kernels clear bit 10 in bank 4 to workaround a BIOS/GART TLB
3298 * issue on AMD K8s, allow bit 10 to be clear when setting all
3299 * other bits in order to avoid an uncaught #GP in the guest.
Sean Christophersonf5223a32022-05-12 22:27:15 +00003300 *
3301 * UNIXWARE clears bit 0 of MC1_CTL to ignore correctable,
3302 * single-bit ECC data errors.
Jue Wang281b5272022-06-10 10:11:32 -07003303 */
3304 if (is_mci_control_msr(msr) &&
3305 data != 0 && (data | (1 << 10) | 1) != ~(u64)0)
3306 return 1;
Borislav Petkov191c8132019-04-18 18:32:50 +02003307
Jue Wang281b5272022-06-10 10:11:32 -07003308 /*
3309 * All CPUs allow writing 0 to MCi_STATUS MSRs to clear the MSR.
3310 * AMD-based CPUs allow non-zero values, but if and only if
3311 * HWCR[McStatusWrEn] is set.
3312 */
3313 if (!msr_info->host_initiated && is_mci_status_msr(msr) &&
3314 data != 0 && !can_set_mci_status(vcpu))
3315 return 1;
Borislav Petkov191c8132019-04-18 18:32:50 +02003316
Jue Wang281b5272022-06-10 10:11:32 -07003317 offset = array_index_nospec(msr - MSR_IA32_MC0_CTL,
3318 last_msr + 1 - MSR_IA32_MC0_CTL);
3319 vcpu->arch.mce_banks[offset] = data;
3320 break;
Huang Ying890ca9a2009-05-11 16:48:15 +08003321 default:
Huang Ying890ca9a2009-05-11 16:48:15 +08003322 return 1;
3323 }
3324 return 0;
3325}
3326
Vitaly Kuznetsov2635b5c2020-05-25 16:41:20 +02003327static inline bool kvm_pv_async_pf_enabled(struct kvm_vcpu *vcpu)
3328{
3329 u64 mask = KVM_ASYNC_PF_ENABLED | KVM_ASYNC_PF_DELIVERY_AS_INT;
3330
3331 return (vcpu->arch.apf.msr_en_val & mask) == mask;
3332}
3333
Gleb Natapov344d9582010-10-14 11:22:50 +02003334static int kvm_pv_enable_async_pf(struct kvm_vcpu *vcpu, u64 data)
3335{
3336 gpa_t gpa = data & ~0x3f;
3337
Vitaly Kuznetsov2635b5c2020-05-25 16:41:20 +02003338 /* Bits 4:5 are reserved, Should be zero */
3339 if (data & 0x30)
Gleb Natapov344d9582010-10-14 11:22:50 +02003340 return 1;
3341
Oliver Upton66570e962020-08-18 15:24:28 +00003342 if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_VMEXIT) &&
3343 (data & KVM_ASYNC_PF_DELIVERY_AS_PF_VMEXIT))
3344 return 1;
3345
3346 if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_INT) &&
3347 (data & KVM_ASYNC_PF_DELIVERY_AS_INT))
3348 return 1;
3349
Wanpeng Li9d3c4472020-06-29 18:26:31 +08003350 if (!lapic_in_kernel(vcpu))
Vitaly Kuznetsovd831de12020-09-11 11:31:47 +02003351 return data ? 1 : 0;
Wanpeng Li9d3c4472020-06-29 18:26:31 +08003352
Vitaly Kuznetsov2635b5c2020-05-25 16:41:20 +02003353 vcpu->arch.apf.msr_en_val = data;
Gleb Natapov344d9582010-10-14 11:22:50 +02003354
Vitaly Kuznetsov2635b5c2020-05-25 16:41:20 +02003355 if (!kvm_pv_async_pf_enabled(vcpu)) {
Gleb Natapov344d9582010-10-14 11:22:50 +02003356 kvm_clear_async_pf_completion_queue(vcpu);
3357 kvm_async_pf_hash_reset(vcpu);
3358 return 0;
3359 }
3360
Paolo Bonzini4e335d92017-05-02 16:20:18 +02003361 if (kvm_gfn_to_hva_cache_init(vcpu->kvm, &vcpu->arch.apf.data, gpa,
Vitaly Kuznetsov68fd66f2020-05-25 16:41:17 +02003362 sizeof(u64)))
Gleb Natapov344d9582010-10-14 11:22:50 +02003363 return 1;
3364
Gleb Natapov6adba522010-10-14 11:22:55 +02003365 vcpu->arch.apf.send_user_only = !(data & KVM_ASYNC_PF_SEND_ALWAYS);
Wanpeng Li52a5c152017-07-13 18:30:42 -07003366 vcpu->arch.apf.delivery_as_pf_vmexit = data & KVM_ASYNC_PF_DELIVERY_AS_PF_VMEXIT;
Vitaly Kuznetsov2635b5c2020-05-25 16:41:20 +02003367
Gleb Natapov344d9582010-10-14 11:22:50 +02003368 kvm_async_pf_wakeup_all(vcpu);
Vitaly Kuznetsov2635b5c2020-05-25 16:41:20 +02003369
3370 return 0;
3371}
3372
3373static int kvm_pv_enable_async_pf_int(struct kvm_vcpu *vcpu, u64 data)
3374{
3375 /* Bits 8-63 are reserved */
3376 if (data >> 8)
3377 return 1;
3378
3379 if (!lapic_in_kernel(vcpu))
3380 return 1;
3381
3382 vcpu->arch.apf.msr_int_val = data;
3383
3384 vcpu->arch.apf.vec = data & KVM_ASYNC_PF_VEC_MASK;
3385
Gleb Natapov344d9582010-10-14 11:22:50 +02003386 return 0;
3387}
3388
Glauber Costa12f9a482011-02-01 14:16:40 -05003389static void kvmclock_reset(struct kvm_vcpu *vcpu)
3390{
Michal Luczaj52491a382022-10-13 21:12:19 +00003391 kvm_gpc_deactivate(vcpu->kvm, &vcpu->arch.pv_time);
Paolo Bonzini49dedf02019-10-10 12:49:22 +02003392 vcpu->arch.time = 0;
Glauber Costa12f9a482011-02-01 14:16:40 -05003393}
3394
Sean Christopherson77809382020-03-20 14:28:18 -07003395static void kvm_vcpu_flush_tlb_all(struct kvm_vcpu *vcpu)
Wanpeng Lif38a7b72017-12-12 17:33:04 -08003396{
3397 ++vcpu->stat.tlb_flush;
Sean Christophersone27bc042022-01-28 00:51:50 +00003398 static_call(kvm_x86_flush_tlb_all)(vcpu);
Wanpeng Lif38a7b72017-12-12 17:33:04 -08003399}
3400
Vitaly Kuznetsov0baedd72020-03-25 12:28:24 -04003401static void kvm_vcpu_flush_tlb_guest(struct kvm_vcpu *vcpu)
3402{
3403 ++vcpu->stat.tlb_flush;
Lai Jiangshanb53e84e2021-06-01 01:22:56 +08003404
3405 if (!tdp_enabled) {
Lai Jiangshan61b05a9f2021-10-19 19:01:54 +08003406 /*
Lai Jiangshanb53e84e2021-06-01 01:22:56 +08003407 * A TLB flush on behalf of the guest is equivalent to
3408 * INVPCID(all), toggling CR4.PGE, etc., which requires
Lai Jiangshan61b05a9f2021-10-19 19:01:54 +08003409 * a forced sync of the shadow page tables. Ensure all the
3410 * roots are synced and the guest TLB in hardware is clean.
Lai Jiangshanb53e84e2021-06-01 01:22:56 +08003411 */
Lai Jiangshan61b05a9f2021-10-19 19:01:54 +08003412 kvm_mmu_sync_roots(vcpu);
3413 kvm_mmu_sync_prev_roots(vcpu);
Lai Jiangshanb53e84e2021-06-01 01:22:56 +08003414 }
3415
Sean Christophersone27bc042022-01-28 00:51:50 +00003416 static_call(kvm_x86_flush_tlb_guest)(vcpu);
Vitaly Kuznetsov0baedd72020-03-25 12:28:24 -04003417}
3418
Sean Christopherson40e5f9082021-11-25 01:49:43 +00003419
3420static inline void kvm_vcpu_flush_tlb_current(struct kvm_vcpu *vcpu)
3421{
3422 ++vcpu->stat.tlb_flush;
Sean Christophersone27bc042022-01-28 00:51:50 +00003423 static_call(kvm_x86_flush_tlb_current)(vcpu);
Sean Christopherson40e5f9082021-11-25 01:49:43 +00003424}
3425
3426/*
3427 * Service "local" TLB flush requests, which are specific to the current MMU
3428 * context. In addition to the generic event handling in vcpu_enter_guest(),
3429 * TLB flushes that are targeted at an MMU context also need to be serviced
3430 * prior before nested VM-Enter/VM-Exit.
3431 */
3432void kvm_service_local_tlb_flush_requests(struct kvm_vcpu *vcpu)
3433{
3434 if (kvm_check_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu))
3435 kvm_vcpu_flush_tlb_current(vcpu);
3436
3437 if (kvm_check_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu))
3438 kvm_vcpu_flush_tlb_guest(vcpu);
3439}
3440EXPORT_SYMBOL_GPL(kvm_service_local_tlb_flush_requests);
3441
Glauber Costac9aaa892011-07-11 15:28:14 -04003442static void record_steal_time(struct kvm_vcpu *vcpu)
3443{
David Woodhouse7e2175e2021-11-02 17:36:39 +00003444 struct gfn_to_hva_cache *ghc = &vcpu->arch.st.cache;
3445 struct kvm_steal_time __user *st;
3446 struct kvm_memslots *slots;
Paolo Bonzini901d3762022-08-04 15:28:32 +02003447 gpa_t gpa = vcpu->arch.st.msr_val & KVM_STEAL_VALID_BITS;
David Woodhouse7e2175e2021-11-02 17:36:39 +00003448 u64 steal;
3449 u32 version;
Boris Ostrovskyb0431382019-12-05 03:45:32 +00003450
David Woodhouse30b5c852021-03-01 12:53:09 +00003451 if (kvm_xen_msr_enabled(vcpu->kvm)) {
3452 kvm_xen_runstate_set_running(vcpu);
3453 return;
3454 }
3455
Glauber Costac9aaa892011-07-11 15:28:14 -04003456 if (!(vcpu->arch.st.msr_val & KVM_MSR_ENABLED))
3457 return;
3458
David Woodhouse7e2175e2021-11-02 17:36:39 +00003459 if (WARN_ON_ONCE(current->mm != vcpu->kvm->mm))
Glauber Costac9aaa892011-07-11 15:28:14 -04003460 return;
3461
David Woodhouse7e2175e2021-11-02 17:36:39 +00003462 slots = kvm_memslots(vcpu->kvm);
3463
3464 if (unlikely(slots->generation != ghc->generation ||
Paolo Bonzini901d3762022-08-04 15:28:32 +02003465 gpa != ghc->gpa ||
David Woodhouse7e2175e2021-11-02 17:36:39 +00003466 kvm_is_error_hva(ghc->hva) || !ghc->memslot)) {
David Woodhouse7e2175e2021-11-02 17:36:39 +00003467 /* We rely on the fact that it fits in a single page. */
3468 BUILD_BUG_ON((sizeof(*st) - 1) & KVM_STEAL_VALID_BITS);
3469
Paolo Bonzini901d3762022-08-04 15:28:32 +02003470 if (kvm_gfn_to_hva_cache_init(vcpu->kvm, ghc, gpa, sizeof(*st)) ||
David Woodhouse7e2175e2021-11-02 17:36:39 +00003471 kvm_is_error_hva(ghc->hva) || !ghc->memslot)
3472 return;
3473 }
3474
3475 st = (struct kvm_steal_time __user *)ghc->hva;
Wanpeng Lif38a7b72017-12-12 17:33:04 -08003476 /*
3477 * Doing a TLB flush here, on the guest's behalf, can avoid
3478 * expensive IPIs.
3479 */
Oliver Upton66570e962020-08-18 15:24:28 +00003480 if (guest_pv_has(vcpu, KVM_FEATURE_PV_TLB_FLUSH)) {
David Woodhouse7e2175e2021-11-02 17:36:39 +00003481 u8 st_preempted = 0;
3482 int err = -EFAULT;
3483
Paolo Bonzini3e067fd2021-11-12 02:53:41 -05003484 if (!user_access_begin(st, sizeof(*st)))
3485 return;
3486
David Woodhouse7e2175e2021-11-02 17:36:39 +00003487 asm volatile("1: xchgb %0, %2\n"
3488 "xor %1, %1\n"
3489 "2:\n"
3490 _ASM_EXTABLE_UA(1b, 2b)
David Woodhouse964b7aa2021-11-14 08:59:02 +00003491 : "+q" (st_preempted),
3492 "+&r" (err),
3493 "+m" (st->preempted));
David Woodhouse7e2175e2021-11-02 17:36:39 +00003494 if (err)
3495 goto out;
3496
3497 user_access_end();
3498
3499 vcpu->arch.st.preempted = 0;
Lai Jiangshanaf3511f2021-06-01 01:46:28 +08003500
Oliver Upton66570e962020-08-18 15:24:28 +00003501 trace_kvm_pv_tlb_flush(vcpu->vcpu_id,
Lai Jiangshanaf3511f2021-06-01 01:46:28 +08003502 st_preempted & KVM_VCPU_FLUSH_TLB);
3503 if (st_preempted & KVM_VCPU_FLUSH_TLB)
Oliver Upton66570e962020-08-18 15:24:28 +00003504 kvm_vcpu_flush_tlb_guest(vcpu);
David Woodhouse7e2175e2021-11-02 17:36:39 +00003505
3506 if (!user_access_begin(st, sizeof(*st)))
3507 goto dirty;
Wanpeng Li1eff0ad2021-05-18 05:00:33 -07003508 } else {
Paolo Bonzini3e067fd2021-11-12 02:53:41 -05003509 if (!user_access_begin(st, sizeof(*st)))
3510 return;
3511
David Woodhouse7e2175e2021-11-02 17:36:39 +00003512 unsafe_put_user(0, &st->preempted, out);
3513 vcpu->arch.st.preempted = 0;
Oliver Upton66570e962020-08-18 15:24:28 +00003514 }
Pan Xinhui0b9f6c42016-11-02 05:08:35 -04003515
David Woodhouse7e2175e2021-11-02 17:36:39 +00003516 unsafe_get_user(version, &st->version, out);
3517 if (version & 1)
3518 version += 1; /* first time write, random junk */
Wanpeng Li35f3fae12016-05-03 11:43:10 +08003519
David Woodhouse7e2175e2021-11-02 17:36:39 +00003520 version += 1;
3521 unsafe_put_user(version, &st->version, out);
Wanpeng Li35f3fae12016-05-03 11:43:10 +08003522
3523 smp_wmb();
3524
David Woodhouse7e2175e2021-11-02 17:36:39 +00003525 unsafe_get_user(steal, &st->steal, out);
3526 steal += current->sched_info.run_delay -
Liang Chenc54cdf12016-03-16 19:33:16 +08003527 vcpu->arch.st.last_steal;
3528 vcpu->arch.st.last_steal = current->sched_info.run_delay;
David Woodhouse7e2175e2021-11-02 17:36:39 +00003529 unsafe_put_user(steal, &st->steal, out);
Wanpeng Li35f3fae12016-05-03 11:43:10 +08003530
David Woodhouse7e2175e2021-11-02 17:36:39 +00003531 version += 1;
3532 unsafe_put_user(version, &st->version, out);
Wanpeng Li35f3fae12016-05-03 11:43:10 +08003533
David Woodhouse7e2175e2021-11-02 17:36:39 +00003534 out:
3535 user_access_end();
3536 dirty:
3537 mark_page_dirty_in_slot(vcpu->kvm, ghc->memslot, gpa_to_gfn(ghc->gpa));
Glauber Costac9aaa892011-07-11 15:28:14 -04003538}
3539
Will Auld8fe8ab42012-11-29 12:42:12 -08003540int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
Carsten Otte15c4a642007-10-30 18:44:17 +01003541{
Gleb Natapov57537852012-01-15 14:17:22 +02003542 bool pr = false;
Will Auld8fe8ab42012-11-29 12:42:12 -08003543 u32 msr = msr_info->index;
3544 u64 data = msr_info->data;
Gleb Natapov57537852012-01-15 14:17:22 +02003545
Joao Martins1232f8e2018-06-13 06:10:37 -04003546 if (msr && msr == vcpu->kvm->arch.xen_hvm_config.msr)
Joao Martins23200b72018-06-13 09:55:44 -04003547 return kvm_xen_write_hypercall_page(vcpu, data);
Joao Martins1232f8e2018-06-13 06:10:37 -04003548
Carsten Otte15c4a642007-10-30 18:44:17 +01003549 switch (msr) {
Borislav Petkov2e32b712013-02-19 19:33:13 +01003550 case MSR_AMD64_NB_CFG:
Borislav Petkov2e32b712013-02-19 19:33:13 +01003551 case MSR_IA32_UCODE_WRITE:
3552 case MSR_VM_HSAVE_PA:
3553 case MSR_AMD64_PATCH_LOADER:
3554 case MSR_AMD64_BU_CFG2:
Ladi Prosek405a353a2017-04-06 15:22:20 +02003555 case MSR_AMD64_DC_CFG:
Eduardo Habkost0e1b8692018-12-17 22:34:18 -02003556 case MSR_F15H_EX_CFG:
Borislav Petkov2e32b712013-02-19 19:33:13 +01003557 break;
3558
Wanpeng Li518e7b92018-02-28 14:03:31 +08003559 case MSR_IA32_UCODE_REV:
3560 if (msr_info->host_initiated)
3561 vcpu->arch.microcode_version = data;
3562 break;
Sean Christopherson0cf91352019-03-07 15:43:02 -08003563 case MSR_IA32_ARCH_CAPABILITIES:
3564 if (!msr_info->host_initiated)
3565 return 1;
3566 vcpu->arch.arch_capabilities = data;
3567 break;
Vitaly Kuznetsovd574c532020-07-10 17:25:59 +02003568 case MSR_IA32_PERF_CAPABILITIES: {
3569 struct kvm_msr_entry msr_ent = {.index = msr, .data = 0};
3570
3571 if (!msr_info->host_initiated)
3572 return 1;
Vitaly Kuznetsov1aa2abb2021-12-16 17:52:13 +01003573 if (kvm_get_msr_feature(&msr_ent))
Vitaly Kuznetsovd574c532020-07-10 17:25:59 +02003574 return 1;
3575 if (data & ~msr_ent.data)
3576 return 1;
3577
3578 vcpu->arch.perf_capabilities = data;
Sean Christopherson17a024a2022-07-27 23:34:22 +00003579 kvm_pmu_refresh(vcpu);
Vitaly Kuznetsovd574c532020-07-10 17:25:59 +02003580 return 0;
Sean Christopherson17a024a2022-07-27 23:34:22 +00003581 }
Carsten Otte15c4a642007-10-30 18:44:17 +01003582 case MSR_EFER:
Sean Christopherson11988492019-04-02 08:19:15 -07003583 return set_efer(vcpu, msr_info);
Andre Przywara8f1589d2009-06-24 12:44:33 +02003584 case MSR_K7_HWCR:
3585 data &= ~(u64)0x40; /* ignore flush filter disable */
Joerg Roedel82494022010-02-24 18:59:16 +01003586 data &= ~(u64)0x100; /* ignore ignne emulation enable */
Nicolae Mogoreanua223c312012-02-21 13:44:21 -08003587 data &= ~(u64)0x8; /* ignore TLB cache disable */
Borislav Petkov191c8132019-04-18 18:32:50 +02003588
3589 /* Handle McStatusWrEn */
3590 if (data == BIT_ULL(18)) {
3591 vcpu->arch.msr_hwcr = data;
3592 } else if (data != 0) {
Christoffer Dalla737f252012-06-03 21:17:48 +03003593 vcpu_unimpl(vcpu, "unimplemented HWCR wrmsr: 0x%llx\n",
3594 data);
Andre Przywara8f1589d2009-06-24 12:44:33 +02003595 return 1;
3596 }
Carsten Otte15c4a642007-10-30 18:44:17 +01003597 break;
Andre Przywaraf7c6d142009-07-02 15:04:14 +02003598 case MSR_FAM10H_MMIO_CONF_BASE:
3599 if (data != 0) {
Christoffer Dalla737f252012-06-03 21:17:48 +03003600 vcpu_unimpl(vcpu, "unimplemented MMIO_CONF_BASE wrmsr: "
3601 "0x%llx\n", data);
Andre Przywaraf7c6d142009-07-02 15:04:14 +02003602 return 1;
3603 }
Carsten Otte15c4a642007-10-30 18:44:17 +01003604 break;
Jue Wang281b5272022-06-10 10:11:32 -07003605 case 0x200 ... MSR_IA32_MC0_CTL2 - 1:
3606 case MSR_IA32_MCx_CTL2(KVM_MAX_MCE_BANKS) ... 0x2ff:
Xiao Guangrongff536042015-06-15 16:55:22 +08003607 return kvm_mtrr_set_msr(vcpu, msr, data);
Carsten Otte15c4a642007-10-30 18:44:17 +01003608 case MSR_IA32_APICBASE:
Jan Kiszka58cb6282014-01-24 16:48:44 +01003609 return kvm_set_apic_base(vcpu, msr_info);
Xiaoyao Libf10bd02020-06-16 15:33:07 +08003610 case APIC_BASE_MSR ... APIC_BASE_MSR + 0xff:
Gleb Natapov0105d1a2009-07-05 17:39:36 +03003611 return kvm_x2apic_msr_write(vcpu, msr, data);
Dave Hansen09141ec2020-03-05 09:47:06 -08003612 case MSR_IA32_TSC_DEADLINE:
Liu, Jinsonga3e06bb2011-09-22 16:55:52 +08003613 kvm_set_lapic_tscdeadline_msr(vcpu, data);
3614 break;
Will Auldba904632012-11-29 12:42:50 -08003615 case MSR_IA32_TSC_ADJUST:
Radim Krčmářd6321d42017-08-05 00:12:49 +02003616 if (guest_cpuid_has(vcpu, X86_FEATURE_TSC_ADJUST)) {
Will Auldba904632012-11-29 12:42:50 -08003617 if (!msr_info->host_initiated) {
Chris J Argesd913b902014-11-12 21:00:39 -06003618 s64 adj = data - vcpu->arch.ia32_tsc_adjust_msr;
Haozhong Zhangd7add052015-08-07 11:24:32 +08003619 adjust_tsc_offset_guest(vcpu, adj);
Zelin Dengd9130a22021-04-28 10:22:01 +08003620 /* Before back to guest, tsc_timestamp must be adjusted
3621 * as well, otherwise guest's percpu pvclock time could jump.
3622 */
3623 kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
Will Auldba904632012-11-29 12:42:50 -08003624 }
3625 vcpu->arch.ia32_tsc_adjust_msr = data;
3626 }
3627 break;
Like Xubef6ecc2022-04-11 18:19:33 +08003628 case MSR_IA32_MISC_ENABLE: {
3629 u64 old_val = vcpu->arch.ia32_misc_enable_msr;
Like Xud10551732022-04-11 18:19:41 +08003630
Sean Christopherson9fc22292022-06-11 00:57:49 +00003631 if (!msr_info->host_initiated) {
3632 /* RO bits */
3633 if ((old_val ^ data) & MSR_IA32_MISC_ENABLE_PMU_RO_MASK)
3634 return 1;
Like Xubef6ecc2022-04-11 18:19:33 +08003635
Sean Christopherson9fc22292022-06-11 00:57:49 +00003636 /* R bits, i.e. writes are ignored, but don't fault. */
3637 data = data & ~MSR_IA32_MISC_ENABLE_EMON;
3638 data |= old_val & MSR_IA32_MISC_ENABLE_EMON;
3639 }
3640
Wanpeng Li511a85562019-05-21 14:06:54 +08003641 if (!kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_MISC_ENABLE_NO_MWAIT) &&
Like Xubef6ecc2022-04-11 18:19:33 +08003642 ((old_val ^ data) & MSR_IA32_MISC_ENABLE_MWAIT)) {
Wanpeng Li511a85562019-05-21 14:06:54 +08003643 if (!guest_cpuid_has(vcpu, X86_FEATURE_XMM3))
3644 return 1;
3645 vcpu->arch.ia32_misc_enable_msr = data;
Xiaoyao Liaedbaf42020-07-09 12:34:23 +08003646 kvm_update_cpuid_runtime(vcpu);
Wanpeng Li511a85562019-05-21 14:06:54 +08003647 } else {
3648 vcpu->arch.ia32_misc_enable_msr = data;
3649 }
Carsten Otte15c4a642007-10-30 18:44:17 +01003650 break;
Like Xubef6ecc2022-04-11 18:19:33 +08003651 }
Paolo Bonzini64d60672015-05-07 11:36:11 +02003652 case MSR_IA32_SMBASE:
3653 if (!msr_info->host_initiated)
3654 return 1;
3655 vcpu->arch.smbase = data;
3656 break;
Paolo Bonzini73f624f2019-06-06 14:32:59 +02003657 case MSR_IA32_POWER_CTL:
3658 vcpu->arch.msr_ia32_power_ctl = data;
3659 break;
Paolo Bonzinidd259932018-04-13 11:38:35 +02003660 case MSR_IA32_TSC:
Paolo Bonzini0c899c22020-09-24 14:45:27 +02003661 if (msr_info->host_initiated) {
3662 kvm_synchronize_tsc(vcpu, data);
3663 } else {
Ilias Stamatis9b399df2021-05-26 19:44:10 +01003664 u64 adj = kvm_compute_l1_tsc_offset(vcpu, data) - vcpu->arch.l1_tsc_offset;
Paolo Bonzini0c899c22020-09-24 14:45:27 +02003665 adjust_tsc_offset_guest(vcpu, adj);
3666 vcpu->arch.ia32_tsc_adjust_msr += adj;
3667 }
Paolo Bonzinidd259932018-04-13 11:38:35 +02003668 break;
Aaron Lewis864e2ab2019-10-21 16:30:26 -07003669 case MSR_IA32_XSS:
3670 if (!msr_info->host_initiated &&
3671 !guest_cpuid_has(vcpu, X86_FEATURE_XSAVES))
3672 return 1;
3673 /*
Sean Christophersona1bead22020-03-02 15:57:00 -08003674 * KVM supports exposing PT to the guest, but does not support
3675 * IA32_XSS[bit 8]. Guests have to use RDMSR/WRMSR rather than
3676 * XSAVES/XRSTORS to save/restore PT MSRs.
Aaron Lewis864e2ab2019-10-21 16:30:26 -07003677 */
Sean Christopherson938c8742022-05-24 21:56:23 +08003678 if (data & ~kvm_caps.supported_xss)
Aaron Lewis864e2ab2019-10-21 16:30:26 -07003679 return 1;
3680 vcpu->arch.ia32_xss = data;
Like Xu4c282e52022-01-26 17:22:25 +00003681 kvm_update_cpuid_runtime(vcpu);
Aaron Lewis864e2ab2019-10-21 16:30:26 -07003682 break;
Liran Alon52797bf2017-11-15 13:43:14 +02003683 case MSR_SMI_COUNT:
3684 if (!msr_info->host_initiated)
3685 return 1;
3686 vcpu->arch.smi_count = data;
3687 break;
Glauber Costa11c6bff2010-05-11 12:17:41 -04003688 case MSR_KVM_WALL_CLOCK_NEW:
Oliver Upton66570e962020-08-18 15:24:28 +00003689 if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE2))
3690 return 1;
3691
Joao Martins629b5342018-06-28 15:06:43 -04003692 vcpu->kvm->arch.wall_clock = data;
3693 kvm_write_wall_clock(vcpu->kvm, data, 0);
Oliver Upton66570e962020-08-18 15:24:28 +00003694 break;
Glauber de Oliveira Costa18068522008-02-15 17:52:47 -02003695 case MSR_KVM_WALL_CLOCK:
Oliver Upton66570e962020-08-18 15:24:28 +00003696 if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE))
3697 return 1;
3698
Joao Martins629b5342018-06-28 15:06:43 -04003699 vcpu->kvm->arch.wall_clock = data;
3700 kvm_write_wall_clock(vcpu->kvm, data, 0);
Glauber de Oliveira Costa18068522008-02-15 17:52:47 -02003701 break;
Glauber Costa11c6bff2010-05-11 12:17:41 -04003702 case MSR_KVM_SYSTEM_TIME_NEW:
Oliver Upton66570e962020-08-18 15:24:28 +00003703 if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE2))
3704 return 1;
3705
Oliver Upton5b9bb0e2020-08-18 15:24:26 +00003706 kvm_write_system_time(vcpu, data, false, msr_info->host_initiated);
Glauber de Oliveira Costa18068522008-02-15 17:52:47 -02003707 break;
Oliver Upton5b9bb0e2020-08-18 15:24:26 +00003708 case MSR_KVM_SYSTEM_TIME:
Oliver Upton66570e962020-08-18 15:24:28 +00003709 if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE))
3710 return 1;
3711
3712 kvm_write_system_time(vcpu, data, true, msr_info->host_initiated);
Oliver Upton5b9bb0e2020-08-18 15:24:26 +00003713 break;
Gleb Natapov344d9582010-10-14 11:22:50 +02003714 case MSR_KVM_ASYNC_PF_EN:
Oliver Upton66570e962020-08-18 15:24:28 +00003715 if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF))
3716 return 1;
3717
Gleb Natapov344d9582010-10-14 11:22:50 +02003718 if (kvm_pv_enable_async_pf(vcpu, data))
3719 return 1;
3720 break;
Vitaly Kuznetsov2635b5c2020-05-25 16:41:20 +02003721 case MSR_KVM_ASYNC_PF_INT:
Oliver Upton66570e962020-08-18 15:24:28 +00003722 if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_INT))
3723 return 1;
3724
Vitaly Kuznetsov2635b5c2020-05-25 16:41:20 +02003725 if (kvm_pv_enable_async_pf_int(vcpu, data))
3726 return 1;
3727 break;
Vitaly Kuznetsov557a9612020-05-25 16:41:21 +02003728 case MSR_KVM_ASYNC_PF_ACK:
Vitaly Kuznetsov0a31df62021-07-22 14:30:18 +02003729 if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_INT))
Oliver Upton66570e962020-08-18 15:24:28 +00003730 return 1;
Vitaly Kuznetsov557a9612020-05-25 16:41:21 +02003731 if (data & 0x1) {
3732 vcpu->arch.apf.pageready_pending = false;
3733 kvm_check_async_pf_completion(vcpu);
3734 }
3735 break;
Glauber Costac9aaa892011-07-11 15:28:14 -04003736 case MSR_KVM_STEAL_TIME:
Oliver Upton66570e962020-08-18 15:24:28 +00003737 if (!guest_pv_has(vcpu, KVM_FEATURE_STEAL_TIME))
3738 return 1;
Glauber Costac9aaa892011-07-11 15:28:14 -04003739
3740 if (unlikely(!sched_info_on()))
3741 return 1;
3742
3743 if (data & KVM_STEAL_RESERVED_MASK)
3744 return 1;
3745
Glauber Costac9aaa892011-07-11 15:28:14 -04003746 vcpu->arch.st.msr_val = data;
3747
3748 if (!(data & KVM_MSR_ENABLED))
3749 break;
3750
Glauber Costac9aaa892011-07-11 15:28:14 -04003751 kvm_make_request(KVM_REQ_STEAL_UPDATE, vcpu);
3752
3753 break;
Michael S. Tsirkinae7a2a32012-06-24 19:25:07 +03003754 case MSR_KVM_PV_EOI_EN:
Oliver Upton66570e962020-08-18 15:24:28 +00003755 if (!guest_pv_has(vcpu, KVM_FEATURE_PV_EOI))
3756 return 1;
3757
Vitaly Kuznetsov77c33232021-11-08 16:28:18 +01003758 if (kvm_lapic_set_pv_eoi(vcpu, data, sizeof(u8)))
Michael S. Tsirkinae7a2a32012-06-24 19:25:07 +03003759 return 1;
3760 break;
Glauber Costac9aaa892011-07-11 15:28:14 -04003761
Marcelo Tosatti2d5ba192019-06-03 19:52:44 -03003762 case MSR_KVM_POLL_CONTROL:
Oliver Upton66570e962020-08-18 15:24:28 +00003763 if (!guest_pv_has(vcpu, KVM_FEATURE_POLL_CONTROL))
3764 return 1;
3765
Marcelo Tosatti2d5ba192019-06-03 19:52:44 -03003766 /* only enable bit supported */
3767 if (data & (-1ULL << 1))
3768 return 1;
3769
3770 vcpu->arch.msr_kvm_poll_control = data;
3771 break;
3772
Huang Ying890ca9a2009-05-11 16:48:15 +08003773 case MSR_IA32_MCG_CTL:
3774 case MSR_IA32_MCG_STATUS:
Chen Yucong81760dc2014-09-23 10:44:35 +08003775 case MSR_IA32_MC0_CTL ... MSR_IA32_MCx_CTL(KVM_MAX_MCE_BANKS) - 1:
Jue Wang281b5272022-06-10 10:11:32 -07003776 case MSR_IA32_MC0_CTL2 ... MSR_IA32_MCx_CTL2(KVM_MAX_MCE_BANKS) - 1:
Wanpeng Li9ffd9862017-10-19 06:47:56 -07003777 return set_msr_mce(vcpu, msr_info);
Andre Przywara71db6022009-06-12 22:01:29 +02003778
Wei Huang6912ac32015-06-12 01:34:56 -04003779 case MSR_K7_PERFCTR0 ... MSR_K7_PERFCTR3:
3780 case MSR_P6_PERFCTR0 ... MSR_P6_PERFCTR1:
Gustavo A. R. Silvadf561f662020-08-23 17:36:59 -05003781 pr = true;
3782 fallthrough;
Wei Huang6912ac32015-06-12 01:34:56 -04003783 case MSR_K7_EVNTSEL0 ... MSR_K7_EVNTSEL3:
3784 case MSR_P6_EVNTSEL0 ... MSR_P6_EVNTSEL1:
Wei Huangc6702c92015-06-19 13:44:45 +02003785 if (kvm_pmu_is_valid_msr(vcpu, msr))
Paolo Bonziniafd80d82013-03-28 17:18:35 +01003786 return kvm_pmu_set_msr(vcpu, msr_info);
Gleb Natapov57537852012-01-15 14:17:22 +02003787
3788 if (pr || data != 0)
Christoffer Dalla737f252012-06-03 21:17:48 +03003789 vcpu_unimpl(vcpu, "disabled perfctr wrmsr: "
3790 "0x%x data 0x%llx\n", msr, data);
Gleb Natapov57537852012-01-15 14:17:22 +02003791 break;
Jes Sorensen84e0cef2010-09-01 11:42:04 +02003792 case MSR_K7_CLK_CTL:
3793 /*
3794 * Ignore all writes to this no longer documented MSR.
3795 * Writes are only relevant for old K7 processors,
3796 * all pre-dating SVM, but a recommended workaround from
Guo Chao4a969982012-06-28 15:17:27 +08003797 * AMD for these chips. It is possible to specify the
Jes Sorensen84e0cef2010-09-01 11:42:04 +02003798 * affected processor models on the command line, hence
3799 * the need to ignore the workaround.
3800 */
3801 break;
Gleb Natapov55cd8e52010-01-17 15:51:22 +02003802 case HV_X64_MSR_GUEST_OS_ID ... HV_X64_MSR_SINT15:
Jon Doronf97f5a52020-05-29 16:45:40 +03003803 case HV_X64_MSR_SYNDBG_CONTROL ... HV_X64_MSR_SYNDBG_PENDING_BUFFER:
3804 case HV_X64_MSR_SYNDBG_OPTIONS:
Andrey Smetanine7d95132015-07-03 15:01:37 +03003805 case HV_X64_MSR_CRASH_P0 ... HV_X64_MSR_CRASH_P4:
3806 case HV_X64_MSR_CRASH_CTL:
Andrey Smetanin1f4b34f2015-11-30 19:22:21 +03003807 case HV_X64_MSR_STIMER0_CONFIG ... HV_X64_MSR_STIMER3_COUNT:
Vitaly Kuznetsova2e164e2018-03-01 15:15:12 +01003808 case HV_X64_MSR_REENLIGHTENMENT_CONTROL:
3809 case HV_X64_MSR_TSC_EMULATION_CONTROL:
3810 case HV_X64_MSR_TSC_EMULATION_STATUS:
Andrey Smetanine7d95132015-07-03 15:01:37 +03003811 return kvm_hv_set_msr_common(vcpu, msr, data,
3812 msr_info->host_initiated);
john cooper91c9c3e2011-01-21 00:21:00 -05003813 case MSR_IA32_BBL_CR_CTL3:
3814 /* Drop writes to this legacy MSR -- see rdmsr
3815 * counterpart for further detail.
3816 */
Eyal Moscovicifab0aa32017-11-08 14:32:08 +02003817 if (report_ignored_msrs)
3818 vcpu_unimpl(vcpu, "ignored wrmsr: 0x%x data 0x%llx\n",
3819 msr, data);
john cooper91c9c3e2011-01-21 00:21:00 -05003820 break;
Boris Ostrovsky2b036c62012-01-09 14:00:35 -05003821 case MSR_AMD64_OSVW_ID_LENGTH:
Radim Krčmářd6321d42017-08-05 00:12:49 +02003822 if (!guest_cpuid_has(vcpu, X86_FEATURE_OSVW))
Boris Ostrovsky2b036c62012-01-09 14:00:35 -05003823 return 1;
3824 vcpu->arch.osvw.length = data;
3825 break;
3826 case MSR_AMD64_OSVW_STATUS:
Radim Krčmářd6321d42017-08-05 00:12:49 +02003827 if (!guest_cpuid_has(vcpu, X86_FEATURE_OSVW))
Boris Ostrovsky2b036c62012-01-09 14:00:35 -05003828 return 1;
3829 vcpu->arch.osvw.status = data;
3830 break;
Kyle Hueydb2336a2017-03-20 01:16:28 -07003831 case MSR_PLATFORM_INFO:
3832 if (!msr_info->host_initiated ||
Kyle Hueydb2336a2017-03-20 01:16:28 -07003833 (!(data & MSR_PLATFORM_INFO_CPUID_FAULT) &&
3834 cpuid_fault_enabled(vcpu)))
3835 return 1;
3836 vcpu->arch.msr_platform_info = data;
3837 break;
3838 case MSR_MISC_FEATURES_ENABLES:
3839 if (data & ~MSR_MISC_FEATURES_ENABLES_CPUID_FAULT ||
3840 (data & MSR_MISC_FEATURES_ENABLES_CPUID_FAULT &&
3841 !supports_cpuid_fault(vcpu)))
3842 return 1;
3843 vcpu->arch.msr_misc_features_enables = data;
3844 break;
Jing Liu820a6ee2022-01-05 04:35:21 -08003845#ifdef CONFIG_X86_64
3846 case MSR_IA32_XFD:
3847 if (!msr_info->host_initiated &&
3848 !guest_cpuid_has(vcpu, X86_FEATURE_XFD))
3849 return 1;
3850
Leonardo Bras988896b2022-02-17 02:30:30 -03003851 if (data & ~kvm_guest_supported_xfd(vcpu))
Jing Liu820a6ee2022-01-05 04:35:21 -08003852 return 1;
3853
3854 fpu_update_guest_xfd(&vcpu->arch.guest_fpu, data);
3855 break;
Jing Liu548e8362022-01-05 04:35:24 -08003856 case MSR_IA32_XFD_ERR:
3857 if (!msr_info->host_initiated &&
3858 !guest_cpuid_has(vcpu, X86_FEATURE_XFD))
3859 return 1;
3860
Leonardo Bras988896b2022-02-17 02:30:30 -03003861 if (data & ~kvm_guest_supported_xfd(vcpu))
Jing Liu548e8362022-01-05 04:35:24 -08003862 return 1;
3863
3864 vcpu->arch.guest_fpu.xfd_err = data;
3865 break;
Jing Liu820a6ee2022-01-05 04:35:21 -08003866#endif
Sean Christopherson157fc492022-06-11 00:57:54 +00003867 case MSR_IA32_PEBS_ENABLE:
3868 case MSR_IA32_DS_AREA:
3869 case MSR_PEBS_DATA_CFG:
Sean Christophersonff81a902022-06-11 00:57:55 +00003870 case MSR_F15H_PERF_CTL0 ... MSR_F15H_PERF_CTR5:
Sean Christopherson157fc492022-06-11 00:57:54 +00003871 if (kvm_pmu_is_valid_msr(vcpu, msr))
3872 return kvm_pmu_set_msr(vcpu, msr_info);
3873 /*
3874 * Userspace is allowed to write '0' to MSRs that KVM reports
3875 * as to-be-saved, even if an MSRs isn't fully supported.
3876 */
3877 return !msr_info->host_initiated || data;
Carsten Otte15c4a642007-10-30 18:44:17 +01003878 default:
Wei Huangc6702c92015-06-19 13:44:45 +02003879 if (kvm_pmu_is_valid_msr(vcpu, msr))
Paolo Bonziniafd80d82013-03-28 17:18:35 +01003880 return kvm_pmu_set_msr(vcpu, msr_info);
Peter Xu6abe9c12020-06-22 18:04:41 -04003881 return KVM_MSR_RET_INVALID;
Carsten Otte15c4a642007-10-30 18:44:17 +01003882 }
3883 return 0;
3884}
3885EXPORT_SYMBOL_GPL(kvm_set_msr_common);
3886
Paolo Bonzini44883f02018-07-26 13:01:52 +02003887static int get_msr_mce(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata, bool host)
Huang Ying890ca9a2009-05-11 16:48:15 +08003888{
3889 u64 data;
3890 u64 mcg_cap = vcpu->arch.mcg_cap;
3891 unsigned bank_num = mcg_cap & 0xff;
Jue Wang281b5272022-06-10 10:11:32 -07003892 u32 offset, last_msr;
Huang Ying890ca9a2009-05-11 16:48:15 +08003893
3894 switch (msr) {
3895 case MSR_IA32_P5_MC_ADDR:
3896 case MSR_IA32_P5_MC_TYPE:
3897 data = 0;
3898 break;
3899 case MSR_IA32_MCG_CAP:
3900 data = vcpu->arch.mcg_cap;
3901 break;
3902 case MSR_IA32_MCG_CTL:
Paolo Bonzini44883f02018-07-26 13:01:52 +02003903 if (!(mcg_cap & MCG_CTL_P) && !host)
Huang Ying890ca9a2009-05-11 16:48:15 +08003904 return 1;
3905 data = vcpu->arch.mcg_ctl;
3906 break;
3907 case MSR_IA32_MCG_STATUS:
3908 data = vcpu->arch.mcg_status;
3909 break;
Jue Wang281b5272022-06-10 10:11:32 -07003910 case MSR_IA32_MC0_CTL2 ... MSR_IA32_MCx_CTL2(KVM_MAX_MCE_BANKS) - 1:
3911 last_msr = MSR_IA32_MCx_CTL2(bank_num) - 1;
3912 if (msr > last_msr)
3913 return 1;
Marios Pomonis6ec4c5e2019-12-11 12:47:49 -08003914
Jue Wang281b5272022-06-10 10:11:32 -07003915 if (!(mcg_cap & MCG_CMCI_P) && !host)
3916 return 1;
3917 offset = array_index_nospec(msr - MSR_IA32_MC0_CTL2,
3918 last_msr + 1 - MSR_IA32_MC0_CTL2);
3919 data = vcpu->arch.mci_ctl2_banks[offset];
3920 break;
3921 case MSR_IA32_MC0_CTL ... MSR_IA32_MCx_CTL(KVM_MAX_MCE_BANKS) - 1:
3922 last_msr = MSR_IA32_MCx_CTL(bank_num) - 1;
3923 if (msr > last_msr)
3924 return 1;
3925
3926 offset = array_index_nospec(msr - MSR_IA32_MC0_CTL,
3927 last_msr + 1 - MSR_IA32_MC0_CTL);
3928 data = vcpu->arch.mce_banks[offset];
3929 break;
3930 default:
Huang Ying890ca9a2009-05-11 16:48:15 +08003931 return 1;
3932 }
3933 *pdata = data;
3934 return 0;
3935}
3936
Paolo Bonzini609e36d2015-04-08 15:30:38 +02003937int kvm_get_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
Carsten Otte15c4a642007-10-30 18:44:17 +01003938{
Paolo Bonzini609e36d2015-04-08 15:30:38 +02003939 switch (msr_info->index) {
Carsten Otte15c4a642007-10-30 18:44:17 +01003940 case MSR_IA32_PLATFORM_ID:
Carsten Otte15c4a642007-10-30 18:44:17 +01003941 case MSR_IA32_EBL_CR_POWERON:
Alexander Grafb5e2fec2008-07-22 08:00:45 +02003942 case MSR_IA32_LASTBRANCHFROMIP:
3943 case MSR_IA32_LASTBRANCHTOIP:
3944 case MSR_IA32_LASTINTFROMIP:
3945 case MSR_IA32_LASTINTTOIP:
Brijesh Singh059e5c32021-04-27 06:16:36 -05003946 case MSR_AMD64_SYSCFG:
Paolo Bonzini3afb1122015-09-18 17:33:04 +02003947 case MSR_K8_TSEG_ADDR:
3948 case MSR_K8_TSEG_MASK:
Avi Kivity61a6bd62008-12-29 17:32:28 +02003949 case MSR_VM_HSAVE_PA:
Andre Przywara1fdbd482009-06-24 12:44:34 +02003950 case MSR_K8_INT_PENDING_MSG:
Andre Przywarac323c0e2009-06-24 15:37:05 +02003951 case MSR_AMD64_NB_CFG:
Andre Przywaraf7c6d142009-07-02 15:04:14 +02003952 case MSR_FAM10H_MMIO_CONF_BASE:
Borislav Petkov2e32b712013-02-19 19:33:13 +01003953 case MSR_AMD64_BU_CFG2:
Dmitry Bilunov0c2df2a2016-05-31 17:38:24 +03003954 case MSR_IA32_PERF_CTL:
Ladi Prosek405a353a2017-04-06 15:22:20 +02003955 case MSR_AMD64_DC_CFG:
Eduardo Habkost0e1b8692018-12-17 22:34:18 -02003956 case MSR_F15H_EX_CFG:
Venkatesh Srinivas2ca1a062020-04-16 11:42:54 -07003957 /*
3958 * Intel Sandy Bridge CPUs must support the RAPL (running average power
3959 * limit) MSRs. Just return 0, as we do not want to expose the host
3960 * data here. Do not conditionalize this on CPUID, as KVM does not do
3961 * so for existing CPU-specific MSRs.
3962 */
3963 case MSR_RAPL_POWER_UNIT:
3964 case MSR_PP0_ENERGY_STATUS: /* Power plane 0 (core) */
3965 case MSR_PP1_ENERGY_STATUS: /* Power plane 1 (graphics uncore) */
3966 case MSR_PKG_ENERGY_STATUS: /* Total package */
3967 case MSR_DRAM_ENERGY_STATUS: /* DRAM controller */
Paolo Bonzini609e36d2015-04-08 15:30:38 +02003968 msr_info->data = 0;
Carsten Otte15c4a642007-10-30 18:44:17 +01003969 break;
Sean Christopherson157fc492022-06-11 00:57:54 +00003970 case MSR_IA32_PEBS_ENABLE:
3971 case MSR_IA32_DS_AREA:
3972 case MSR_PEBS_DATA_CFG:
Janakarajan Natarajanc51eb522018-02-05 13:24:52 -06003973 case MSR_F15H_PERF_CTL0 ... MSR_F15H_PERF_CTR5:
Vitaly Kuznetsovc28fa562021-03-29 14:48:04 +02003974 if (kvm_pmu_is_valid_msr(vcpu, msr_info->index))
3975 return kvm_pmu_get_msr(vcpu, msr_info);
Sean Christopherson157fc492022-06-11 00:57:54 +00003976 /*
Sean Christophersonbfbcc812022-06-08 22:45:12 +00003977 * Userspace is allowed to read MSRs that KVM reports as
3978 * to-be-saved, even if an MSR isn't fully supported.
Sean Christopherson157fc492022-06-11 00:57:54 +00003979 */
Vitaly Kuznetsovc28fa562021-03-29 14:48:04 +02003980 if (!msr_info->host_initiated)
3981 return 1;
3982 msr_info->data = 0;
3983 break;
Wei Huang6912ac32015-06-12 01:34:56 -04003984 case MSR_K7_EVNTSEL0 ... MSR_K7_EVNTSEL3:
3985 case MSR_K7_PERFCTR0 ... MSR_K7_PERFCTR3:
3986 case MSR_P6_PERFCTR0 ... MSR_P6_PERFCTR1:
3987 case MSR_P6_EVNTSEL0 ... MSR_P6_EVNTSEL1:
Wei Huangc6702c92015-06-19 13:44:45 +02003988 if (kvm_pmu_is_valid_msr(vcpu, msr_info->index))
Wei Wangcbd71752020-05-29 15:43:44 +08003989 return kvm_pmu_get_msr(vcpu, msr_info);
Paolo Bonzini609e36d2015-04-08 15:30:38 +02003990 msr_info->data = 0;
Gleb Natapov57537852012-01-15 14:17:22 +02003991 break;
Marcelo Tosatti742bc672011-07-29 19:44:21 -03003992 case MSR_IA32_UCODE_REV:
Wanpeng Li518e7b92018-02-28 14:03:31 +08003993 msr_info->data = vcpu->arch.microcode_version;
Marcelo Tosatti742bc672011-07-29 19:44:21 -03003994 break;
Sean Christopherson0cf91352019-03-07 15:43:02 -08003995 case MSR_IA32_ARCH_CAPABILITIES:
3996 if (!msr_info->host_initiated &&
3997 !guest_cpuid_has(vcpu, X86_FEATURE_ARCH_CAPABILITIES))
3998 return 1;
3999 msr_info->data = vcpu->arch.arch_capabilities;
4000 break;
Vitaly Kuznetsovd574c532020-07-10 17:25:59 +02004001 case MSR_IA32_PERF_CAPABILITIES:
4002 if (!msr_info->host_initiated &&
4003 !guest_cpuid_has(vcpu, X86_FEATURE_PDCM))
4004 return 1;
4005 msr_info->data = vcpu->arch.perf_capabilities;
4006 break;
Paolo Bonzini73f624f2019-06-06 14:32:59 +02004007 case MSR_IA32_POWER_CTL:
4008 msr_info->data = vcpu->arch.msr_ia32_power_ctl;
4009 break;
Maxim Levitskycc5b54d2020-09-21 13:38:05 +03004010 case MSR_IA32_TSC: {
4011 /*
4012 * Intel SDM states that MSR_IA32_TSC read adds the TSC offset
4013 * even when not intercepted. AMD manual doesn't explicitly
4014 * state this but appears to behave the same.
4015 *
Maxim Levitskyee6fa052020-09-21 13:38:05 +03004016 * On userspace reads and writes, however, we unconditionally
Paolo Bonzinic0623f52020-10-21 18:05:58 -04004017 * return L1's TSC value to ensure backwards-compatible
Maxim Levitskyee6fa052020-09-21 13:38:05 +03004018 * behavior for migration.
Maxim Levitskycc5b54d2020-09-21 13:38:05 +03004019 */
Ilias Stamatisfe3eb502021-05-26 19:44:11 +01004020 u64 offset, ratio;
Maxim Levitskycc5b54d2020-09-21 13:38:05 +03004021
Ilias Stamatisfe3eb502021-05-26 19:44:11 +01004022 if (msr_info->host_initiated) {
4023 offset = vcpu->arch.l1_tsc_offset;
4024 ratio = vcpu->arch.l1_tsc_scaling_ratio;
4025 } else {
4026 offset = vcpu->arch.tsc_offset;
4027 ratio = vcpu->arch.tsc_scaling_ratio;
4028 }
4029
Jinrong Liang62711e52022-01-25 17:59:07 +08004030 msr_info->data = kvm_scale_tsc(rdtsc(), ratio) + offset;
Paolo Bonzinidd259932018-04-13 11:38:35 +02004031 break;
Maxim Levitskycc5b54d2020-09-21 13:38:05 +03004032 }
Avi Kivity9ba075a2008-05-26 20:06:35 +03004033 case MSR_MTRRcap:
Jue Wang281b5272022-06-10 10:11:32 -07004034 case 0x200 ... MSR_IA32_MC0_CTL2 - 1:
4035 case MSR_IA32_MCx_CTL2(KVM_MAX_MCE_BANKS) ... 0x2ff:
Xiao Guangrongff536042015-06-15 16:55:22 +08004036 return kvm_mtrr_get_msr(vcpu, msr_info->index, &msr_info->data);
Carsten Otte15c4a642007-10-30 18:44:17 +01004037 case 0xcd: /* fsb frequency */
Paolo Bonzini609e36d2015-04-08 15:30:38 +02004038 msr_info->data = 3;
Carsten Otte15c4a642007-10-30 18:44:17 +01004039 break;
Jes Sorensen7b914092010-09-09 12:06:46 +02004040 /*
4041 * MSR_EBC_FREQUENCY_ID
4042 * Conservative value valid for even the basic CPU models.
4043 * Models 0,1: 000 in bits 23:21 indicating a bus speed of
4044 * 100MHz, model 2 000 in bits 18:16 indicating 100MHz,
4045 * and 266MHz for model 3, or 4. Set Core Clock
4046 * Frequency to System Bus Frequency Ratio to 1 (bits
4047 * 31:24) even though these are only valid for CPU
4048 * models > 2, however guests may end up dividing or
4049 * multiplying by zero otherwise.
4050 */
4051 case MSR_EBC_FREQUENCY_ID:
Paolo Bonzini609e36d2015-04-08 15:30:38 +02004052 msr_info->data = 1 << 24;
Jes Sorensen7b914092010-09-09 12:06:46 +02004053 break;
Carsten Otte15c4a642007-10-30 18:44:17 +01004054 case MSR_IA32_APICBASE:
Paolo Bonzini609e36d2015-04-08 15:30:38 +02004055 msr_info->data = kvm_get_apic_base(vcpu);
Carsten Otte15c4a642007-10-30 18:44:17 +01004056 break;
Xiaoyao Libf10bd02020-06-16 15:33:07 +08004057 case APIC_BASE_MSR ... APIC_BASE_MSR + 0xff:
Paolo Bonzini609e36d2015-04-08 15:30:38 +02004058 return kvm_x2apic_msr_read(vcpu, msr_info->index, &msr_info->data);
Dave Hansen09141ec2020-03-05 09:47:06 -08004059 case MSR_IA32_TSC_DEADLINE:
Paolo Bonzini609e36d2015-04-08 15:30:38 +02004060 msr_info->data = kvm_get_lapic_tscdeadline_msr(vcpu);
Liu, Jinsonga3e06bb2011-09-22 16:55:52 +08004061 break;
Will Auldba904632012-11-29 12:42:50 -08004062 case MSR_IA32_TSC_ADJUST:
Paolo Bonzini609e36d2015-04-08 15:30:38 +02004063 msr_info->data = (u64)vcpu->arch.ia32_tsc_adjust_msr;
Will Auldba904632012-11-29 12:42:50 -08004064 break;
Carsten Otte15c4a642007-10-30 18:44:17 +01004065 case MSR_IA32_MISC_ENABLE:
Paolo Bonzini609e36d2015-04-08 15:30:38 +02004066 msr_info->data = vcpu->arch.ia32_misc_enable_msr;
Carsten Otte15c4a642007-10-30 18:44:17 +01004067 break;
Paolo Bonzini64d60672015-05-07 11:36:11 +02004068 case MSR_IA32_SMBASE:
4069 if (!msr_info->host_initiated)
4070 return 1;
4071 msr_info->data = vcpu->arch.smbase;
Carsten Otte15c4a642007-10-30 18:44:17 +01004072 break;
Liran Alon52797bf2017-11-15 13:43:14 +02004073 case MSR_SMI_COUNT:
4074 msr_info->data = vcpu->arch.smi_count;
4075 break;
Alexander Graf847f0ad2008-02-21 12:11:01 +01004076 case MSR_IA32_PERF_STATUS:
4077 /* TSC increment by tick */
Paolo Bonzini609e36d2015-04-08 15:30:38 +02004078 msr_info->data = 1000ULL;
Alexander Graf847f0ad2008-02-21 12:11:01 +01004079 /* CPU multiplier */
Nicolas Ioossb0996ae2015-06-29 18:39:23 +08004080 msr_info->data |= (((uint64_t)4ULL) << 40);
Alexander Graf847f0ad2008-02-21 12:11:01 +01004081 break;
Carsten Otte15c4a642007-10-30 18:44:17 +01004082 case MSR_EFER:
Paolo Bonzini609e36d2015-04-08 15:30:38 +02004083 msr_info->data = vcpu->arch.efer;
Carsten Otte15c4a642007-10-30 18:44:17 +01004084 break;
Glauber de Oliveira Costa18068522008-02-15 17:52:47 -02004085 case MSR_KVM_WALL_CLOCK:
Oliver Upton1930e5d2020-10-27 16:10:41 -07004086 if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE))
4087 return 1;
4088
4089 msr_info->data = vcpu->kvm->arch.wall_clock;
4090 break;
Glauber Costa11c6bff2010-05-11 12:17:41 -04004091 case MSR_KVM_WALL_CLOCK_NEW:
Oliver Upton1930e5d2020-10-27 16:10:41 -07004092 if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE2))
4093 return 1;
4094
Paolo Bonzini609e36d2015-04-08 15:30:38 +02004095 msr_info->data = vcpu->kvm->arch.wall_clock;
Glauber de Oliveira Costa18068522008-02-15 17:52:47 -02004096 break;
4097 case MSR_KVM_SYSTEM_TIME:
Oliver Upton1930e5d2020-10-27 16:10:41 -07004098 if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE))
4099 return 1;
4100
4101 msr_info->data = vcpu->arch.time;
4102 break;
Glauber Costa11c6bff2010-05-11 12:17:41 -04004103 case MSR_KVM_SYSTEM_TIME_NEW:
Oliver Upton1930e5d2020-10-27 16:10:41 -07004104 if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE2))
4105 return 1;
4106
Paolo Bonzini609e36d2015-04-08 15:30:38 +02004107 msr_info->data = vcpu->arch.time;
Glauber de Oliveira Costa18068522008-02-15 17:52:47 -02004108 break;
Gleb Natapov344d9582010-10-14 11:22:50 +02004109 case MSR_KVM_ASYNC_PF_EN:
Oliver Upton1930e5d2020-10-27 16:10:41 -07004110 if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF))
4111 return 1;
4112
Vitaly Kuznetsov2635b5c2020-05-25 16:41:20 +02004113 msr_info->data = vcpu->arch.apf.msr_en_val;
4114 break;
4115 case MSR_KVM_ASYNC_PF_INT:
Oliver Upton1930e5d2020-10-27 16:10:41 -07004116 if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_INT))
4117 return 1;
4118
Vitaly Kuznetsov2635b5c2020-05-25 16:41:20 +02004119 msr_info->data = vcpu->arch.apf.msr_int_val;
Gleb Natapov344d9582010-10-14 11:22:50 +02004120 break;
Vitaly Kuznetsov557a9612020-05-25 16:41:21 +02004121 case MSR_KVM_ASYNC_PF_ACK:
Vitaly Kuznetsov0a31df62021-07-22 14:30:18 +02004122 if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_INT))
Oliver Upton1930e5d2020-10-27 16:10:41 -07004123 return 1;
4124
Vitaly Kuznetsov557a9612020-05-25 16:41:21 +02004125 msr_info->data = 0;
4126 break;
Glauber Costac9aaa892011-07-11 15:28:14 -04004127 case MSR_KVM_STEAL_TIME:
Oliver Upton1930e5d2020-10-27 16:10:41 -07004128 if (!guest_pv_has(vcpu, KVM_FEATURE_STEAL_TIME))
4129 return 1;
4130
Paolo Bonzini609e36d2015-04-08 15:30:38 +02004131 msr_info->data = vcpu->arch.st.msr_val;
Glauber Costac9aaa892011-07-11 15:28:14 -04004132 break;
Michael S. Tsirkin1d921282012-08-26 18:00:29 +03004133 case MSR_KVM_PV_EOI_EN:
Oliver Upton1930e5d2020-10-27 16:10:41 -07004134 if (!guest_pv_has(vcpu, KVM_FEATURE_PV_EOI))
4135 return 1;
4136
Paolo Bonzini609e36d2015-04-08 15:30:38 +02004137 msr_info->data = vcpu->arch.pv_eoi.msr_val;
Michael S. Tsirkin1d921282012-08-26 18:00:29 +03004138 break;
Marcelo Tosatti2d5ba192019-06-03 19:52:44 -03004139 case MSR_KVM_POLL_CONTROL:
Oliver Upton1930e5d2020-10-27 16:10:41 -07004140 if (!guest_pv_has(vcpu, KVM_FEATURE_POLL_CONTROL))
4141 return 1;
4142
Marcelo Tosatti2d5ba192019-06-03 19:52:44 -03004143 msr_info->data = vcpu->arch.msr_kvm_poll_control;
4144 break;
Huang Ying890ca9a2009-05-11 16:48:15 +08004145 case MSR_IA32_P5_MC_ADDR:
4146 case MSR_IA32_P5_MC_TYPE:
4147 case MSR_IA32_MCG_CAP:
4148 case MSR_IA32_MCG_CTL:
4149 case MSR_IA32_MCG_STATUS:
Chen Yucong81760dc2014-09-23 10:44:35 +08004150 case MSR_IA32_MC0_CTL ... MSR_IA32_MCx_CTL(KVM_MAX_MCE_BANKS) - 1:
Jue Wang281b5272022-06-10 10:11:32 -07004151 case MSR_IA32_MC0_CTL2 ... MSR_IA32_MCx_CTL2(KVM_MAX_MCE_BANKS) - 1:
Paolo Bonzini44883f02018-07-26 13:01:52 +02004152 return get_msr_mce(vcpu, msr_info->index, &msr_info->data,
4153 msr_info->host_initiated);
Aaron Lewis864e2ab2019-10-21 16:30:26 -07004154 case MSR_IA32_XSS:
4155 if (!msr_info->host_initiated &&
4156 !guest_cpuid_has(vcpu, X86_FEATURE_XSAVES))
4157 return 1;
4158 msr_info->data = vcpu->arch.ia32_xss;
4159 break;
Jes Sorensen84e0cef2010-09-01 11:42:04 +02004160 case MSR_K7_CLK_CTL:
4161 /*
4162 * Provide expected ramp-up count for K7. All other
4163 * are set to zero, indicating minimum divisors for
4164 * every field.
4165 *
4166 * This prevents guest kernels on AMD host with CPU
4167 * type 6, model 8 and higher from exploding due to
4168 * the rdmsr failing.
4169 */
Paolo Bonzini609e36d2015-04-08 15:30:38 +02004170 msr_info->data = 0x20000000;
Jes Sorensen84e0cef2010-09-01 11:42:04 +02004171 break;
Gleb Natapov55cd8e52010-01-17 15:51:22 +02004172 case HV_X64_MSR_GUEST_OS_ID ... HV_X64_MSR_SINT15:
Jon Doronf97f5a52020-05-29 16:45:40 +03004173 case HV_X64_MSR_SYNDBG_CONTROL ... HV_X64_MSR_SYNDBG_PENDING_BUFFER:
4174 case HV_X64_MSR_SYNDBG_OPTIONS:
Andrey Smetanine7d95132015-07-03 15:01:37 +03004175 case HV_X64_MSR_CRASH_P0 ... HV_X64_MSR_CRASH_P4:
4176 case HV_X64_MSR_CRASH_CTL:
Andrey Smetanin1f4b34f2015-11-30 19:22:21 +03004177 case HV_X64_MSR_STIMER0_CONFIG ... HV_X64_MSR_STIMER3_COUNT:
Vitaly Kuznetsova2e164e2018-03-01 15:15:12 +01004178 case HV_X64_MSR_REENLIGHTENMENT_CONTROL:
4179 case HV_X64_MSR_TSC_EMULATION_CONTROL:
4180 case HV_X64_MSR_TSC_EMULATION_STATUS:
Andrey Smetanine83d5882015-07-03 15:01:34 +03004181 return kvm_hv_get_msr_common(vcpu,
Paolo Bonzini44883f02018-07-26 13:01:52 +02004182 msr_info->index, &msr_info->data,
4183 msr_info->host_initiated);
john cooper91c9c3e2011-01-21 00:21:00 -05004184 case MSR_IA32_BBL_CR_CTL3:
4185 /* This legacy MSR exists but isn't fully documented in current
4186 * silicon. It is however accessed by winxp in very narrow
4187 * scenarios where it sets bit #19, itself documented as
4188 * a "reserved" bit. Best effort attempt to source coherent
4189 * read data here should the balance of the register be
4190 * interpreted by the guest:
4191 *
4192 * L2 cache control register 3: 64GB range, 256KB size,
4193 * enabled, latency 0x1, configured
4194 */
Paolo Bonzini609e36d2015-04-08 15:30:38 +02004195 msr_info->data = 0xbe702111;
john cooper91c9c3e2011-01-21 00:21:00 -05004196 break;
Boris Ostrovsky2b036c62012-01-09 14:00:35 -05004197 case MSR_AMD64_OSVW_ID_LENGTH:
Radim Krčmářd6321d42017-08-05 00:12:49 +02004198 if (!guest_cpuid_has(vcpu, X86_FEATURE_OSVW))
Boris Ostrovsky2b036c62012-01-09 14:00:35 -05004199 return 1;
Paolo Bonzini609e36d2015-04-08 15:30:38 +02004200 msr_info->data = vcpu->arch.osvw.length;
Boris Ostrovsky2b036c62012-01-09 14:00:35 -05004201 break;
4202 case MSR_AMD64_OSVW_STATUS:
Radim Krčmářd6321d42017-08-05 00:12:49 +02004203 if (!guest_cpuid_has(vcpu, X86_FEATURE_OSVW))
Boris Ostrovsky2b036c62012-01-09 14:00:35 -05004204 return 1;
Paolo Bonzini609e36d2015-04-08 15:30:38 +02004205 msr_info->data = vcpu->arch.osvw.status;
Boris Ostrovsky2b036c62012-01-09 14:00:35 -05004206 break;
Kyle Hueydb2336a2017-03-20 01:16:28 -07004207 case MSR_PLATFORM_INFO:
Drew Schmitt6fbbde92018-08-20 10:32:15 -07004208 if (!msr_info->host_initiated &&
4209 !vcpu->kvm->arch.guest_can_read_msr_platform_info)
4210 return 1;
Kyle Hueydb2336a2017-03-20 01:16:28 -07004211 msr_info->data = vcpu->arch.msr_platform_info;
4212 break;
4213 case MSR_MISC_FEATURES_ENABLES:
4214 msr_info->data = vcpu->arch.msr_misc_features_enables;
4215 break;
Borislav Petkov191c8132019-04-18 18:32:50 +02004216 case MSR_K7_HWCR:
4217 msr_info->data = vcpu->arch.msr_hwcr;
4218 break;
Jing Liu820a6ee2022-01-05 04:35:21 -08004219#ifdef CONFIG_X86_64
4220 case MSR_IA32_XFD:
4221 if (!msr_info->host_initiated &&
4222 !guest_cpuid_has(vcpu, X86_FEATURE_XFD))
4223 return 1;
4224
4225 msr_info->data = vcpu->arch.guest_fpu.fpstate->xfd;
4226 break;
Jing Liu548e8362022-01-05 04:35:24 -08004227 case MSR_IA32_XFD_ERR:
4228 if (!msr_info->host_initiated &&
4229 !guest_cpuid_has(vcpu, X86_FEATURE_XFD))
4230 return 1;
4231
4232 msr_info->data = vcpu->arch.guest_fpu.xfd_err;
4233 break;
Jing Liu820a6ee2022-01-05 04:35:21 -08004234#endif
Carsten Otte15c4a642007-10-30 18:44:17 +01004235 default:
Wei Huangc6702c92015-06-19 13:44:45 +02004236 if (kvm_pmu_is_valid_msr(vcpu, msr_info->index))
Wei Wangcbd71752020-05-29 15:43:44 +08004237 return kvm_pmu_get_msr(vcpu, msr_info);
Peter Xu6abe9c12020-06-22 18:04:41 -04004238 return KVM_MSR_RET_INVALID;
Carsten Otte15c4a642007-10-30 18:44:17 +01004239 }
Carsten Otte15c4a642007-10-30 18:44:17 +01004240 return 0;
4241}
4242EXPORT_SYMBOL_GPL(kvm_get_msr_common);
4243
Carsten Otte313a3dc2007-10-11 19:16:52 +02004244/*
4245 * Read or write a bunch of msrs. All parameters are kernel addresses.
4246 *
4247 * @return number of msrs set successfully.
4248 */
4249static int __msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs *msrs,
4250 struct kvm_msr_entry *entries,
4251 int (*do_msr)(struct kvm_vcpu *vcpu,
4252 unsigned index, u64 *data))
4253{
Tom Lendacky801e4592018-02-21 13:39:51 -06004254 int i;
Carsten Otte313a3dc2007-10-11 19:16:52 +02004255
Carsten Otte313a3dc2007-10-11 19:16:52 +02004256 for (i = 0; i < msrs->nmsrs; ++i)
4257 if (do_msr(vcpu, entries[i].index, &entries[i].data))
4258 break;
4259
Carsten Otte313a3dc2007-10-11 19:16:52 +02004260 return i;
4261}
4262
4263/*
4264 * Read or write a bunch of msrs. Parameters are user addresses.
4265 *
4266 * @return number of msrs set successfully.
4267 */
4268static int msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs __user *user_msrs,
4269 int (*do_msr)(struct kvm_vcpu *vcpu,
4270 unsigned index, u64 *data),
4271 int writeback)
4272{
4273 struct kvm_msrs msrs;
4274 struct kvm_msr_entry *entries;
4275 int r, n;
4276 unsigned size;
4277
4278 r = -EFAULT;
Jordan Borgner0e96f312018-10-28 12:58:28 +00004279 if (copy_from_user(&msrs, user_msrs, sizeof(msrs)))
Carsten Otte313a3dc2007-10-11 19:16:52 +02004280 goto out;
4281
4282 r = -E2BIG;
4283 if (msrs.nmsrs >= MAX_IO_MSRS)
4284 goto out;
4285
Carsten Otte313a3dc2007-10-11 19:16:52 +02004286 size = sizeof(struct kvm_msr_entry) * msrs.nmsrs;
Sasha Levinff5c2c02011-12-04 19:36:29 +02004287 entries = memdup_user(user_msrs->entries, size);
4288 if (IS_ERR(entries)) {
4289 r = PTR_ERR(entries);
Carsten Otte313a3dc2007-10-11 19:16:52 +02004290 goto out;
Sasha Levinff5c2c02011-12-04 19:36:29 +02004291 }
Carsten Otte313a3dc2007-10-11 19:16:52 +02004292
4293 r = n = __msr_io(vcpu, &msrs, entries, do_msr);
4294 if (r < 0)
4295 goto out_free;
4296
4297 r = -EFAULT;
4298 if (writeback && copy_to_user(user_msrs->entries, entries, size))
4299 goto out_free;
4300
4301 r = n;
4302
4303out_free:
Avi Kivity7a73c022010-07-22 23:24:52 +03004304 kfree(entries);
Carsten Otte313a3dc2007-10-11 19:16:52 +02004305out:
4306 return r;
4307}
4308
Wanpeng Li4d5422c2018-03-12 04:53:02 -07004309static inline bool kvm_can_mwait_in_guest(void)
4310{
4311 return boot_cpu_has(X86_FEATURE_MWAIT) &&
KarimAllah Ahmed8e9b29b2018-04-11 11:16:03 +02004312 !boot_cpu_has_bug(X86_BUG_MONITOR) &&
4313 boot_cpu_has(X86_FEATURE_ARAT);
Wanpeng Li4d5422c2018-03-12 04:53:02 -07004314}
4315
Vitaly Kuznetsovc21d54f2020-09-29 17:09:43 +02004316static int kvm_ioctl_get_supported_hv_cpuid(struct kvm_vcpu *vcpu,
4317 struct kvm_cpuid2 __user *cpuid_arg)
4318{
4319 struct kvm_cpuid2 cpuid;
4320 int r;
4321
4322 r = -EFAULT;
4323 if (copy_from_user(&cpuid, cpuid_arg, sizeof(cpuid)))
4324 return r;
4325
4326 r = kvm_get_hv_cpuid(vcpu, &cpuid, cpuid_arg->entries);
4327 if (r)
4328 return r;
4329
4330 r = -EFAULT;
4331 if (copy_to_user(cpuid_arg, &cpuid, sizeof(cpuid)))
4332 return r;
4333
4334 return 0;
4335}
4336
Alexander Graf784aa3d2014-07-14 18:27:35 +02004337int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
Zhang Xiantao018d00d2007-11-15 23:07:47 +08004338{
Wanpeng Li4d5422c2018-03-12 04:53:02 -07004339 int r = 0;
Zhang Xiantao018d00d2007-11-15 23:07:47 +08004340
4341 switch (ext) {
4342 case KVM_CAP_IRQCHIP:
4343 case KVM_CAP_HLT:
4344 case KVM_CAP_MMU_SHADOW_CACHE_CONTROL:
Zhang Xiantao018d00d2007-11-15 23:07:47 +08004345 case KVM_CAP_SET_TSS_ADDR:
Dan Kenigsberg07716712007-11-21 17:10:04 +02004346 case KVM_CAP_EXT_CPUID:
Borislav Petkov9c15bb12013-09-22 16:44:50 +02004347 case KVM_CAP_EXT_EMUL_CPUID:
Gerd Hoffmannc8076602009-02-04 17:52:04 +01004348 case KVM_CAP_CLOCKSOURCE:
Sheng Yang78376992008-01-28 05:10:22 +08004349 case KVM_CAP_PIT:
Marcelo Tosattia28e4f52008-02-22 12:21:36 -05004350 case KVM_CAP_NOP_IO_DELAY:
Marcelo Tosatti62d9f0d2008-04-11 13:24:45 -03004351 case KVM_CAP_MP_STATE:
Avi Kivityed848622008-07-29 11:30:57 +03004352 case KVM_CAP_SYNC_MMU:
Lai Jiangshana355c852010-12-14 17:57:47 +08004353 case KVM_CAP_USER_NMI:
Marcelo Tosatti52d939a2008-12-30 15:55:06 -02004354 case KVM_CAP_REINJECT_CONTROL:
Gleb Natapov49256632009-02-04 17:28:14 +02004355 case KVM_CAP_IRQ_INJECT_STATUS:
Gregory Haskinsd34e6b12009-07-07 17:08:49 -04004356 case KVM_CAP_IOEVENTFD:
Michael S. Tsirkinf848a5a2014-03-31 21:50:38 +03004357 case KVM_CAP_IOEVENTFD_NO_LENGTH:
Jan Kiszkac5ff41ce2009-05-14 22:42:53 +02004358 case KVM_CAP_PIT2:
Beth Kone9f42752009-07-07 11:50:38 -04004359 case KVM_CAP_PIT_STATE2:
Sheng Yangb927a3c2009-07-21 10:42:48 +08004360 case KVM_CAP_SET_IDENTITY_MAP_ADDR:
Jan Kiszka3cfc3092009-11-12 01:04:25 +01004361 case KVM_CAP_VCPU_EVENTS:
Gleb Natapov55cd8e52010-01-17 15:51:22 +02004362 case KVM_CAP_HYPERV:
Gleb Natapov10388a02010-01-17 15:51:23 +02004363 case KVM_CAP_HYPERV_VAPIC:
Gleb Natapovc25bc162010-01-17 15:51:24 +02004364 case KVM_CAP_HYPERV_SPIN:
Andrey Smetanin5c9194122015-11-10 15:36:34 +03004365 case KVM_CAP_HYPERV_SYNIC:
Roman Kaganefc479e2017-06-22 16:51:01 +03004366 case KVM_CAP_HYPERV_SYNIC2:
Roman Kagand3457c82017-07-14 17:13:20 +03004367 case KVM_CAP_HYPERV_VP_INDEX:
Roman Kaganfaeb7832018-02-01 16:48:32 +03004368 case KVM_CAP_HYPERV_EVENTFD:
Vitaly Kuznetsovc1aea912018-05-16 17:21:31 +02004369 case KVM_CAP_HYPERV_TLBFLUSH:
Vitaly Kuznetsov214ff832018-09-26 19:02:59 +02004370 case KVM_CAP_HYPERV_SEND_IPI:
Vitaly Kuznetsov2bc39972018-12-10 18:21:56 +01004371 case KVM_CAP_HYPERV_CPUID:
Vitaly Kuznetsov644f7062021-05-21 11:51:36 +02004372 case KVM_CAP_HYPERV_ENFORCE_CPUID:
Vitaly Kuznetsovc21d54f2020-09-29 17:09:43 +02004373 case KVM_CAP_SYS_HYPERV_CPUID:
Zhai, Edwinab9f4ec2010-01-29 14:38:44 +08004374 case KVM_CAP_PCI_SEGMENT:
Jan Kiszkaa1efbe72010-02-15 10:45:43 +01004375 case KVM_CAP_DEBUGREGS:
Jan Kiszkad2be1652010-02-23 17:47:57 +01004376 case KVM_CAP_X86_ROBUST_SINGLESTEP:
Sheng Yang2d5b5a62010-06-13 17:29:39 +08004377 case KVM_CAP_XSAVE:
Gleb Natapov344d9582010-10-14 11:22:50 +02004378 case KVM_CAP_ASYNC_PF:
Vitaly Kuznetsov72de5fa4c2020-05-25 16:41:22 +02004379 case KVM_CAP_ASYNC_PF_INT:
Joerg Roedel92a1f122011-03-25 09:44:51 +01004380 case KVM_CAP_GET_TSC_KHZ:
Eric B Munson1c0b28c2012-03-10 14:37:27 -05004381 case KVM_CAP_KVMCLOCK_CTRL:
Xiao Guangrong4d8b81a2012-08-21 11:02:51 +08004382 case KVM_CAP_READONLY_MEM:
Paolo Bonzini5f66b622014-01-29 18:10:45 +01004383 case KVM_CAP_HYPERV_TIME:
Gabriel L. Somlo100943c2014-02-27 23:06:17 -05004384 case KVM_CAP_IOAPIC_POLARITY_IGNORED:
Radim Krčmářdefcf512015-01-08 15:59:30 +01004385 case KVM_CAP_TSC_DEADLINE_TIMER:
Nadav Amit90de4a12015-04-13 01:53:41 +03004386 case KVM_CAP_DISABLE_QUIRKS:
Paolo Bonzinid71ba782015-07-29 11:56:48 +02004387 case KVM_CAP_SET_BOOT_CPU_ID:
Steve Rutherford49df6392015-07-29 23:21:40 -07004388 case KVM_CAP_SPLIT_IRQCHIP:
Paolo Bonzini460df4c2017-02-08 11:50:15 +01004389 case KVM_CAP_IMMEDIATE_EXIT:
Eric Hankland66bb8a02019-07-10 18:25:15 -07004390 case KVM_CAP_PMU_EVENT_FILTER:
Tom Lendacky801e4592018-02-21 13:39:51 -06004391 case KVM_CAP_GET_MSR_FEATURES:
Drew Schmitt6fbbde92018-08-20 10:32:15 -07004392 case KVM_CAP_MSR_PLATFORM_INFO:
Jim Mattsonc4f55192018-10-16 14:29:24 -07004393 case KVM_CAP_EXCEPTION_PAYLOAD:
Chenyi Qianged235112022-05-24 21:56:21 +08004394 case KVM_CAP_X86_TRIPLE_FAULT_EVENT:
Peter Xub9b27822020-05-05 11:47:50 -04004395 case KVM_CAP_SET_GUEST_DEBUG:
Jim Mattson1aa561b2020-06-03 16:56:21 -07004396 case KVM_CAP_LAST_CPU:
Alexander Graf1ae09952020-09-25 16:34:16 +02004397 case KVM_CAP_X86_USER_SPACE_MSR:
Alexander Graf1a1552542020-09-25 16:34:21 +02004398 case KVM_CAP_X86_MSR_FILTER:
Oliver Upton66570e962020-08-18 15:24:28 +00004399 case KVM_CAP_ENFORCE_PV_FEATURE_CPUID:
Sean Christophersonfe7e9482021-04-12 16:21:43 +12004400#ifdef CONFIG_X86_SGX_KVM
4401 case KVM_CAP_SGX_ATTRIBUTE:
4402#endif
Nathan Tempelman54526d12021-04-08 22:32:14 +00004403 case KVM_CAP_VM_COPY_ENC_CONTEXT_FROM:
Paolo Bonzini30d7c5d2021-11-18 04:41:34 -05004404 case KVM_CAP_VM_MOVE_ENC_CONTEXT_FROM:
Maxim Levitsky6dba9402021-06-07 12:02:02 +03004405 case KVM_CAP_SREGS2:
Aaron Lewis19238e72021-05-10 07:48:33 -07004406 case KVM_CAP_EXIT_ON_EMULATION_FAILURE:
Oliver Upton828ca892021-09-16 18:15:38 +00004407 case KVM_CAP_VCPU_ATTRIBUTES:
Paolo Bonzinidd6e6312022-01-26 07:49:45 -05004408 case KVM_CAP_SYS_ATTRIBUTES:
Paolo Bonzini8a289782022-02-15 13:16:36 -05004409 case KVM_CAP_VAPIC:
Aaron Lewis127770a2022-02-14 21:29:51 +00004410 case KVM_CAP_ENABLE_CAP:
Ben Gardon084cc29f2022-06-13 21:25:21 +00004411 case KVM_CAP_VM_DISABLE_NX_HUGE_PAGES:
Zhang Xiantao018d00d2007-11-15 23:07:47 +08004412 r = 1;
4413 break;
Ashish Kalra0dbb1122021-06-08 18:05:43 +00004414 case KVM_CAP_EXIT_HYPERCALL:
4415 r = KVM_EXIT_HYPERCALL_VALID_MASK;
4416 break;
Maxim Levitsky7e582cc2021-04-01 16:54:45 +03004417 case KVM_CAP_SET_GUEST_DEBUG2:
4418 return KVM_GUESTDBG_VALID_MASK;
Paolo Bonzinib59b1532021-02-26 04:54:45 -05004419#ifdef CONFIG_KVM_XEN
Joao Martins23200b72018-06-13 09:55:44 -04004420 case KVM_CAP_XEN_HVM:
4421 r = KVM_XEN_HVM_CONFIG_HYPERCALL_MSR |
David Woodhouse8d4e7e82020-12-04 01:02:04 +00004422 KVM_XEN_HVM_CONFIG_INTERCEPT_HCALL |
David Woodhouse14243b32021-12-10 16:36:23 +00004423 KVM_XEN_HVM_CONFIG_SHARED_INFO |
David Woodhouse661a20f2022-03-03 15:41:25 +00004424 KVM_XEN_HVM_CONFIG_EVTCHN_2LEVEL |
4425 KVM_XEN_HVM_CONFIG_EVTCHN_SEND;
David Woodhouse30b5c852021-03-01 12:53:09 +00004426 if (sched_info_on())
4427 r |= KVM_XEN_HVM_CONFIG_RUNSTATE;
Joao Martins23200b72018-06-13 09:55:44 -04004428 break;
Paolo Bonzinib59b1532021-02-26 04:54:45 -05004429#endif
Ken Hofsass01643c52018-01-31 16:03:36 -08004430 case KVM_CAP_SYNC_REGS:
4431 r = KVM_SYNC_X86_VALID_FIELDS;
4432 break;
Paolo Bonzinie3fd9a92016-11-09 17:48:15 +01004433 case KVM_CAP_ADJUST_CLOCK:
Oliver Uptonc68dc1b2021-09-16 18:15:35 +00004434 r = KVM_CLOCK_VALID_FLAGS;
Paolo Bonzinie3fd9a92016-11-09 17:48:15 +01004435 break;
Wanpeng Li4d5422c2018-03-12 04:53:02 -07004436 case KVM_CAP_X86_DISABLE_EXITS:
Wanpeng Lib5170062019-05-21 14:06:53 +08004437 r |= KVM_X86_DISABLE_EXITS_HLT | KVM_X86_DISABLE_EXITS_PAUSE |
4438 KVM_X86_DISABLE_EXITS_CSTATE;
Wanpeng Li4d5422c2018-03-12 04:53:02 -07004439 if(kvm_can_mwait_in_guest())
4440 r |= KVM_X86_DISABLE_EXITS_MWAIT;
Michael S. Tsirkin668fffa2017-04-21 12:27:17 +02004441 break;
Paolo Bonzini6d396b52015-04-01 14:25:33 +02004442 case KVM_CAP_X86_SMM:
4443 /* SMBASE is usually relocated above 1M on modern chipsets,
4444 * and SMM handlers might indeed rely on 4G segment limits,
4445 * so do not report SMM to be available if real mode is
4446 * emulated via vm86 mode. Still, do not go to great lengths
4447 * to avoid userspace's usage of the feature, because it is a
4448 * fringe case that is not enabled except via specific settings
4449 * of the module parameters.
4450 */
Jason Baronb36464772021-01-14 22:27:56 -05004451 r = static_call(kvm_x86_has_emulated_msr)(kvm, MSR_IA32_SMBASE);
Paolo Bonzini6d396b52015-04-01 14:25:33 +02004452 break;
Avi Kivityf7252302008-02-20 11:53:16 +02004453 case KVM_CAP_NR_VCPUS:
Vitaly Kuznetsov2845e732021-11-16 17:34:43 +01004454 r = min_t(unsigned int, num_online_cpus(), KVM_MAX_VCPUS);
Sasha Levin8c3ba3342011-07-18 17:17:15 +03004455 break;
4456 case KVM_CAP_MAX_VCPUS:
Avi Kivityf7252302008-02-20 11:53:16 +02004457 r = KVM_MAX_VCPUS;
4458 break;
Thomas Hutha86cb412019-05-23 18:43:08 +02004459 case KVM_CAP_MAX_VCPU_ID:
Juergen Grossa1c42dd2021-09-13 15:57:44 +02004460 r = KVM_MAX_VCPU_IDS;
Thomas Hutha86cb412019-05-23 18:43:08 +02004461 break;
Marcelo Tosattia68a6a72009-10-01 19:28:39 -03004462 case KVM_CAP_PV_MMU: /* obsolete */
4463 r = 0;
Marcelo Tosatti2f333bc2008-02-22 12:21:37 -05004464 break;
Huang Ying890ca9a2009-05-11 16:48:15 +08004465 case KVM_CAP_MCE:
4466 r = KVM_MAX_MCE_BANKS;
4467 break;
Sheng Yang2d5b5a62010-06-13 17:29:39 +08004468 case KVM_CAP_XCRS:
Borislav Petkovd366bf72016-04-04 22:25:02 +02004469 r = boot_cpu_has(X86_FEATURE_XSAVE);
Sheng Yang2d5b5a62010-06-13 17:29:39 +08004470 break;
Joerg Roedel92a1f122011-03-25 09:44:51 +01004471 case KVM_CAP_TSC_CONTROL:
David Woodhouseffbb61d2022-02-25 14:53:02 +00004472 case KVM_CAP_VM_TSC_CONTROL:
Sean Christopherson938c8742022-05-24 21:56:23 +08004473 r = kvm_caps.has_tsc_control;
Joerg Roedel92a1f122011-03-25 09:44:51 +01004474 break;
Radim Krčmář371313132016-07-12 22:09:27 +02004475 case KVM_CAP_X2APIC_API:
4476 r = KVM_X2APIC_API_VALID_FLAGS;
4477 break;
Jim Mattson8fcc4b52018-07-10 11:27:20 +02004478 case KVM_CAP_NESTED_STATE:
Paolo Bonzini33b22172020-04-17 10:24:18 -04004479 r = kvm_x86_ops.nested_ops->get_state ?
4480 kvm_x86_ops.nested_ops->get_state(NULL, NULL, 0) : 0;
Jim Mattson8fcc4b52018-07-10 11:27:20 +02004481 break;
Tianyu Lan344c6c82019-08-22 22:30:20 +08004482 case KVM_CAP_HYPERV_DIRECT_TLBFLUSH:
Sean Christophersonafaf0b22020-03-21 13:26:00 -07004483 r = kvm_x86_ops.enable_direct_tlbflush != NULL;
Vitaly Kuznetsov5a0165f2019-08-28 09:59:05 +02004484 break;
4485 case KVM_CAP_HYPERV_ENLIGHTENED_VMCS:
Paolo Bonzini33b22172020-04-17 10:24:18 -04004486 r = kvm_x86_ops.nested_ops->enable_evmcs != NULL;
Tianyu Lan344c6c82019-08-22 22:30:20 +08004487 break;
Mohammed Gamal3edd6832020-07-10 17:48:11 +02004488 case KVM_CAP_SMALLER_MAXPHYADDR:
4489 r = (int) allow_smaller_maxphyaddr;
4490 break;
Andrew Jones004a0122020-08-04 19:06:04 +02004491 case KVM_CAP_STEAL_TIME:
4492 r = sched_info_on();
4493 break;
Chenyi Qiangfe6b6bc2020-11-06 17:03:14 +08004494 case KVM_CAP_X86_BUS_LOCK_EXIT:
Sean Christopherson938c8742022-05-24 21:56:23 +08004495 if (kvm_caps.has_bus_lock_exit)
Chenyi Qiangfe6b6bc2020-11-06 17:03:14 +08004496 r = KVM_BUS_LOCK_DETECTION_OFF |
4497 KVM_BUS_LOCK_DETECTION_EXIT;
4498 else
4499 r = 0;
4500 break;
Guang Zengbe50b202022-01-05 04:35:29 -08004501 case KVM_CAP_XSAVE2: {
4502 u64 guest_perm = xstate_get_guest_group_perm();
4503
Sean Christopherson938c8742022-05-24 21:56:23 +08004504 r = xstate_required_size(kvm_caps.supported_xcr0 & guest_perm, false);
Guang Zengbe50b202022-01-05 04:35:29 -08004505 if (r < sizeof(struct kvm_xsave))
4506 r = sizeof(struct kvm_xsave);
4507 break;
Ben Gardon1c4dc572022-06-13 21:25:20 +00004508 }
David Dunnba7bb6632022-02-23 22:57:41 +00004509 case KVM_CAP_PMU_CAPABILITY:
4510 r = enable_pmu ? KVM_CAP_PMU_VALID_MASK : 0;
4511 break;
Oliver Upton6d849192022-03-01 06:03:47 +00004512 case KVM_CAP_DISABLE_QUIRKS2:
4513 r = KVM_X86_VALID_QUIRKS;
4514 break;
Tao Xu2f4073e2022-05-24 21:56:24 +08004515 case KVM_CAP_X86_NOTIFY_VMEXIT:
4516 r = kvm_caps.has_notify_vmexit;
4517 break;
Zhang Xiantao018d00d2007-11-15 23:07:47 +08004518 default:
Zhang Xiantao018d00d2007-11-15 23:07:47 +08004519 break;
4520 }
4521 return r;
Sean Christopherson56f289a2022-01-27 07:31:53 -08004522}
Zhang Xiantao018d00d2007-11-15 23:07:47 +08004523
Sean Christopherson56f289a2022-01-27 07:31:53 -08004524static inline void __user *kvm_get_attr_addr(struct kvm_device_attr *attr)
4525{
4526 void __user *uaddr = (void __user*)(unsigned long)attr->addr;
4527
4528 if ((u64)(unsigned long)uaddr != attr->addr)
Sean Christopherson6e37ec882022-02-02 00:51:57 +00004529 return ERR_PTR_USR(-EFAULT);
Sean Christopherson56f289a2022-01-27 07:31:53 -08004530 return uaddr;
Zhang Xiantao018d00d2007-11-15 23:07:47 +08004531}
4532
Paolo Bonzinidd6e6312022-01-26 07:49:45 -05004533static int kvm_x86_dev_get_attr(struct kvm_device_attr *attr)
4534{
4535 u64 __user *uaddr = kvm_get_attr_addr(attr);
4536
4537 if (attr->group)
4538 return -ENXIO;
4539
4540 if (IS_ERR(uaddr))
4541 return PTR_ERR(uaddr);
4542
4543 switch (attr->attr) {
4544 case KVM_X86_XCOMP_GUEST_SUPP:
Sean Christopherson938c8742022-05-24 21:56:23 +08004545 if (put_user(kvm_caps.supported_xcr0, uaddr))
Paolo Bonzinidd6e6312022-01-26 07:49:45 -05004546 return -EFAULT;
4547 return 0;
4548 default:
4549 return -ENXIO;
4550 break;
4551 }
4552}
4553
4554static int kvm_x86_dev_has_attr(struct kvm_device_attr *attr)
4555{
4556 if (attr->group)
4557 return -ENXIO;
4558
4559 switch (attr->attr) {
4560 case KVM_X86_XCOMP_GUEST_SUPP:
4561 return 0;
4562 default:
4563 return -ENXIO;
4564 }
4565}
4566
Carsten Otte043405e2007-10-10 17:16:19 +02004567long kvm_arch_dev_ioctl(struct file *filp,
4568 unsigned int ioctl, unsigned long arg)
4569{
4570 void __user *argp = (void __user *)arg;
4571 long r;
4572
4573 switch (ioctl) {
4574 case KVM_GET_MSR_INDEX_LIST: {
4575 struct kvm_msr_list __user *user_msr_list = argp;
4576 struct kvm_msr_list msr_list;
4577 unsigned n;
4578
4579 r = -EFAULT;
Jordan Borgner0e96f312018-10-28 12:58:28 +00004580 if (copy_from_user(&msr_list, user_msr_list, sizeof(msr_list)))
Carsten Otte043405e2007-10-10 17:16:19 +02004581 goto out;
4582 n = msr_list.nmsrs;
Paolo Bonzini62ef68b2015-05-05 12:08:55 +02004583 msr_list.nmsrs = num_msrs_to_save + num_emulated_msrs;
Jordan Borgner0e96f312018-10-28 12:58:28 +00004584 if (copy_to_user(user_msr_list, &msr_list, sizeof(msr_list)))
Carsten Otte043405e2007-10-10 17:16:19 +02004585 goto out;
4586 r = -E2BIG;
Jan Kiszkae125e7b2009-07-02 21:45:47 +02004587 if (n < msr_list.nmsrs)
Carsten Otte043405e2007-10-10 17:16:19 +02004588 goto out;
4589 r = -EFAULT;
4590 if (copy_to_user(user_msr_list->indices, &msrs_to_save,
4591 num_msrs_to_save * sizeof(u32)))
4592 goto out;
Jan Kiszkae125e7b2009-07-02 21:45:47 +02004593 if (copy_to_user(user_msr_list->indices + num_msrs_to_save,
Carsten Otte043405e2007-10-10 17:16:19 +02004594 &emulated_msrs,
Paolo Bonzini62ef68b2015-05-05 12:08:55 +02004595 num_emulated_msrs * sizeof(u32)))
Carsten Otte043405e2007-10-10 17:16:19 +02004596 goto out;
4597 r = 0;
4598 break;
4599 }
Borislav Petkov9c15bb12013-09-22 16:44:50 +02004600 case KVM_GET_SUPPORTED_CPUID:
4601 case KVM_GET_EMULATED_CPUID: {
Avi Kivity674eea0f2008-02-11 18:37:23 +02004602 struct kvm_cpuid2 __user *cpuid_arg = argp;
4603 struct kvm_cpuid2 cpuid;
4604
4605 r = -EFAULT;
Jordan Borgner0e96f312018-10-28 12:58:28 +00004606 if (copy_from_user(&cpuid, cpuid_arg, sizeof(cpuid)))
Avi Kivity674eea0f2008-02-11 18:37:23 +02004607 goto out;
Borislav Petkov9c15bb12013-09-22 16:44:50 +02004608
4609 r = kvm_dev_ioctl_get_cpuid(&cpuid, cpuid_arg->entries,
4610 ioctl);
Avi Kivity674eea0f2008-02-11 18:37:23 +02004611 if (r)
4612 goto out;
4613
4614 r = -EFAULT;
Jordan Borgner0e96f312018-10-28 12:58:28 +00004615 if (copy_to_user(cpuid_arg, &cpuid, sizeof(cpuid)))
Avi Kivity674eea0f2008-02-11 18:37:23 +02004616 goto out;
4617 r = 0;
4618 break;
4619 }
Xiaoyao Licf6c26e2020-02-29 10:52:12 +08004620 case KVM_X86_GET_MCE_CAP_SUPPORTED:
Huang Ying890ca9a2009-05-11 16:48:15 +08004621 r = -EFAULT;
Sean Christopherson938c8742022-05-24 21:56:23 +08004622 if (copy_to_user(argp, &kvm_caps.supported_mce_cap,
4623 sizeof(kvm_caps.supported_mce_cap)))
Huang Ying890ca9a2009-05-11 16:48:15 +08004624 goto out;
4625 r = 0;
4626 break;
Tom Lendacky801e4592018-02-21 13:39:51 -06004627 case KVM_GET_MSR_FEATURE_INDEX_LIST: {
4628 struct kvm_msr_list __user *user_msr_list = argp;
4629 struct kvm_msr_list msr_list;
4630 unsigned int n;
4631
4632 r = -EFAULT;
4633 if (copy_from_user(&msr_list, user_msr_list, sizeof(msr_list)))
4634 goto out;
4635 n = msr_list.nmsrs;
4636 msr_list.nmsrs = num_msr_based_features;
4637 if (copy_to_user(user_msr_list, &msr_list, sizeof(msr_list)))
4638 goto out;
4639 r = -E2BIG;
4640 if (n < msr_list.nmsrs)
4641 goto out;
4642 r = -EFAULT;
4643 if (copy_to_user(user_msr_list->indices, &msr_based_features,
4644 num_msr_based_features * sizeof(u32)))
4645 goto out;
4646 r = 0;
4647 break;
4648 }
4649 case KVM_GET_MSRS:
4650 r = msr_io(NULL, argp, do_get_msr_feature, 1);
4651 break;
Vitaly Kuznetsovc21d54f2020-09-29 17:09:43 +02004652 case KVM_GET_SUPPORTED_HV_CPUID:
4653 r = kvm_ioctl_get_supported_hv_cpuid(NULL, argp);
4654 break;
Paolo Bonzinidd6e6312022-01-26 07:49:45 -05004655 case KVM_GET_DEVICE_ATTR: {
4656 struct kvm_device_attr attr;
4657 r = -EFAULT;
4658 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
4659 break;
4660 r = kvm_x86_dev_get_attr(&attr);
4661 break;
4662 }
4663 case KVM_HAS_DEVICE_ATTR: {
4664 struct kvm_device_attr attr;
4665 r = -EFAULT;
4666 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
4667 break;
4668 r = kvm_x86_dev_has_attr(&attr);
4669 break;
4670 }
Carsten Otte043405e2007-10-10 17:16:19 +02004671 default:
4672 r = -EINVAL;
Xiaoyao Licf6c26e2020-02-29 10:52:12 +08004673 break;
Carsten Otte043405e2007-10-10 17:16:19 +02004674 }
4675out:
4676 return r;
4677}
4678
Sheng Yangf5f48ee2010-06-30 12:25:15 +08004679static void wbinvd_ipi(void *garbage)
4680{
4681 wbinvd();
4682}
4683
4684static bool need_emulate_wbinvd(struct kvm_vcpu *vcpu)
4685{
Alex Williamsone0f0bbc2013-10-30 11:02:30 -06004686 return kvm_arch_has_noncoherent_dma(vcpu->kvm);
Sheng Yangf5f48ee2010-06-30 12:25:15 +08004687}
4688
Carsten Otte313a3dc2007-10-11 19:16:52 +02004689void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
4690{
Sheng Yangf5f48ee2010-06-30 12:25:15 +08004691 /* Address WBINVD may be executed by guest */
4692 if (need_emulate_wbinvd(vcpu)) {
Jason Baronb36464772021-01-14 22:27:56 -05004693 if (static_call(kvm_x86_has_wbinvd_exit)())
Sheng Yangf5f48ee2010-06-30 12:25:15 +08004694 cpumask_set_cpu(cpu, vcpu->arch.wbinvd_dirty_mask);
4695 else if (vcpu->cpu != -1 && vcpu->cpu != cpu)
4696 smp_call_function_single(vcpu->cpu,
4697 wbinvd_ipi, NULL, 1);
4698 }
4699
Jason Baronb36464772021-01-14 22:27:56 -05004700 static_call(kvm_x86_vcpu_load)(vcpu, cpu);
Zachary Amsden0dd6a6e2012-02-03 15:43:56 -02004701
Babu Moger37486132020-05-12 18:59:06 -05004702 /* Save host pkru register if supported */
4703 vcpu->arch.host_pkru = read_pkru();
4704
Zachary Amsden0dd6a6e2012-02-03 15:43:56 -02004705 /* Apply any externally detected TSC adjustments (due to suspend) */
4706 if (unlikely(vcpu->arch.tsc_offset_adjustment)) {
4707 adjust_tsc_offset_host(vcpu, vcpu->arch.tsc_offset_adjustment);
4708 vcpu->arch.tsc_offset_adjustment = 0;
Guo Hui Liu105b21b2014-09-12 13:43:19 +08004709 kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
Zachary Amsden0dd6a6e2012-02-03 15:43:56 -02004710 }
4711
Vitaly Kuznetsovb0c39dc2018-01-24 14:23:36 +01004712 if (unlikely(vcpu->cpu != cpu) || kvm_check_tsc_unstable()) {
Zachary Amsden6f526ec2012-02-03 15:43:54 -02004713 s64 tsc_delta = !vcpu->arch.last_host_tsc ? 0 :
Andy Lutomirski4ea16362015-06-25 18:44:07 +02004714 rdtsc() - vcpu->arch.last_host_tsc;
Zachary Amsdene48672f2010-08-19 22:07:23 -10004715 if (tsc_delta < 0)
4716 mark_tsc_unstable("KVM discovered backwards TSC");
Yunhong Jiangce7a0582016-06-13 14:20:01 -07004717
Vitaly Kuznetsovb0c39dc2018-01-24 14:23:36 +01004718 if (kvm_check_tsc_unstable()) {
Ilias Stamatis9b399df2021-05-26 19:44:10 +01004719 u64 offset = kvm_compute_l1_tsc_offset(vcpu,
Zachary Amsdenb183aa52012-02-03 15:43:53 -02004720 vcpu->arch.last_guest_tsc);
Luiz Capitulinoa545ab62016-09-07 14:47:19 -04004721 kvm_vcpu_write_tsc_offset(vcpu, offset);
Zachary Amsdenc2855452010-09-18 14:38:15 -10004722 vcpu->arch.tsc_catchup = 1;
Zachary Amsdenc2855452010-09-18 14:38:15 -10004723 }
Paolo Bonzinia749e242017-06-29 17:14:50 +02004724
4725 if (kvm_lapic_hv_timer_in_use(vcpu))
4726 kvm_lapic_restart_hv_timer(vcpu);
4727
Marcelo Tosattid98d07c2012-11-27 23:29:04 -02004728 /*
4729 * On a host with synchronized TSC, there is no need to update
4730 * kvmclock on vcpu->cpu migration
4731 */
4732 if (!vcpu->kvm->arch.use_master_clock || vcpu->cpu == -1)
Marcelo Tosatti0061d53d2013-05-09 20:21:41 -03004733 kvm_make_request(KVM_REQ_GLOBAL_CLOCK_UPDATE, vcpu);
Zachary Amsdenc2855452010-09-18 14:38:15 -10004734 if (vcpu->cpu != cpu)
Radim Krčmář1bd20092017-04-26 22:32:20 +02004735 kvm_make_request(KVM_REQ_MIGRATE_TIMER, vcpu);
Zachary Amsdene48672f2010-08-19 22:07:23 -10004736 vcpu->cpu = cpu;
Zachary Amsden6b7d7e72009-10-09 16:26:08 -10004737 }
Glauber Costac9aaa892011-07-11 15:28:14 -04004738
Glauber Costac9aaa892011-07-11 15:28:14 -04004739 kvm_make_request(KVM_REQ_STEAL_UPDATE, vcpu);
Carsten Otte313a3dc2007-10-11 19:16:52 +02004740}
4741
Pan Xinhui0b9f6c42016-11-02 05:08:35 -04004742static void kvm_steal_time_set_preempted(struct kvm_vcpu *vcpu)
4743{
David Woodhouse7e2175e2021-11-02 17:36:39 +00004744 struct gfn_to_hva_cache *ghc = &vcpu->arch.st.cache;
4745 struct kvm_steal_time __user *st;
4746 struct kvm_memslots *slots;
4747 static const u8 preempted = KVM_VCPU_PREEMPTED;
Paolo Bonzinic3c28d22022-08-04 15:28:32 +02004748 gpa_t gpa = vcpu->arch.st.msr_val & KVM_STEAL_VALID_BITS;
Boris Ostrovskyb0431382019-12-05 03:45:32 +00004749
Paolo Bonzini6cd88242022-06-07 10:09:03 -04004750 /*
4751 * The vCPU can be marked preempted if and only if the VM-Exit was on
4752 * an instruction boundary and will not trigger guest emulation of any
4753 * kind (see vcpu_run). Vendor specific code controls (conservatively)
4754 * when this is true, for example allowing the vCPU to be marked
4755 * preempted if and only if the VM-Exit was due to a host interrupt.
4756 */
4757 if (!vcpu->arch.at_instruction_boundary) {
4758 vcpu->stat.preemption_other++;
4759 return;
4760 }
4761
4762 vcpu->stat.preemption_reported++;
Pan Xinhui0b9f6c42016-11-02 05:08:35 -04004763 if (!(vcpu->arch.st.msr_val & KVM_MSR_ENABLED))
4764 return;
4765
Boris Ostrovskya6bd8112019-12-06 15:36:12 +00004766 if (vcpu->arch.st.preempted)
Boris Ostrovsky8c6de562019-10-30 19:01:31 +00004767 return;
Pan Xinhui0b9f6c42016-11-02 05:08:35 -04004768
David Woodhouse7e2175e2021-11-02 17:36:39 +00004769 /* This happens on process exit */
4770 if (unlikely(current->mm != vcpu->kvm->mm))
Wanpeng Li9c1a0742021-04-23 16:23:20 +08004771 return;
Pan Xinhui0b9f6c42016-11-02 05:08:35 -04004772
David Woodhouse7e2175e2021-11-02 17:36:39 +00004773 slots = kvm_memslots(vcpu->kvm);
Boris Ostrovskyb0431382019-12-05 03:45:32 +00004774
David Woodhouse7e2175e2021-11-02 17:36:39 +00004775 if (unlikely(slots->generation != ghc->generation ||
Paolo Bonzinic3c28d22022-08-04 15:28:32 +02004776 gpa != ghc->gpa ||
David Woodhouse7e2175e2021-11-02 17:36:39 +00004777 kvm_is_error_hva(ghc->hva) || !ghc->memslot))
4778 return;
Boris Ostrovskyb0431382019-12-05 03:45:32 +00004779
David Woodhouse7e2175e2021-11-02 17:36:39 +00004780 st = (struct kvm_steal_time __user *)ghc->hva;
4781 BUILD_BUG_ON(sizeof(st->preempted) != sizeof(preempted));
4782
4783 if (!copy_to_user_nofault(&st->preempted, &preempted, sizeof(preempted)))
4784 vcpu->arch.st.preempted = KVM_VCPU_PREEMPTED;
4785
4786 mark_page_dirty_in_slot(vcpu->kvm, ghc->memslot, gpa_to_gfn(ghc->gpa));
Pan Xinhui0b9f6c42016-11-02 05:08:35 -04004787}
4788
Carsten Otte313a3dc2007-10-11 19:16:52 +02004789void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
4790{
Wanpeng Li9c1a0742021-04-23 16:23:20 +08004791 int idx;
4792
Paolo Bonzini54aa83c2022-06-07 10:07:11 -04004793 if (vcpu->preempted) {
4794 if (!vcpu->arch.guest_state_protected)
4795 vcpu->arch.preempted_in_kernel = !static_call(kvm_x86_get_cpl)(vcpu);
Longpeng(Mike)de63ad42017-08-08 12:05:33 +08004796
Paolo Bonzini54aa83c2022-06-07 10:07:11 -04004797 /*
4798 * Take the srcu lock as memslots will be accessed to check the gfn
4799 * cache generation against the memslots generation.
4800 */
4801 idx = srcu_read_lock(&vcpu->kvm->srcu);
4802 if (kvm_xen_msr_enabled(vcpu->kvm))
4803 kvm_xen_runstate_set_preempted(vcpu);
4804 else
4805 kvm_steal_time_set_preempted(vcpu);
4806 srcu_read_unlock(&vcpu->kvm->srcu, idx);
4807 }
David Woodhouse30b5c852021-03-01 12:53:09 +00004808
Jason Baronb36464772021-01-14 22:27:56 -05004809 static_call(kvm_x86_vcpu_put)(vcpu);
Andy Lutomirski4ea16362015-06-25 18:44:07 +02004810 vcpu->arch.last_host_tsc = rdtsc();
Carsten Otte313a3dc2007-10-11 19:16:52 +02004811}
4812
Carsten Otte313a3dc2007-10-11 19:16:52 +02004813static int kvm_vcpu_ioctl_get_lapic(struct kvm_vcpu *vcpu,
4814 struct kvm_lapic_state *s)
4815{
Paolo Bonzini37c4dbf2021-11-22 19:43:10 -05004816 static_call_cond(kvm_x86_sync_pir_to_irr)(vcpu);
Andrey Smetanind62caab2015-11-10 15:36:33 +03004817
Radim Krčmářa92e2542016-07-12 22:09:22 +02004818 return kvm_apic_get_state(vcpu, s);
Carsten Otte313a3dc2007-10-11 19:16:52 +02004819}
4820
4821static int kvm_vcpu_ioctl_set_lapic(struct kvm_vcpu *vcpu,
4822 struct kvm_lapic_state *s)
4823{
Radim Krčmářa92e2542016-07-12 22:09:22 +02004824 int r;
4825
4826 r = kvm_apic_set_state(vcpu, s);
4827 if (r)
4828 return r;
Gleb Natapovcb142eb2009-08-09 15:17:40 +03004829 update_cr8_intercept(vcpu);
Carsten Otte313a3dc2007-10-11 19:16:52 +02004830
4831 return 0;
4832}
4833
Matt Gingell127a4572015-11-17 17:32:05 +01004834static int kvm_cpu_accept_dm_intr(struct kvm_vcpu *vcpu)
4835{
Paolo Bonzini71cc8492020-11-27 09:18:20 +01004836 /*
4837 * We can accept userspace's request for interrupt injection
4838 * as long as we have a place to store the interrupt number.
4839 * The actual injection will happen when the CPU is able to
4840 * deliver the interrupt.
4841 */
4842 if (kvm_cpu_has_extint(vcpu))
4843 return false;
4844
4845 /* Acknowledging ExtINT does not happen if LINT0 is masked. */
Matt Gingell127a4572015-11-17 17:32:05 +01004846 return (!lapic_in_kernel(vcpu) ||
4847 kvm_apic_accept_pic_intr(vcpu));
4848}
4849
Matt Gingell782d4222015-11-16 15:26:00 -08004850static int kvm_vcpu_ready_for_interrupt_injection(struct kvm_vcpu *vcpu)
4851{
Paolo Bonzinifa7a5492021-07-14 17:37:49 -04004852 /*
4853 * Do not cause an interrupt window exit if an exception
4854 * is pending or an event needs reinjection; userspace
4855 * might want to inject the interrupt manually using KVM_SET_REGS
4856 * or KVM_SET_SREGS. For that to work, we must be at an
4857 * instruction boundary and with no events half-injected.
4858 */
4859 return (kvm_arch_interrupt_allowed(vcpu) &&
4860 kvm_cpu_accept_dm_intr(vcpu) &&
4861 !kvm_event_needs_reinjection(vcpu) &&
Sean Christopherson7709aba2022-08-30 23:16:08 +00004862 !kvm_is_exception_pending(vcpu));
Matt Gingell782d4222015-11-16 15:26:00 -08004863}
4864
Zhang Xiantaof77bc6a2007-11-21 04:36:41 +08004865static int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu,
4866 struct kvm_interrupt *irq)
4867{
Chen Gang02cdb502013-02-27 11:33:25 +08004868 if (irq->irq >= KVM_NR_INTERRUPTS)
Zhang Xiantaof77bc6a2007-11-21 04:36:41 +08004869 return -EINVAL;
Steve Rutherford1c1a9ce2015-07-30 11:27:16 +02004870
4871 if (!irqchip_in_kernel(vcpu->kvm)) {
4872 kvm_queue_interrupt(vcpu, irq->irq, false);
4873 kvm_make_request(KVM_REQ_EVENT, vcpu);
4874 return 0;
4875 }
4876
4877 /*
4878 * With in-kernel LAPIC, we only use this to inject EXTINT, so
4879 * fail for in-kernel 8259.
4880 */
4881 if (pic_in_kernel(vcpu->kvm))
Zhang Xiantaof77bc6a2007-11-21 04:36:41 +08004882 return -ENXIO;
Zhang Xiantaof77bc6a2007-11-21 04:36:41 +08004883
Steve Rutherford1c1a9ce2015-07-30 11:27:16 +02004884 if (vcpu->arch.pending_external_vector != -1)
4885 return -EEXIST;
Zhang Xiantaof77bc6a2007-11-21 04:36:41 +08004886
Steve Rutherford1c1a9ce2015-07-30 11:27:16 +02004887 vcpu->arch.pending_external_vector = irq->irq;
Matt Gingell934bf652015-11-16 15:26:05 -08004888 kvm_make_request(KVM_REQ_EVENT, vcpu);
Zhang Xiantaof77bc6a2007-11-21 04:36:41 +08004889 return 0;
4890}
4891
Jan Kiszkac4abb7c2008-09-26 09:30:55 +02004892static int kvm_vcpu_ioctl_nmi(struct kvm_vcpu *vcpu)
4893{
Jan Kiszkac4abb7c2008-09-26 09:30:55 +02004894 kvm_inject_nmi(vcpu);
Jan Kiszkac4abb7c2008-09-26 09:30:55 +02004895
4896 return 0;
4897}
4898
Paolo Bonzinif0778252015-04-01 15:06:40 +02004899static int kvm_vcpu_ioctl_smi(struct kvm_vcpu *vcpu)
4900{
Paolo Bonzini64d60672015-05-07 11:36:11 +02004901 kvm_make_request(KVM_REQ_SMI, vcpu);
4902
Paolo Bonzinif0778252015-04-01 15:06:40 +02004903 return 0;
4904}
4905
Avi Kivityb209749f2007-10-22 16:50:39 +02004906static int vcpu_ioctl_tpr_access_reporting(struct kvm_vcpu *vcpu,
4907 struct kvm_tpr_access_ctl *tac)
4908{
4909 if (tac->flags)
4910 return -EINVAL;
4911 vcpu->arch.tpr_access_reporting = !!tac->enabled;
4912 return 0;
4913}
4914
Huang Ying890ca9a2009-05-11 16:48:15 +08004915static int kvm_vcpu_ioctl_x86_setup_mce(struct kvm_vcpu *vcpu,
4916 u64 mcg_cap)
4917{
4918 int r;
4919 unsigned bank_num = mcg_cap & 0xff, bank;
4920
4921 r = -EINVAL;
Jim Mattsonc4e0e4ab2020-05-11 15:56:16 -07004922 if (!bank_num || bank_num > KVM_MAX_MCE_BANKS)
Huang Ying890ca9a2009-05-11 16:48:15 +08004923 goto out;
Sean Christopherson938c8742022-05-24 21:56:23 +08004924 if (mcg_cap & ~(kvm_caps.supported_mce_cap | 0xff | 0xff0000))
Huang Ying890ca9a2009-05-11 16:48:15 +08004925 goto out;
4926 r = 0;
4927 vcpu->arch.mcg_cap = mcg_cap;
4928 /* Init IA32_MCG_CTL to all 1s */
4929 if (mcg_cap & MCG_CTL_P)
4930 vcpu->arch.mcg_ctl = ~(u64)0;
Jue Wang281b5272022-06-10 10:11:32 -07004931 /* Init IA32_MCi_CTL to all 1s, IA32_MCi_CTL2 to all 0s */
4932 for (bank = 0; bank < bank_num; bank++) {
Huang Ying890ca9a2009-05-11 16:48:15 +08004933 vcpu->arch.mce_banks[bank*4] = ~(u64)0;
Jue Wang281b5272022-06-10 10:11:32 -07004934 if (mcg_cap & MCG_CMCI_P)
4935 vcpu->arch.mci_ctl2_banks[bank] = 0;
4936 }
Sean Christophersonf83894b2022-07-08 15:48:10 -07004937
4938 kvm_apic_after_set_mcg_cap(vcpu);
Ashok Rajc45dcc72016-06-22 14:59:56 +08004939
Jason Baronb36464772021-01-14 22:27:56 -05004940 static_call(kvm_x86_setup_mce)(vcpu);
Huang Ying890ca9a2009-05-11 16:48:15 +08004941out:
4942 return r;
4943}
4944
Jue Wangaebc3ca2022-06-10 10:11:33 -07004945/*
4946 * Validate this is an UCNA (uncorrectable no action) error by checking the
4947 * MCG_STATUS and MCi_STATUS registers:
4948 * - none of the bits for Machine Check Exceptions are set
4949 * - both the VAL (valid) and UC (uncorrectable) bits are set
4950 * MCI_STATUS_PCC - Processor Context Corrupted
4951 * MCI_STATUS_S - Signaled as a Machine Check Exception
4952 * MCI_STATUS_AR - Software recoverable Action Required
4953 */
4954static bool is_ucna(struct kvm_x86_mce *mce)
4955{
4956 return !mce->mcg_status &&
4957 !(mce->status & (MCI_STATUS_PCC | MCI_STATUS_S | MCI_STATUS_AR)) &&
4958 (mce->status & MCI_STATUS_VAL) &&
4959 (mce->status & MCI_STATUS_UC);
4960}
4961
4962static int kvm_vcpu_x86_set_ucna(struct kvm_vcpu *vcpu, struct kvm_x86_mce *mce, u64* banks)
4963{
4964 u64 mcg_cap = vcpu->arch.mcg_cap;
4965
4966 banks[1] = mce->status;
4967 banks[2] = mce->addr;
4968 banks[3] = mce->misc;
4969 vcpu->arch.mcg_status = mce->mcg_status;
4970
4971 if (!(mcg_cap & MCG_CMCI_P) ||
4972 !(vcpu->arch.mci_ctl2_banks[mce->bank] & MCI_CTL2_CMCI_EN))
4973 return 0;
4974
4975 if (lapic_in_kernel(vcpu))
4976 kvm_apic_local_deliver(vcpu->arch.apic, APIC_LVTCMCI);
4977
4978 return 0;
4979}
4980
Huang Ying890ca9a2009-05-11 16:48:15 +08004981static int kvm_vcpu_ioctl_x86_set_mce(struct kvm_vcpu *vcpu,
4982 struct kvm_x86_mce *mce)
4983{
4984 u64 mcg_cap = vcpu->arch.mcg_cap;
4985 unsigned bank_num = mcg_cap & 0xff;
4986 u64 *banks = vcpu->arch.mce_banks;
4987
4988 if (mce->bank >= bank_num || !(mce->status & MCI_STATUS_VAL))
4989 return -EINVAL;
Jue Wangaebc3ca2022-06-10 10:11:33 -07004990
4991 banks += array_index_nospec(4 * mce->bank, 4 * bank_num);
4992
4993 if (is_ucna(mce))
4994 return kvm_vcpu_x86_set_ucna(vcpu, mce, banks);
4995
Huang Ying890ca9a2009-05-11 16:48:15 +08004996 /*
4997 * if IA32_MCG_CTL is not all 1s, the uncorrected error
4998 * reporting is disabled
4999 */
5000 if ((mce->status & MCI_STATUS_UC) && (mcg_cap & MCG_CTL_P) &&
5001 vcpu->arch.mcg_ctl != ~(u64)0)
5002 return 0;
Huang Ying890ca9a2009-05-11 16:48:15 +08005003 /*
5004 * if IA32_MCi_CTL is not all 1s, the uncorrected error
5005 * reporting is disabled for the bank
5006 */
5007 if ((mce->status & MCI_STATUS_UC) && banks[0] != ~(u64)0)
5008 return 0;
5009 if (mce->status & MCI_STATUS_UC) {
5010 if ((vcpu->arch.mcg_status & MCG_STATUS_MCIP) ||
Avi Kivityfc78f512009-12-07 12:16:48 +02005011 !kvm_read_cr4_bits(vcpu, X86_CR4_MCE)) {
Avi Kivitya8eeb042010-05-10 12:34:53 +03005012 kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
Huang Ying890ca9a2009-05-11 16:48:15 +08005013 return 0;
5014 }
5015 if (banks[1] & MCI_STATUS_VAL)
5016 mce->status |= MCI_STATUS_OVER;
5017 banks[2] = mce->addr;
5018 banks[3] = mce->misc;
5019 vcpu->arch.mcg_status = mce->mcg_status;
5020 banks[1] = mce->status;
5021 kvm_queue_exception(vcpu, MC_VECTOR);
5022 } else if (!(banks[1] & MCI_STATUS_VAL)
5023 || !(banks[1] & MCI_STATUS_UC)) {
5024 if (banks[1] & MCI_STATUS_VAL)
5025 mce->status |= MCI_STATUS_OVER;
5026 banks[2] = mce->addr;
5027 banks[3] = mce->misc;
5028 banks[1] = mce->status;
5029 } else
5030 banks[1] |= MCI_STATUS_OVER;
5031 return 0;
5032}
5033
Jan Kiszka3cfc3092009-11-12 01:04:25 +01005034static void kvm_vcpu_ioctl_x86_get_vcpu_events(struct kvm_vcpu *vcpu,
5035 struct kvm_vcpu_events *events)
5036{
Sean Christopherson7709aba2022-08-30 23:16:08 +00005037 struct kvm_queued_exception *ex;
Sean Christophersond4963e32022-08-30 23:16:01 +00005038
Avi Kivity7460fb4a2011-09-20 13:43:14 +03005039 process_nmi(vcpu);
Jim Mattson59073aa2018-10-16 14:29:20 -07005040
Jay Zhou1f7becf2021-01-18 16:47:20 +08005041 if (kvm_check_request(KVM_REQ_SMI, vcpu))
5042 process_smi(vcpu);
5043
Wanpeng Li664f8e22017-08-24 03:35:09 -07005044 /*
Sean Christopherson7709aba2022-08-30 23:16:08 +00005045 * KVM's ABI only allows for one exception to be migrated. Luckily,
5046 * the only time there can be two queued exceptions is if there's a
5047 * non-exiting _injected_ exception, and a pending exiting exception.
5048 * In that case, ignore the VM-Exiting exception as it's an extension
5049 * of the injected exception.
5050 */
5051 if (vcpu->arch.exception_vmexit.pending &&
5052 !vcpu->arch.exception.pending &&
5053 !vcpu->arch.exception.injected)
5054 ex = &vcpu->arch.exception_vmexit;
5055 else
5056 ex = &vcpu->arch.exception;
5057
5058 /*
Sean Christophersond4963e32022-08-30 23:16:01 +00005059 * In guest mode, payload delivery should be deferred if the exception
5060 * will be intercepted by L1, e.g. KVM should not modifying CR2 if L1
5061 * intercepts #PF, ditto for DR6 and #DBs. If the per-VM capability,
5062 * KVM_CAP_EXCEPTION_PAYLOAD, is not set, userspace may or may not
5063 * propagate the payload and so it cannot be safely deferred. Deliver
5064 * the payload if the capability hasn't been requested.
Oliver Uptona06230b2020-02-07 02:36:06 -08005065 */
5066 if (!vcpu->kvm->arch.exception_payload_enabled &&
Sean Christophersond4963e32022-08-30 23:16:01 +00005067 ex->pending && ex->has_payload)
5068 kvm_deliver_exception_payload(vcpu, ex);
Oliver Uptona06230b2020-02-07 02:36:06 -08005069
5070 /*
Jim Mattson59073aa2018-10-16 14:29:20 -07005071 * The API doesn't provide the instruction length for software
5072 * exceptions, so don't report them. As long as the guest RIP
5073 * isn't advanced, we should expect to encounter the exception
5074 * again.
Wanpeng Li664f8e22017-08-24 03:35:09 -07005075 */
Sean Christophersond4963e32022-08-30 23:16:01 +00005076 if (kvm_exception_is_soft(ex->vector)) {
Jim Mattson59073aa2018-10-16 14:29:20 -07005077 events->exception.injected = 0;
5078 events->exception.pending = 0;
5079 } else {
Sean Christophersond4963e32022-08-30 23:16:01 +00005080 events->exception.injected = ex->injected;
5081 events->exception.pending = ex->pending;
Jim Mattson59073aa2018-10-16 14:29:20 -07005082 /*
5083 * For ABI compatibility, deliberately conflate
5084 * pending and injected exceptions when
5085 * KVM_CAP_EXCEPTION_PAYLOAD isn't enabled.
5086 */
5087 if (!vcpu->kvm->arch.exception_payload_enabled)
Sean Christophersond4963e32022-08-30 23:16:01 +00005088 events->exception.injected |= ex->pending;
Jim Mattson59073aa2018-10-16 14:29:20 -07005089 }
Sean Christophersond4963e32022-08-30 23:16:01 +00005090 events->exception.nr = ex->vector;
5091 events->exception.has_error_code = ex->has_error_code;
5092 events->exception.error_code = ex->error_code;
5093 events->exception_has_payload = ex->has_payload;
5094 events->exception_payload = ex->payload;
Jan Kiszka3cfc3092009-11-12 01:04:25 +01005095
Jan Kiszka03b82a32010-02-15 10:45:41 +01005096 events->interrupt.injected =
Liran Alon04140b42018-03-23 03:01:31 +03005097 vcpu->arch.interrupt.injected && !vcpu->arch.interrupt.soft;
Jan Kiszka3cfc3092009-11-12 01:04:25 +01005098 events->interrupt.nr = vcpu->arch.interrupt.nr;
Jan Kiszka03b82a32010-02-15 10:45:41 +01005099 events->interrupt.soft = 0;
Jason Baronb36464772021-01-14 22:27:56 -05005100 events->interrupt.shadow = static_call(kvm_x86_get_interrupt_shadow)(vcpu);
Jan Kiszka3cfc3092009-11-12 01:04:25 +01005101
5102 events->nmi.injected = vcpu->arch.nmi_injected;
Avi Kivity7460fb4a2011-09-20 13:43:14 +03005103 events->nmi.pending = vcpu->arch.nmi_pending != 0;
Jason Baronb36464772021-01-14 22:27:56 -05005104 events->nmi.masked = static_call(kvm_x86_get_nmi_mask)(vcpu);
Vasiliy Kulikov97e69aa2010-10-30 22:54:47 +04005105 events->nmi.pad = 0;
Jan Kiszka3cfc3092009-11-12 01:04:25 +01005106
Jan Kiszka66450a22013-03-13 12:42:34 +01005107 events->sipi_vector = 0; /* never valid when reporting to user space */
Jan Kiszka3cfc3092009-11-12 01:04:25 +01005108
Paolo Bonzinif0778252015-04-01 15:06:40 +02005109 events->smi.smm = is_smm(vcpu);
5110 events->smi.pending = vcpu->arch.smi_pending;
5111 events->smi.smm_inside_nmi =
5112 !!(vcpu->arch.hflags & HF_SMM_INSIDE_NMI_MASK);
5113 events->smi.latched_init = kvm_lapic_latched_init(vcpu);
5114
Jan Kiszkadab4b912009-12-06 18:24:15 +01005115 events->flags = (KVM_VCPUEVENT_VALID_NMI_PENDING
Paolo Bonzinif0778252015-04-01 15:06:40 +02005116 | KVM_VCPUEVENT_VALID_SHADOW
5117 | KVM_VCPUEVENT_VALID_SMM);
Jim Mattson59073aa2018-10-16 14:29:20 -07005118 if (vcpu->kvm->arch.exception_payload_enabled)
5119 events->flags |= KVM_VCPUEVENT_VALID_PAYLOAD;
Chenyi Qianged235112022-05-24 21:56:21 +08005120 if (vcpu->kvm->arch.triple_fault_event) {
5121 events->triple_fault.pending = kvm_test_request(KVM_REQ_TRIPLE_FAULT, vcpu);
5122 events->flags |= KVM_VCPUEVENT_VALID_TRIPLE_FAULT;
5123 }
Jim Mattson59073aa2018-10-16 14:29:20 -07005124
Vasiliy Kulikov97e69aa2010-10-30 22:54:47 +04005125 memset(&events->reserved, 0, sizeof(events->reserved));
Jan Kiszka3cfc3092009-11-12 01:04:25 +01005126}
5127
Sean Christophersondc872752021-06-09 11:56:15 -07005128static void kvm_smm_changed(struct kvm_vcpu *vcpu, bool entering_smm);
Xiao Guangrong6ef4e072016-12-24 10:00:42 +01005129
Jan Kiszka3cfc3092009-11-12 01:04:25 +01005130static int kvm_vcpu_ioctl_x86_set_vcpu_events(struct kvm_vcpu *vcpu,
5131 struct kvm_vcpu_events *events)
5132{
Jan Kiszkadab4b912009-12-06 18:24:15 +01005133 if (events->flags & ~(KVM_VCPUEVENT_VALID_NMI_PENDING
Jan Kiszka48005f62010-02-19 19:38:07 +01005134 | KVM_VCPUEVENT_VALID_SIPI_VECTOR
Paolo Bonzinif0778252015-04-01 15:06:40 +02005135 | KVM_VCPUEVENT_VALID_SHADOW
Jim Mattson59073aa2018-10-16 14:29:20 -07005136 | KVM_VCPUEVENT_VALID_SMM
Chenyi Qianged235112022-05-24 21:56:21 +08005137 | KVM_VCPUEVENT_VALID_PAYLOAD
5138 | KVM_VCPUEVENT_VALID_TRIPLE_FAULT))
Jan Kiszka3cfc3092009-11-12 01:04:25 +01005139 return -EINVAL;
5140
Jim Mattson59073aa2018-10-16 14:29:20 -07005141 if (events->flags & KVM_VCPUEVENT_VALID_PAYLOAD) {
5142 if (!vcpu->kvm->arch.exception_payload_enabled)
5143 return -EINVAL;
5144 if (events->exception.pending)
5145 events->exception.injected = 0;
5146 else
5147 events->exception_has_payload = 0;
5148 } else {
5149 events->exception.pending = 0;
5150 events->exception_has_payload = 0;
5151 }
5152
5153 if ((events->exception.injected || events->exception.pending) &&
5154 (events->exception.nr > 31 || events->exception.nr == NMI_VECTOR))
Paolo Bonzini78e546c2016-06-01 14:09:20 +02005155 return -EINVAL;
5156
David Hildenbrand28bf2882017-03-23 11:46:03 +01005157 /* INITs are latched while in SMM */
5158 if (events->flags & KVM_VCPUEVENT_VALID_SMM &&
5159 (events->smi.smm || events->smi.pending) &&
5160 vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED)
5161 return -EINVAL;
5162
Avi Kivity7460fb4a2011-09-20 13:43:14 +03005163 process_nmi(vcpu);
Sean Christopherson7709aba2022-08-30 23:16:08 +00005164
5165 /*
5166 * Flag that userspace is stuffing an exception, the next KVM_RUN will
5167 * morph the exception to a VM-Exit if appropriate. Do this only for
5168 * pending exceptions, already-injected exceptions are not subject to
5169 * intercpetion. Note, userspace that conflates pending and injected
5170 * is hosed, and will incorrectly convert an injected exception into a
5171 * pending exception, which in turn may cause a spurious VM-Exit.
5172 */
5173 vcpu->arch.exception_from_userspace = events->exception.pending;
5174
5175 vcpu->arch.exception_vmexit.pending = false;
5176
Jim Mattson59073aa2018-10-16 14:29:20 -07005177 vcpu->arch.exception.injected = events->exception.injected;
5178 vcpu->arch.exception.pending = events->exception.pending;
Sean Christophersond4963e32022-08-30 23:16:01 +00005179 vcpu->arch.exception.vector = events->exception.nr;
Jan Kiszka3cfc3092009-11-12 01:04:25 +01005180 vcpu->arch.exception.has_error_code = events->exception.has_error_code;
5181 vcpu->arch.exception.error_code = events->exception.error_code;
Jim Mattson59073aa2018-10-16 14:29:20 -07005182 vcpu->arch.exception.has_payload = events->exception_has_payload;
5183 vcpu->arch.exception.payload = events->exception_payload;
Jan Kiszka3cfc3092009-11-12 01:04:25 +01005184
Liran Alon04140b42018-03-23 03:01:31 +03005185 vcpu->arch.interrupt.injected = events->interrupt.injected;
Jan Kiszka3cfc3092009-11-12 01:04:25 +01005186 vcpu->arch.interrupt.nr = events->interrupt.nr;
5187 vcpu->arch.interrupt.soft = events->interrupt.soft;
Jan Kiszka48005f62010-02-19 19:38:07 +01005188 if (events->flags & KVM_VCPUEVENT_VALID_SHADOW)
Jason Baronb36464772021-01-14 22:27:56 -05005189 static_call(kvm_x86_set_interrupt_shadow)(vcpu,
5190 events->interrupt.shadow);
Jan Kiszka3cfc3092009-11-12 01:04:25 +01005191
5192 vcpu->arch.nmi_injected = events->nmi.injected;
Jan Kiszkadab4b912009-12-06 18:24:15 +01005193 if (events->flags & KVM_VCPUEVENT_VALID_NMI_PENDING)
5194 vcpu->arch.nmi_pending = events->nmi.pending;
Jason Baronb36464772021-01-14 22:27:56 -05005195 static_call(kvm_x86_set_nmi_mask)(vcpu, events->nmi.masked);
Jan Kiszka3cfc3092009-11-12 01:04:25 +01005196
Jan Kiszka66450a22013-03-13 12:42:34 +01005197 if (events->flags & KVM_VCPUEVENT_VALID_SIPI_VECTOR &&
Paolo Bonzinibce87cc2016-01-08 13:48:51 +01005198 lapic_in_kernel(vcpu))
Jan Kiszka66450a22013-03-13 12:42:34 +01005199 vcpu->arch.apic->sipi_vector = events->sipi_vector;
Jan Kiszka3cfc3092009-11-12 01:04:25 +01005200
Paolo Bonzinif0778252015-04-01 15:06:40 +02005201 if (events->flags & KVM_VCPUEVENT_VALID_SMM) {
Sean Christophersonf7e57072022-01-25 22:03:58 +00005202 if (!!(vcpu->arch.hflags & HF_SMM_MASK) != events->smi.smm) {
5203 kvm_x86_ops.nested_ops->leave_nested(vcpu);
Sean Christophersondc872752021-06-09 11:56:15 -07005204 kvm_smm_changed(vcpu, events->smi.smm);
Sean Christophersonf7e57072022-01-25 22:03:58 +00005205 }
Xiao Guangrong6ef4e072016-12-24 10:00:42 +01005206
Paolo Bonzinif0778252015-04-01 15:06:40 +02005207 vcpu->arch.smi_pending = events->smi.pending;
Wanpeng Lif4ef1912017-08-01 16:05:25 -07005208
5209 if (events->smi.smm) {
5210 if (events->smi.smm_inside_nmi)
5211 vcpu->arch.hflags |= HF_SMM_INSIDE_NMI_MASK;
Paolo Bonzinif0778252015-04-01 15:06:40 +02005212 else
Wanpeng Lif4ef1912017-08-01 16:05:25 -07005213 vcpu->arch.hflags &= ~HF_SMM_INSIDE_NMI_MASK;
Liran Alonff90afa2019-11-11 11:16:39 +02005214 }
5215
5216 if (lapic_in_kernel(vcpu)) {
5217 if (events->smi.latched_init)
5218 set_bit(KVM_APIC_INIT, &vcpu->arch.apic->pending_events);
5219 else
5220 clear_bit(KVM_APIC_INIT, &vcpu->arch.apic->pending_events);
Paolo Bonzinif0778252015-04-01 15:06:40 +02005221 }
5222 }
5223
Chenyi Qianged235112022-05-24 21:56:21 +08005224 if (events->flags & KVM_VCPUEVENT_VALID_TRIPLE_FAULT) {
5225 if (!vcpu->kvm->arch.triple_fault_event)
5226 return -EINVAL;
5227 if (events->triple_fault.pending)
5228 kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
5229 else
5230 kvm_clear_request(KVM_REQ_TRIPLE_FAULT, vcpu);
5231 }
5232
Avi Kivity3842d132010-07-27 12:30:24 +03005233 kvm_make_request(KVM_REQ_EVENT, vcpu);
5234
Jan Kiszka3cfc3092009-11-12 01:04:25 +01005235 return 0;
5236}
5237
Jan Kiszkaa1efbe72010-02-15 10:45:43 +01005238static void kvm_vcpu_ioctl_x86_get_debugregs(struct kvm_vcpu *vcpu,
5239 struct kvm_debugregs *dbgregs)
5240{
Jan Kiszka73aaf249e2014-01-04 18:47:16 +01005241 unsigned long val;
5242
Jan Kiszkaa1efbe72010-02-15 10:45:43 +01005243 memcpy(dbgregs->db, vcpu->arch.db, sizeof(vcpu->arch.db));
Nadav Amit16f8a6f2014-10-03 01:10:05 +03005244 kvm_get_dr(vcpu, 6, &val);
Jan Kiszka73aaf249e2014-01-04 18:47:16 +01005245 dbgregs->dr6 = val;
Jan Kiszkaa1efbe72010-02-15 10:45:43 +01005246 dbgregs->dr7 = vcpu->arch.dr7;
5247 dbgregs->flags = 0;
Vasiliy Kulikov97e69aa2010-10-30 22:54:47 +04005248 memset(&dbgregs->reserved, 0, sizeof(dbgregs->reserved));
Jan Kiszkaa1efbe72010-02-15 10:45:43 +01005249}
5250
5251static int kvm_vcpu_ioctl_x86_set_debugregs(struct kvm_vcpu *vcpu,
5252 struct kvm_debugregs *dbgregs)
5253{
5254 if (dbgregs->flags)
5255 return -EINVAL;
5256
Paolo Bonzinifd238002020-11-13 08:31:09 -05005257 if (!kvm_dr6_valid(dbgregs->dr6))
Paolo Bonzinid14bdb52016-06-01 14:09:23 +02005258 return -EINVAL;
Paolo Bonzinifd238002020-11-13 08:31:09 -05005259 if (!kvm_dr7_valid(dbgregs->dr7))
Paolo Bonzinid14bdb52016-06-01 14:09:23 +02005260 return -EINVAL;
5261
Jan Kiszkaa1efbe72010-02-15 10:45:43 +01005262 memcpy(vcpu->arch.db, dbgregs->db, sizeof(vcpu->arch.db));
Nadav Amitae561ed2015-04-02 03:10:37 +03005263 kvm_update_dr0123(vcpu);
Jan Kiszkaa1efbe72010-02-15 10:45:43 +01005264 vcpu->arch.dr6 = dbgregs->dr6;
5265 vcpu->arch.dr7 = dbgregs->dr7;
Jan Kiszka9926c9f2014-01-04 18:47:15 +01005266 kvm_update_dr7(vcpu);
Jan Kiszkaa1efbe72010-02-15 10:45:43 +01005267
Jan Kiszkaa1efbe72010-02-15 10:45:43 +01005268 return 0;
5269}
5270
Sheng Yang2d5b5a62010-06-13 17:29:39 +08005271static void kvm_vcpu_ioctl_x86_get_xsave(struct kvm_vcpu *vcpu,
5272 struct kvm_xsave *guest_xsave)
5273{
Thomas Gleixnerd69c1382021-10-22 20:55:53 +02005274 if (fpstate_is_confidential(&vcpu->arch.guest_fpu))
Tom Lendackyed02b212020-12-10 11:10:01 -06005275 return;
5276
Thomas Gleixnerd69c1382021-10-22 20:55:53 +02005277 fpu_copy_guest_fpstate_to_uabi(&vcpu->arch.guest_fpu,
5278 guest_xsave->region,
5279 sizeof(guest_xsave->region),
5280 vcpu->arch.pkru);
Sheng Yang2d5b5a62010-06-13 17:29:39 +08005281}
5282
Guang Zengbe50b202022-01-05 04:35:29 -08005283static void kvm_vcpu_ioctl_x86_get_xsave2(struct kvm_vcpu *vcpu,
5284 u8 *state, unsigned int size)
5285{
5286 if (fpstate_is_confidential(&vcpu->arch.guest_fpu))
5287 return;
5288
5289 fpu_copy_guest_fpstate_to_uabi(&vcpu->arch.guest_fpu,
5290 state, size, vcpu->arch.pkru);
5291}
5292
Sheng Yang2d5b5a62010-06-13 17:29:39 +08005293static int kvm_vcpu_ioctl_x86_set_xsave(struct kvm_vcpu *vcpu,
5294 struct kvm_xsave *guest_xsave)
5295{
Thomas Gleixnerd69c1382021-10-22 20:55:53 +02005296 if (fpstate_is_confidential(&vcpu->arch.guest_fpu))
Tom Lendackyed02b212020-12-10 11:10:01 -06005297 return 0;
5298
Thomas Gleixnerd69c1382021-10-22 20:55:53 +02005299 return fpu_copy_uabi_to_guest_fpstate(&vcpu->arch.guest_fpu,
5300 guest_xsave->region,
Sean Christopherson938c8742022-05-24 21:56:23 +08005301 kvm_caps.supported_xcr0,
5302 &vcpu->arch.pkru);
Sheng Yang2d5b5a62010-06-13 17:29:39 +08005303}
5304
5305static void kvm_vcpu_ioctl_x86_get_xcrs(struct kvm_vcpu *vcpu,
5306 struct kvm_xcrs *guest_xcrs)
5307{
Borislav Petkovd366bf72016-04-04 22:25:02 +02005308 if (!boot_cpu_has(X86_FEATURE_XSAVE)) {
Sheng Yang2d5b5a62010-06-13 17:29:39 +08005309 guest_xcrs->nr_xcrs = 0;
5310 return;
5311 }
5312
5313 guest_xcrs->nr_xcrs = 1;
5314 guest_xcrs->flags = 0;
5315 guest_xcrs->xcrs[0].xcr = XCR_XFEATURE_ENABLED_MASK;
5316 guest_xcrs->xcrs[0].value = vcpu->arch.xcr0;
5317}
5318
5319static int kvm_vcpu_ioctl_x86_set_xcrs(struct kvm_vcpu *vcpu,
5320 struct kvm_xcrs *guest_xcrs)
5321{
5322 int i, r = 0;
5323
Borislav Petkovd366bf72016-04-04 22:25:02 +02005324 if (!boot_cpu_has(X86_FEATURE_XSAVE))
Sheng Yang2d5b5a62010-06-13 17:29:39 +08005325 return -EINVAL;
5326
5327 if (guest_xcrs->nr_xcrs > KVM_MAX_XCRS || guest_xcrs->flags)
5328 return -EINVAL;
5329
5330 for (i = 0; i < guest_xcrs->nr_xcrs; i++)
5331 /* Only support XCR0 currently */
Paolo Bonzinic67a04c2013-10-17 16:50:47 +02005332 if (guest_xcrs->xcrs[i].xcr == XCR_XFEATURE_ENABLED_MASK) {
Sheng Yang2d5b5a62010-06-13 17:29:39 +08005333 r = __kvm_set_xcr(vcpu, XCR_XFEATURE_ENABLED_MASK,
Paolo Bonzinic67a04c2013-10-17 16:50:47 +02005334 guest_xcrs->xcrs[i].value);
Sheng Yang2d5b5a62010-06-13 17:29:39 +08005335 break;
5336 }
5337 if (r)
5338 r = -EINVAL;
5339 return r;
5340}
5341
Eric B Munson1c0b28c2012-03-10 14:37:27 -05005342/*
5343 * kvm_set_guest_paused() indicates to the guest kernel that it has been
5344 * stopped by the hypervisor. This function will be called from the host only.
5345 * EINVAL is returned when the host attempts to set the flag for a guest that
5346 * does not support pv clocks.
5347 */
5348static int kvm_set_guest_paused(struct kvm_vcpu *vcpu)
5349{
David Woodhouse916d3602022-03-03 15:41:14 +00005350 if (!vcpu->arch.pv_time.active)
Eric B Munson1c0b28c2012-03-10 14:37:27 -05005351 return -EINVAL;
Marcelo Tosatti51d59c62012-08-03 15:57:49 -03005352 vcpu->arch.pvclock_set_guest_stopped_request = true;
Eric B Munson1c0b28c2012-03-10 14:37:27 -05005353 kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
5354 return 0;
5355}
5356
Oliver Upton828ca892021-09-16 18:15:38 +00005357static int kvm_arch_tsc_has_attr(struct kvm_vcpu *vcpu,
5358 struct kvm_device_attr *attr)
5359{
5360 int r;
5361
5362 switch (attr->attr) {
5363 case KVM_VCPU_TSC_OFFSET:
5364 r = 0;
5365 break;
5366 default:
5367 r = -ENXIO;
5368 }
5369
5370 return r;
5371}
5372
5373static int kvm_arch_tsc_get_attr(struct kvm_vcpu *vcpu,
5374 struct kvm_device_attr *attr)
5375{
Sean Christopherson56f289a2022-01-27 07:31:53 -08005376 u64 __user *uaddr = kvm_get_attr_addr(attr);
Oliver Upton828ca892021-09-16 18:15:38 +00005377 int r;
5378
Sean Christopherson56f289a2022-01-27 07:31:53 -08005379 if (IS_ERR(uaddr))
5380 return PTR_ERR(uaddr);
Oliver Upton828ca892021-09-16 18:15:38 +00005381
5382 switch (attr->attr) {
5383 case KVM_VCPU_TSC_OFFSET:
5384 r = -EFAULT;
5385 if (put_user(vcpu->arch.l1_tsc_offset, uaddr))
5386 break;
5387 r = 0;
5388 break;
5389 default:
5390 r = -ENXIO;
5391 }
5392
5393 return r;
5394}
5395
5396static int kvm_arch_tsc_set_attr(struct kvm_vcpu *vcpu,
5397 struct kvm_device_attr *attr)
5398{
Sean Christopherson56f289a2022-01-27 07:31:53 -08005399 u64 __user *uaddr = kvm_get_attr_addr(attr);
Oliver Upton828ca892021-09-16 18:15:38 +00005400 struct kvm *kvm = vcpu->kvm;
5401 int r;
5402
Sean Christopherson56f289a2022-01-27 07:31:53 -08005403 if (IS_ERR(uaddr))
5404 return PTR_ERR(uaddr);
Oliver Upton828ca892021-09-16 18:15:38 +00005405
5406 switch (attr->attr) {
5407 case KVM_VCPU_TSC_OFFSET: {
5408 u64 offset, tsc, ns;
5409 unsigned long flags;
5410 bool matched;
5411
5412 r = -EFAULT;
5413 if (get_user(offset, uaddr))
5414 break;
5415
5416 raw_spin_lock_irqsave(&kvm->arch.tsc_write_lock, flags);
5417
5418 matched = (vcpu->arch.virtual_tsc_khz &&
5419 kvm->arch.last_tsc_khz == vcpu->arch.virtual_tsc_khz &&
5420 kvm->arch.last_tsc_offset == offset);
5421
Jinrong Liang62711e52022-01-25 17:59:07 +08005422 tsc = kvm_scale_tsc(rdtsc(), vcpu->arch.l1_tsc_scaling_ratio) + offset;
Oliver Upton828ca892021-09-16 18:15:38 +00005423 ns = get_kvmclock_base_ns();
5424
5425 __kvm_synchronize_tsc(vcpu, offset, tsc, ns, matched);
5426 raw_spin_unlock_irqrestore(&kvm->arch.tsc_write_lock, flags);
5427
5428 r = 0;
5429 break;
5430 }
5431 default:
5432 r = -ENXIO;
5433 }
5434
5435 return r;
5436}
5437
5438static int kvm_vcpu_ioctl_device_attr(struct kvm_vcpu *vcpu,
5439 unsigned int ioctl,
5440 void __user *argp)
5441{
5442 struct kvm_device_attr attr;
5443 int r;
5444
5445 if (copy_from_user(&attr, argp, sizeof(attr)))
5446 return -EFAULT;
5447
5448 if (attr.group != KVM_VCPU_TSC_CTRL)
5449 return -ENXIO;
5450
5451 switch (ioctl) {
5452 case KVM_HAS_DEVICE_ATTR:
5453 r = kvm_arch_tsc_has_attr(vcpu, &attr);
5454 break;
5455 case KVM_GET_DEVICE_ATTR:
5456 r = kvm_arch_tsc_get_attr(vcpu, &attr);
5457 break;
5458 case KVM_SET_DEVICE_ATTR:
5459 r = kvm_arch_tsc_set_attr(vcpu, &attr);
5460 break;
5461 }
5462
5463 return r;
5464}
5465
Andrey Smetanin5c9194122015-11-10 15:36:34 +03005466static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
5467 struct kvm_enable_cap *cap)
5468{
Vitaly Kuznetsov57b119d2018-10-16 18:50:01 +02005469 int r;
5470 uint16_t vmcs_version;
5471 void __user *user_ptr;
5472
Andrey Smetanin5c9194122015-11-10 15:36:34 +03005473 if (cap->flags)
5474 return -EINVAL;
5475
5476 switch (cap->cap) {
Roman Kaganefc479e2017-06-22 16:51:01 +03005477 case KVM_CAP_HYPERV_SYNIC2:
5478 if (cap->args[0])
5479 return -EINVAL;
Gustavo A. R. Silvadf561f662020-08-23 17:36:59 -05005480 fallthrough;
Gustavo A. R. Silvab2869f22019-01-25 12:23:17 -06005481
Andrey Smetanin5c9194122015-11-10 15:36:34 +03005482 case KVM_CAP_HYPERV_SYNIC:
Wanpeng Li546d87e2017-01-03 18:56:19 -08005483 if (!irqchip_in_kernel(vcpu->kvm))
5484 return -EINVAL;
Roman Kaganefc479e2017-06-22 16:51:01 +03005485 return kvm_hv_activate_synic(vcpu, cap->cap ==
5486 KVM_CAP_HYPERV_SYNIC2);
Vitaly Kuznetsov57b119d2018-10-16 18:50:01 +02005487 case KVM_CAP_HYPERV_ENLIGHTENED_VMCS:
Paolo Bonzini33b22172020-04-17 10:24:18 -04005488 if (!kvm_x86_ops.nested_ops->enable_evmcs)
Sean Christopherson51589172018-12-03 13:53:10 -08005489 return -ENOTTY;
Paolo Bonzini33b22172020-04-17 10:24:18 -04005490 r = kvm_x86_ops.nested_ops->enable_evmcs(vcpu, &vmcs_version);
Vitaly Kuznetsov57b119d2018-10-16 18:50:01 +02005491 if (!r) {
5492 user_ptr = (void __user *)(uintptr_t)cap->args[0];
5493 if (copy_to_user(user_ptr, &vmcs_version,
5494 sizeof(vmcs_version)))
5495 r = -EFAULT;
5496 }
5497 return r;
Tianyu Lan344c6c82019-08-22 22:30:20 +08005498 case KVM_CAP_HYPERV_DIRECT_TLBFLUSH:
Sean Christophersonafaf0b22020-03-21 13:26:00 -07005499 if (!kvm_x86_ops.enable_direct_tlbflush)
Tianyu Lan344c6c82019-08-22 22:30:20 +08005500 return -ENOTTY;
5501
Jason Baronb36464772021-01-14 22:27:56 -05005502 return static_call(kvm_x86_enable_direct_tlbflush)(vcpu);
Vitaly Kuznetsov57b119d2018-10-16 18:50:01 +02005503
Vitaly Kuznetsov644f7062021-05-21 11:51:36 +02005504 case KVM_CAP_HYPERV_ENFORCE_CPUID:
5505 return kvm_hv_set_enforce_cpuid(vcpu, cap->args[0]);
5506
Oliver Upton66570e962020-08-18 15:24:28 +00005507 case KVM_CAP_ENFORCE_PV_FEATURE_CPUID:
5508 vcpu->arch.pv_cpuid.enforce = cap->args[0];
Oliver Upton01b4f512020-10-27 16:10:42 -07005509 if (vcpu->arch.pv_cpuid.enforce)
5510 kvm_update_pv_runtime(vcpu);
Oliver Upton66570e962020-08-18 15:24:28 +00005511
5512 return 0;
Andrey Smetanin5c9194122015-11-10 15:36:34 +03005513 default:
5514 return -EINVAL;
5515 }
5516}
5517
Carsten Otte313a3dc2007-10-11 19:16:52 +02005518long kvm_arch_vcpu_ioctl(struct file *filp,
5519 unsigned int ioctl, unsigned long arg)
5520{
5521 struct kvm_vcpu *vcpu = filp->private_data;
5522 void __user *argp = (void __user *)arg;
5523 int r;
Avi Kivityd1ac91d2010-06-20 15:54:43 +03005524 union {
Maxim Levitsky6dba9402021-06-07 12:02:02 +03005525 struct kvm_sregs2 *sregs2;
Avi Kivityd1ac91d2010-06-20 15:54:43 +03005526 struct kvm_lapic_state *lapic;
5527 struct kvm_xsave *xsave;
5528 struct kvm_xcrs *xcrs;
5529 void *buffer;
5530 } u;
Carsten Otte313a3dc2007-10-11 19:16:52 +02005531
Christoffer Dall9b0624712017-12-04 21:35:36 +01005532 vcpu_load(vcpu);
5533
Avi Kivityd1ac91d2010-06-20 15:54:43 +03005534 u.buffer = NULL;
Carsten Otte313a3dc2007-10-11 19:16:52 +02005535 switch (ioctl) {
5536 case KVM_GET_LAPIC: {
Marcelo Tosatti2204ae32009-10-29 13:44:16 -02005537 r = -EINVAL;
Paolo Bonzinibce87cc2016-01-08 13:48:51 +01005538 if (!lapic_in_kernel(vcpu))
Marcelo Tosatti2204ae32009-10-29 13:44:16 -02005539 goto out;
Ben Gardon254272c2019-02-11 11:02:50 -08005540 u.lapic = kzalloc(sizeof(struct kvm_lapic_state),
5541 GFP_KERNEL_ACCOUNT);
Carsten Otte313a3dc2007-10-11 19:16:52 +02005542
Dave Hansenb772ff32008-08-11 10:01:47 -07005543 r = -ENOMEM;
Avi Kivityd1ac91d2010-06-20 15:54:43 +03005544 if (!u.lapic)
Dave Hansenb772ff32008-08-11 10:01:47 -07005545 goto out;
Avi Kivityd1ac91d2010-06-20 15:54:43 +03005546 r = kvm_vcpu_ioctl_get_lapic(vcpu, u.lapic);
Carsten Otte313a3dc2007-10-11 19:16:52 +02005547 if (r)
5548 goto out;
5549 r = -EFAULT;
Avi Kivityd1ac91d2010-06-20 15:54:43 +03005550 if (copy_to_user(argp, u.lapic, sizeof(struct kvm_lapic_state)))
Carsten Otte313a3dc2007-10-11 19:16:52 +02005551 goto out;
5552 r = 0;
5553 break;
5554 }
5555 case KVM_SET_LAPIC: {
Marcelo Tosatti2204ae32009-10-29 13:44:16 -02005556 r = -EINVAL;
Paolo Bonzinibce87cc2016-01-08 13:48:51 +01005557 if (!lapic_in_kernel(vcpu))
Marcelo Tosatti2204ae32009-10-29 13:44:16 -02005558 goto out;
Sasha Levinff5c2c02011-12-04 19:36:29 +02005559 u.lapic = memdup_user(argp, sizeof(*u.lapic));
Christoffer Dall9b0624712017-12-04 21:35:36 +01005560 if (IS_ERR(u.lapic)) {
5561 r = PTR_ERR(u.lapic);
5562 goto out_nofree;
5563 }
Sasha Levinff5c2c02011-12-04 19:36:29 +02005564
Avi Kivityd1ac91d2010-06-20 15:54:43 +03005565 r = kvm_vcpu_ioctl_set_lapic(vcpu, u.lapic);
Carsten Otte313a3dc2007-10-11 19:16:52 +02005566 break;
5567 }
Zhang Xiantaof77bc6a2007-11-21 04:36:41 +08005568 case KVM_INTERRUPT: {
5569 struct kvm_interrupt irq;
5570
5571 r = -EFAULT;
Jordan Borgner0e96f312018-10-28 12:58:28 +00005572 if (copy_from_user(&irq, argp, sizeof(irq)))
Zhang Xiantaof77bc6a2007-11-21 04:36:41 +08005573 goto out;
5574 r = kvm_vcpu_ioctl_interrupt(vcpu, &irq);
Zhang Xiantaof77bc6a2007-11-21 04:36:41 +08005575 break;
5576 }
Jan Kiszkac4abb7c2008-09-26 09:30:55 +02005577 case KVM_NMI: {
5578 r = kvm_vcpu_ioctl_nmi(vcpu);
Jan Kiszkac4abb7c2008-09-26 09:30:55 +02005579 break;
5580 }
Paolo Bonzinif0778252015-04-01 15:06:40 +02005581 case KVM_SMI: {
5582 r = kvm_vcpu_ioctl_smi(vcpu);
5583 break;
5584 }
Carsten Otte313a3dc2007-10-11 19:16:52 +02005585 case KVM_SET_CPUID: {
5586 struct kvm_cpuid __user *cpuid_arg = argp;
5587 struct kvm_cpuid cpuid;
5588
5589 r = -EFAULT;
Jordan Borgner0e96f312018-10-28 12:58:28 +00005590 if (copy_from_user(&cpuid, cpuid_arg, sizeof(cpuid)))
Carsten Otte313a3dc2007-10-11 19:16:52 +02005591 goto out;
5592 r = kvm_vcpu_ioctl_set_cpuid(vcpu, &cpuid, cpuid_arg->entries);
Carsten Otte313a3dc2007-10-11 19:16:52 +02005593 break;
5594 }
Dan Kenigsberg07716712007-11-21 17:10:04 +02005595 case KVM_SET_CPUID2: {
5596 struct kvm_cpuid2 __user *cpuid_arg = argp;
5597 struct kvm_cpuid2 cpuid;
5598
5599 r = -EFAULT;
Jordan Borgner0e96f312018-10-28 12:58:28 +00005600 if (copy_from_user(&cpuid, cpuid_arg, sizeof(cpuid)))
Dan Kenigsberg07716712007-11-21 17:10:04 +02005601 goto out;
5602 r = kvm_vcpu_ioctl_set_cpuid2(vcpu, &cpuid,
Amit Shah19355472009-01-14 16:56:00 +00005603 cpuid_arg->entries);
Dan Kenigsberg07716712007-11-21 17:10:04 +02005604 break;
5605 }
5606 case KVM_GET_CPUID2: {
5607 struct kvm_cpuid2 __user *cpuid_arg = argp;
5608 struct kvm_cpuid2 cpuid;
5609
5610 r = -EFAULT;
Jordan Borgner0e96f312018-10-28 12:58:28 +00005611 if (copy_from_user(&cpuid, cpuid_arg, sizeof(cpuid)))
Dan Kenigsberg07716712007-11-21 17:10:04 +02005612 goto out;
5613 r = kvm_vcpu_ioctl_get_cpuid2(vcpu, &cpuid,
Amit Shah19355472009-01-14 16:56:00 +00005614 cpuid_arg->entries);
Dan Kenigsberg07716712007-11-21 17:10:04 +02005615 if (r)
5616 goto out;
5617 r = -EFAULT;
Jordan Borgner0e96f312018-10-28 12:58:28 +00005618 if (copy_to_user(cpuid_arg, &cpuid, sizeof(cpuid)))
Dan Kenigsberg07716712007-11-21 17:10:04 +02005619 goto out;
5620 r = 0;
5621 break;
5622 }
Tom Lendacky801e4592018-02-21 13:39:51 -06005623 case KVM_GET_MSRS: {
5624 int idx = srcu_read_lock(&vcpu->kvm->srcu);
Paolo Bonzini609e36d2015-04-08 15:30:38 +02005625 r = msr_io(vcpu, argp, do_get_msr, 1);
Tom Lendacky801e4592018-02-21 13:39:51 -06005626 srcu_read_unlock(&vcpu->kvm->srcu, idx);
Carsten Otte313a3dc2007-10-11 19:16:52 +02005627 break;
Tom Lendacky801e4592018-02-21 13:39:51 -06005628 }
5629 case KVM_SET_MSRS: {
5630 int idx = srcu_read_lock(&vcpu->kvm->srcu);
Carsten Otte313a3dc2007-10-11 19:16:52 +02005631 r = msr_io(vcpu, argp, do_set_msr, 0);
Tom Lendacky801e4592018-02-21 13:39:51 -06005632 srcu_read_unlock(&vcpu->kvm->srcu, idx);
Carsten Otte313a3dc2007-10-11 19:16:52 +02005633 break;
Tom Lendacky801e4592018-02-21 13:39:51 -06005634 }
Avi Kivityb209749f2007-10-22 16:50:39 +02005635 case KVM_TPR_ACCESS_REPORTING: {
5636 struct kvm_tpr_access_ctl tac;
5637
5638 r = -EFAULT;
Jordan Borgner0e96f312018-10-28 12:58:28 +00005639 if (copy_from_user(&tac, argp, sizeof(tac)))
Avi Kivityb209749f2007-10-22 16:50:39 +02005640 goto out;
5641 r = vcpu_ioctl_tpr_access_reporting(vcpu, &tac);
5642 if (r)
5643 goto out;
5644 r = -EFAULT;
Jordan Borgner0e96f312018-10-28 12:58:28 +00005645 if (copy_to_user(argp, &tac, sizeof(tac)))
Avi Kivityb209749f2007-10-22 16:50:39 +02005646 goto out;
5647 r = 0;
5648 break;
5649 };
Avi Kivityb93463a2007-10-25 16:52:32 +02005650 case KVM_SET_VAPIC_ADDR: {
5651 struct kvm_vapic_addr va;
Paolo Bonzini7301d6a2016-11-17 15:55:46 +01005652 int idx;
Avi Kivityb93463a2007-10-25 16:52:32 +02005653
5654 r = -EINVAL;
Paolo Bonzini35754c92015-07-29 12:05:37 +02005655 if (!lapic_in_kernel(vcpu))
Avi Kivityb93463a2007-10-25 16:52:32 +02005656 goto out;
5657 r = -EFAULT;
Jordan Borgner0e96f312018-10-28 12:58:28 +00005658 if (copy_from_user(&va, argp, sizeof(va)))
Avi Kivityb93463a2007-10-25 16:52:32 +02005659 goto out;
Paolo Bonzini7301d6a2016-11-17 15:55:46 +01005660 idx = srcu_read_lock(&vcpu->kvm->srcu);
Andy Honigfda4e2e2013-11-20 10:23:22 -08005661 r = kvm_lapic_set_vapic_addr(vcpu, va.vapic_addr);
Paolo Bonzini7301d6a2016-11-17 15:55:46 +01005662 srcu_read_unlock(&vcpu->kvm->srcu, idx);
Avi Kivityb93463a2007-10-25 16:52:32 +02005663 break;
5664 }
Huang Ying890ca9a2009-05-11 16:48:15 +08005665 case KVM_X86_SETUP_MCE: {
5666 u64 mcg_cap;
5667
5668 r = -EFAULT;
Jordan Borgner0e96f312018-10-28 12:58:28 +00005669 if (copy_from_user(&mcg_cap, argp, sizeof(mcg_cap)))
Huang Ying890ca9a2009-05-11 16:48:15 +08005670 goto out;
5671 r = kvm_vcpu_ioctl_x86_setup_mce(vcpu, mcg_cap);
5672 break;
5673 }
5674 case KVM_X86_SET_MCE: {
5675 struct kvm_x86_mce mce;
5676
5677 r = -EFAULT;
Jordan Borgner0e96f312018-10-28 12:58:28 +00005678 if (copy_from_user(&mce, argp, sizeof(mce)))
Huang Ying890ca9a2009-05-11 16:48:15 +08005679 goto out;
5680 r = kvm_vcpu_ioctl_x86_set_mce(vcpu, &mce);
5681 break;
5682 }
Jan Kiszka3cfc3092009-11-12 01:04:25 +01005683 case KVM_GET_VCPU_EVENTS: {
5684 struct kvm_vcpu_events events;
5685
5686 kvm_vcpu_ioctl_x86_get_vcpu_events(vcpu, &events);
5687
5688 r = -EFAULT;
5689 if (copy_to_user(argp, &events, sizeof(struct kvm_vcpu_events)))
5690 break;
5691 r = 0;
5692 break;
5693 }
5694 case KVM_SET_VCPU_EVENTS: {
5695 struct kvm_vcpu_events events;
5696
5697 r = -EFAULT;
5698 if (copy_from_user(&events, argp, sizeof(struct kvm_vcpu_events)))
5699 break;
5700
5701 r = kvm_vcpu_ioctl_x86_set_vcpu_events(vcpu, &events);
5702 break;
5703 }
Jan Kiszkaa1efbe72010-02-15 10:45:43 +01005704 case KVM_GET_DEBUGREGS: {
5705 struct kvm_debugregs dbgregs;
5706
5707 kvm_vcpu_ioctl_x86_get_debugregs(vcpu, &dbgregs);
5708
5709 r = -EFAULT;
5710 if (copy_to_user(argp, &dbgregs,
5711 sizeof(struct kvm_debugregs)))
5712 break;
5713 r = 0;
5714 break;
5715 }
5716 case KVM_SET_DEBUGREGS: {
5717 struct kvm_debugregs dbgregs;
5718
5719 r = -EFAULT;
5720 if (copy_from_user(&dbgregs, argp,
5721 sizeof(struct kvm_debugregs)))
5722 break;
5723
5724 r = kvm_vcpu_ioctl_x86_set_debugregs(vcpu, &dbgregs);
5725 break;
5726 }
Sheng Yang2d5b5a62010-06-13 17:29:39 +08005727 case KVM_GET_XSAVE: {
Guang Zengbe50b202022-01-05 04:35:29 -08005728 r = -EINVAL;
5729 if (vcpu->arch.guest_fpu.uabi_size > sizeof(struct kvm_xsave))
5730 break;
5731
Ben Gardon254272c2019-02-11 11:02:50 -08005732 u.xsave = kzalloc(sizeof(struct kvm_xsave), GFP_KERNEL_ACCOUNT);
Sheng Yang2d5b5a62010-06-13 17:29:39 +08005733 r = -ENOMEM;
Avi Kivityd1ac91d2010-06-20 15:54:43 +03005734 if (!u.xsave)
Sheng Yang2d5b5a62010-06-13 17:29:39 +08005735 break;
5736
Avi Kivityd1ac91d2010-06-20 15:54:43 +03005737 kvm_vcpu_ioctl_x86_get_xsave(vcpu, u.xsave);
Sheng Yang2d5b5a62010-06-13 17:29:39 +08005738
5739 r = -EFAULT;
Avi Kivityd1ac91d2010-06-20 15:54:43 +03005740 if (copy_to_user(argp, u.xsave, sizeof(struct kvm_xsave)))
Sheng Yang2d5b5a62010-06-13 17:29:39 +08005741 break;
5742 r = 0;
5743 break;
5744 }
5745 case KVM_SET_XSAVE: {
Guang Zengbe50b202022-01-05 04:35:29 -08005746 int size = vcpu->arch.guest_fpu.uabi_size;
5747
5748 u.xsave = memdup_user(argp, size);
Christoffer Dall9b0624712017-12-04 21:35:36 +01005749 if (IS_ERR(u.xsave)) {
5750 r = PTR_ERR(u.xsave);
5751 goto out_nofree;
5752 }
Sheng Yang2d5b5a62010-06-13 17:29:39 +08005753
Avi Kivityd1ac91d2010-06-20 15:54:43 +03005754 r = kvm_vcpu_ioctl_x86_set_xsave(vcpu, u.xsave);
Sheng Yang2d5b5a62010-06-13 17:29:39 +08005755 break;
5756 }
Guang Zengbe50b202022-01-05 04:35:29 -08005757
5758 case KVM_GET_XSAVE2: {
5759 int size = vcpu->arch.guest_fpu.uabi_size;
5760
5761 u.xsave = kzalloc(size, GFP_KERNEL_ACCOUNT);
5762 r = -ENOMEM;
5763 if (!u.xsave)
5764 break;
5765
5766 kvm_vcpu_ioctl_x86_get_xsave2(vcpu, u.buffer, size);
5767
5768 r = -EFAULT;
5769 if (copy_to_user(argp, u.xsave, size))
5770 break;
5771
5772 r = 0;
5773 break;
5774 }
5775
Sheng Yang2d5b5a62010-06-13 17:29:39 +08005776 case KVM_GET_XCRS: {
Ben Gardon254272c2019-02-11 11:02:50 -08005777 u.xcrs = kzalloc(sizeof(struct kvm_xcrs), GFP_KERNEL_ACCOUNT);
Sheng Yang2d5b5a62010-06-13 17:29:39 +08005778 r = -ENOMEM;
Avi Kivityd1ac91d2010-06-20 15:54:43 +03005779 if (!u.xcrs)
Sheng Yang2d5b5a62010-06-13 17:29:39 +08005780 break;
5781
Avi Kivityd1ac91d2010-06-20 15:54:43 +03005782 kvm_vcpu_ioctl_x86_get_xcrs(vcpu, u.xcrs);
Sheng Yang2d5b5a62010-06-13 17:29:39 +08005783
5784 r = -EFAULT;
Avi Kivityd1ac91d2010-06-20 15:54:43 +03005785 if (copy_to_user(argp, u.xcrs,
Sheng Yang2d5b5a62010-06-13 17:29:39 +08005786 sizeof(struct kvm_xcrs)))
5787 break;
5788 r = 0;
5789 break;
5790 }
5791 case KVM_SET_XCRS: {
Sasha Levinff5c2c02011-12-04 19:36:29 +02005792 u.xcrs = memdup_user(argp, sizeof(*u.xcrs));
Christoffer Dall9b0624712017-12-04 21:35:36 +01005793 if (IS_ERR(u.xcrs)) {
5794 r = PTR_ERR(u.xcrs);
5795 goto out_nofree;
5796 }
Sheng Yang2d5b5a62010-06-13 17:29:39 +08005797
Avi Kivityd1ac91d2010-06-20 15:54:43 +03005798 r = kvm_vcpu_ioctl_x86_set_xcrs(vcpu, u.xcrs);
Sheng Yang2d5b5a62010-06-13 17:29:39 +08005799 break;
5800 }
Joerg Roedel92a1f122011-03-25 09:44:51 +01005801 case KVM_SET_TSC_KHZ: {
5802 u32 user_tsc_khz;
5803
5804 r = -EINVAL;
Joerg Roedel92a1f122011-03-25 09:44:51 +01005805 user_tsc_khz = (u32)arg;
5806
Sean Christopherson938c8742022-05-24 21:56:23 +08005807 if (kvm_caps.has_tsc_control &&
5808 user_tsc_khz >= kvm_caps.max_guest_tsc_khz)
Joerg Roedel92a1f122011-03-25 09:44:51 +01005809 goto out;
5810
Zachary Amsdencc578282012-02-03 15:43:50 -02005811 if (user_tsc_khz == 0)
5812 user_tsc_khz = tsc_khz;
5813
Haozhong Zhang381d5852015-10-20 15:39:04 +08005814 if (!kvm_set_tsc_khz(vcpu, user_tsc_khz))
5815 r = 0;
Joerg Roedel92a1f122011-03-25 09:44:51 +01005816
Joerg Roedel92a1f122011-03-25 09:44:51 +01005817 goto out;
5818 }
5819 case KVM_GET_TSC_KHZ: {
Zachary Amsdencc578282012-02-03 15:43:50 -02005820 r = vcpu->arch.virtual_tsc_khz;
Joerg Roedel92a1f122011-03-25 09:44:51 +01005821 goto out;
5822 }
Eric B Munson1c0b28c2012-03-10 14:37:27 -05005823 case KVM_KVMCLOCK_CTRL: {
5824 r = kvm_set_guest_paused(vcpu);
5825 goto out;
5826 }
Andrey Smetanin5c9194122015-11-10 15:36:34 +03005827 case KVM_ENABLE_CAP: {
5828 struct kvm_enable_cap cap;
5829
5830 r = -EFAULT;
5831 if (copy_from_user(&cap, argp, sizeof(cap)))
5832 goto out;
5833 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
5834 break;
5835 }
Jim Mattson8fcc4b52018-07-10 11:27:20 +02005836 case KVM_GET_NESTED_STATE: {
5837 struct kvm_nested_state __user *user_kvm_nested_state = argp;
5838 u32 user_data_size;
5839
5840 r = -EINVAL;
Paolo Bonzini33b22172020-04-17 10:24:18 -04005841 if (!kvm_x86_ops.nested_ops->get_state)
Jim Mattson8fcc4b52018-07-10 11:27:20 +02005842 break;
5843
5844 BUILD_BUG_ON(sizeof(user_data_size) != sizeof(user_kvm_nested_state->size));
Liran Alon26b471c2018-09-16 14:28:20 +03005845 r = -EFAULT;
Jim Mattson8fcc4b52018-07-10 11:27:20 +02005846 if (get_user(user_data_size, &user_kvm_nested_state->size))
Liran Alon26b471c2018-09-16 14:28:20 +03005847 break;
Jim Mattson8fcc4b52018-07-10 11:27:20 +02005848
Paolo Bonzini33b22172020-04-17 10:24:18 -04005849 r = kvm_x86_ops.nested_ops->get_state(vcpu, user_kvm_nested_state,
5850 user_data_size);
Jim Mattson8fcc4b52018-07-10 11:27:20 +02005851 if (r < 0)
Liran Alon26b471c2018-09-16 14:28:20 +03005852 break;
Jim Mattson8fcc4b52018-07-10 11:27:20 +02005853
5854 if (r > user_data_size) {
5855 if (put_user(r, &user_kvm_nested_state->size))
Liran Alon26b471c2018-09-16 14:28:20 +03005856 r = -EFAULT;
5857 else
5858 r = -E2BIG;
5859 break;
Jim Mattson8fcc4b52018-07-10 11:27:20 +02005860 }
Liran Alon26b471c2018-09-16 14:28:20 +03005861
Jim Mattson8fcc4b52018-07-10 11:27:20 +02005862 r = 0;
5863 break;
5864 }
5865 case KVM_SET_NESTED_STATE: {
5866 struct kvm_nested_state __user *user_kvm_nested_state = argp;
5867 struct kvm_nested_state kvm_state;
Sean Christophersonad5996d2019-11-22 08:58:18 -08005868 int idx;
Jim Mattson8fcc4b52018-07-10 11:27:20 +02005869
5870 r = -EINVAL;
Paolo Bonzini33b22172020-04-17 10:24:18 -04005871 if (!kvm_x86_ops.nested_ops->set_state)
Jim Mattson8fcc4b52018-07-10 11:27:20 +02005872 break;
5873
Liran Alon26b471c2018-09-16 14:28:20 +03005874 r = -EFAULT;
Jim Mattson8fcc4b52018-07-10 11:27:20 +02005875 if (copy_from_user(&kvm_state, user_kvm_nested_state, sizeof(kvm_state)))
Liran Alon26b471c2018-09-16 14:28:20 +03005876 break;
Jim Mattson8fcc4b52018-07-10 11:27:20 +02005877
Liran Alon26b471c2018-09-16 14:28:20 +03005878 r = -EINVAL;
Jim Mattson8fcc4b52018-07-10 11:27:20 +02005879 if (kvm_state.size < sizeof(kvm_state))
Liran Alon26b471c2018-09-16 14:28:20 +03005880 break;
Jim Mattson8fcc4b52018-07-10 11:27:20 +02005881
5882 if (kvm_state.flags &
Vitaly Kuznetsov8cab6502018-10-16 18:50:09 +02005883 ~(KVM_STATE_NESTED_RUN_PENDING | KVM_STATE_NESTED_GUEST_MODE
Paolo Bonzinicc440cd2020-05-13 13:36:32 -04005884 | KVM_STATE_NESTED_EVMCS | KVM_STATE_NESTED_MTF_PENDING
5885 | KVM_STATE_NESTED_GIF_SET))
Liran Alon26b471c2018-09-16 14:28:20 +03005886 break;
Jim Mattson8fcc4b52018-07-10 11:27:20 +02005887
5888 /* nested_run_pending implies guest_mode. */
Vitaly Kuznetsov8cab6502018-10-16 18:50:09 +02005889 if ((kvm_state.flags & KVM_STATE_NESTED_RUN_PENDING)
5890 && !(kvm_state.flags & KVM_STATE_NESTED_GUEST_MODE))
Liran Alon26b471c2018-09-16 14:28:20 +03005891 break;
Jim Mattson8fcc4b52018-07-10 11:27:20 +02005892
Sean Christophersonad5996d2019-11-22 08:58:18 -08005893 idx = srcu_read_lock(&vcpu->kvm->srcu);
Paolo Bonzini33b22172020-04-17 10:24:18 -04005894 r = kvm_x86_ops.nested_ops->set_state(vcpu, user_kvm_nested_state, &kvm_state);
Sean Christophersonad5996d2019-11-22 08:58:18 -08005895 srcu_read_unlock(&vcpu->kvm->srcu, idx);
Jim Mattson8fcc4b52018-07-10 11:27:20 +02005896 break;
5897 }
Vitaly Kuznetsovc21d54f2020-09-29 17:09:43 +02005898 case KVM_GET_SUPPORTED_HV_CPUID:
5899 r = kvm_ioctl_get_supported_hv_cpuid(vcpu, argp);
Vitaly Kuznetsov2bc39972018-12-10 18:21:56 +01005900 break;
Paolo Bonzinib59b1532021-02-26 04:54:45 -05005901#ifdef CONFIG_KVM_XEN
David Woodhouse3e324612021-02-02 16:53:25 +00005902 case KVM_XEN_VCPU_GET_ATTR: {
5903 struct kvm_xen_vcpu_attr xva;
5904
5905 r = -EFAULT;
5906 if (copy_from_user(&xva, argp, sizeof(xva)))
5907 goto out;
5908 r = kvm_xen_vcpu_get_attr(vcpu, &xva);
5909 if (!r && copy_to_user(argp, &xva, sizeof(xva)))
5910 r = -EFAULT;
5911 break;
5912 }
5913 case KVM_XEN_VCPU_SET_ATTR: {
5914 struct kvm_xen_vcpu_attr xva;
5915
5916 r = -EFAULT;
5917 if (copy_from_user(&xva, argp, sizeof(xva)))
5918 goto out;
5919 r = kvm_xen_vcpu_set_attr(vcpu, &xva);
5920 break;
5921 }
Paolo Bonzinib59b1532021-02-26 04:54:45 -05005922#endif
Maxim Levitsky6dba9402021-06-07 12:02:02 +03005923 case KVM_GET_SREGS2: {
5924 u.sregs2 = kzalloc(sizeof(struct kvm_sregs2), GFP_KERNEL);
5925 r = -ENOMEM;
5926 if (!u.sregs2)
5927 goto out;
5928 __get_sregs2(vcpu, u.sregs2);
5929 r = -EFAULT;
5930 if (copy_to_user(argp, u.sregs2, sizeof(struct kvm_sregs2)))
5931 goto out;
5932 r = 0;
5933 break;
5934 }
5935 case KVM_SET_SREGS2: {
5936 u.sregs2 = memdup_user(argp, sizeof(struct kvm_sregs2));
5937 if (IS_ERR(u.sregs2)) {
5938 r = PTR_ERR(u.sregs2);
5939 u.sregs2 = NULL;
5940 goto out;
5941 }
5942 r = __set_sregs2(vcpu, u.sregs2);
5943 break;
5944 }
Oliver Upton828ca892021-09-16 18:15:38 +00005945 case KVM_HAS_DEVICE_ATTR:
5946 case KVM_GET_DEVICE_ATTR:
5947 case KVM_SET_DEVICE_ATTR:
5948 r = kvm_vcpu_ioctl_device_attr(vcpu, ioctl, argp);
5949 break;
Carsten Otte313a3dc2007-10-11 19:16:52 +02005950 default:
5951 r = -EINVAL;
5952 }
5953out:
Avi Kivityd1ac91d2010-06-20 15:54:43 +03005954 kfree(u.buffer);
Christoffer Dall9b0624712017-12-04 21:35:36 +01005955out_nofree:
5956 vcpu_put(vcpu);
Carsten Otte313a3dc2007-10-11 19:16:52 +02005957 return r;
5958}
5959
Souptick Joarder1499fa82018-04-19 00:49:58 +05305960vm_fault_t kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
Carsten Otte5b1c1492012-01-04 10:25:23 +01005961{
5962 return VM_FAULT_SIGBUS;
5963}
5964
Carsten Otte1fe779f2007-10-29 16:08:35 +01005965static int kvm_vm_ioctl_set_tss_addr(struct kvm *kvm, unsigned long addr)
5966{
5967 int ret;
5968
5969 if (addr > (unsigned int)(-3 * PAGE_SIZE))
Guo Chao951179c2012-11-02 18:33:22 +08005970 return -EINVAL;
Jason Baronb36464772021-01-14 22:27:56 -05005971 ret = static_call(kvm_x86_set_tss_addr)(kvm, addr);
Carsten Otte1fe779f2007-10-29 16:08:35 +01005972 return ret;
5973}
5974
Sheng Yangb927a3c2009-07-21 10:42:48 +08005975static int kvm_vm_ioctl_set_identity_map_addr(struct kvm *kvm,
5976 u64 ident_addr)
5977{
Jason Baronb36464772021-01-14 22:27:56 -05005978 return static_call(kvm_x86_set_identity_map_addr)(kvm, ident_addr);
Sheng Yangb927a3c2009-07-21 10:42:48 +08005979}
5980
Carsten Otte1fe779f2007-10-29 16:08:35 +01005981static int kvm_vm_ioctl_set_nr_mmu_pages(struct kvm *kvm,
Ben Gardonbc8a3d82019-04-08 11:07:30 -07005982 unsigned long kvm_nr_mmu_pages)
Carsten Otte1fe779f2007-10-29 16:08:35 +01005983{
5984 if (kvm_nr_mmu_pages < KVM_MIN_ALLOC_MMU_PAGES)
5985 return -EINVAL;
5986
Marcelo Tosatti79fac952009-12-23 14:35:26 -02005987 mutex_lock(&kvm->slots_lock);
Carsten Otte1fe779f2007-10-29 16:08:35 +01005988
5989 kvm_mmu_change_mmu_pages(kvm, kvm_nr_mmu_pages);
Zhang Xiantaof05e70a2007-12-14 10:01:48 +08005990 kvm->arch.n_requested_mmu_pages = kvm_nr_mmu_pages;
Carsten Otte1fe779f2007-10-29 16:08:35 +01005991
Marcelo Tosatti79fac952009-12-23 14:35:26 -02005992 mutex_unlock(&kvm->slots_lock);
Carsten Otte1fe779f2007-10-29 16:08:35 +01005993 return 0;
5994}
5995
Ben Gardonbc8a3d82019-04-08 11:07:30 -07005996static unsigned long kvm_vm_ioctl_get_nr_mmu_pages(struct kvm *kvm)
Carsten Otte1fe779f2007-10-29 16:08:35 +01005997{
Dave Hansen39de71e2010-08-19 18:11:14 -07005998 return kvm->arch.n_max_mmu_pages;
Carsten Otte1fe779f2007-10-29 16:08:35 +01005999}
6000
Carsten Otte1fe779f2007-10-29 16:08:35 +01006001static int kvm_vm_ioctl_get_irqchip(struct kvm *kvm, struct kvm_irqchip *chip)
6002{
David Hildenbrand90bca052017-04-07 10:50:23 +02006003 struct kvm_pic *pic = kvm->arch.vpic;
Carsten Otte1fe779f2007-10-29 16:08:35 +01006004 int r;
6005
6006 r = 0;
6007 switch (chip->chip_id) {
6008 case KVM_IRQCHIP_PIC_MASTER:
David Hildenbrand90bca052017-04-07 10:50:23 +02006009 memcpy(&chip->chip.pic, &pic->pics[0],
Carsten Otte1fe779f2007-10-29 16:08:35 +01006010 sizeof(struct kvm_pic_state));
6011 break;
6012 case KVM_IRQCHIP_PIC_SLAVE:
David Hildenbrand90bca052017-04-07 10:50:23 +02006013 memcpy(&chip->chip.pic, &pic->pics[1],
Carsten Otte1fe779f2007-10-29 16:08:35 +01006014 sizeof(struct kvm_pic_state));
6015 break;
6016 case KVM_IRQCHIP_IOAPIC:
David Hildenbrand33392b42017-04-07 10:50:27 +02006017 kvm_get_ioapic(kvm, &chip->chip.ioapic);
Carsten Otte1fe779f2007-10-29 16:08:35 +01006018 break;
6019 default:
6020 r = -EINVAL;
6021 break;
6022 }
6023 return r;
6024}
6025
6026static int kvm_vm_ioctl_set_irqchip(struct kvm *kvm, struct kvm_irqchip *chip)
6027{
David Hildenbrand90bca052017-04-07 10:50:23 +02006028 struct kvm_pic *pic = kvm->arch.vpic;
Carsten Otte1fe779f2007-10-29 16:08:35 +01006029 int r;
6030
6031 r = 0;
6032 switch (chip->chip_id) {
6033 case KVM_IRQCHIP_PIC_MASTER:
David Hildenbrand90bca052017-04-07 10:50:23 +02006034 spin_lock(&pic->lock);
6035 memcpy(&pic->pics[0], &chip->chip.pic,
Carsten Otte1fe779f2007-10-29 16:08:35 +01006036 sizeof(struct kvm_pic_state));
David Hildenbrand90bca052017-04-07 10:50:23 +02006037 spin_unlock(&pic->lock);
Carsten Otte1fe779f2007-10-29 16:08:35 +01006038 break;
6039 case KVM_IRQCHIP_PIC_SLAVE:
David Hildenbrand90bca052017-04-07 10:50:23 +02006040 spin_lock(&pic->lock);
6041 memcpy(&pic->pics[1], &chip->chip.pic,
Carsten Otte1fe779f2007-10-29 16:08:35 +01006042 sizeof(struct kvm_pic_state));
David Hildenbrand90bca052017-04-07 10:50:23 +02006043 spin_unlock(&pic->lock);
Carsten Otte1fe779f2007-10-29 16:08:35 +01006044 break;
6045 case KVM_IRQCHIP_IOAPIC:
David Hildenbrand33392b42017-04-07 10:50:27 +02006046 kvm_set_ioapic(kvm, &chip->chip.ioapic);
Carsten Otte1fe779f2007-10-29 16:08:35 +01006047 break;
6048 default:
6049 r = -EINVAL;
6050 break;
6051 }
David Hildenbrand90bca052017-04-07 10:50:23 +02006052 kvm_pic_update_irq(pic);
Carsten Otte1fe779f2007-10-29 16:08:35 +01006053 return r;
6054}
6055
Sheng Yange0f63cb2008-03-04 00:50:59 +08006056static int kvm_vm_ioctl_get_pit(struct kvm *kvm, struct kvm_pit_state *ps)
6057{
Radim Krčmář34f39412016-03-02 22:56:50 +01006058 struct kvm_kpit_state *kps = &kvm->arch.vpit->pit_state;
6059
6060 BUILD_BUG_ON(sizeof(*ps) != sizeof(kps->channels));
6061
6062 mutex_lock(&kps->lock);
6063 memcpy(ps, &kps->channels, sizeof(*ps));
6064 mutex_unlock(&kps->lock);
Saurabh Sengar2da29bc2015-10-30 12:56:11 +05306065 return 0;
Sheng Yange0f63cb2008-03-04 00:50:59 +08006066}
6067
6068static int kvm_vm_ioctl_set_pit(struct kvm *kvm, struct kvm_pit_state *ps)
6069{
Andrew Honig01856042015-11-18 14:50:23 -08006070 int i;
Radim Krčmář09edea72016-03-02 22:56:43 +01006071 struct kvm_pit *pit = kvm->arch.vpit;
6072
6073 mutex_lock(&pit->pit_state.lock);
Radim Krčmář34f39412016-03-02 22:56:50 +01006074 memcpy(&pit->pit_state.channels, ps, sizeof(*ps));
Andrew Honig01856042015-11-18 14:50:23 -08006075 for (i = 0; i < 3; i++)
Radim Krčmář09edea72016-03-02 22:56:43 +01006076 kvm_pit_load_count(pit, i, ps->channels[i].count, 0);
6077 mutex_unlock(&pit->pit_state.lock);
Saurabh Sengar2da29bc2015-10-30 12:56:11 +05306078 return 0;
Beth Kone9f42752009-07-07 11:50:38 -04006079}
6080
6081static int kvm_vm_ioctl_get_pit2(struct kvm *kvm, struct kvm_pit_state2 *ps)
6082{
Beth Kone9f42752009-07-07 11:50:38 -04006083 mutex_lock(&kvm->arch.vpit->pit_state.lock);
6084 memcpy(ps->channels, &kvm->arch.vpit->pit_state.channels,
6085 sizeof(ps->channels));
6086 ps->flags = kvm->arch.vpit->pit_state.flags;
6087 mutex_unlock(&kvm->arch.vpit->pit_state.lock);
Vasiliy Kulikov97e69aa2010-10-30 22:54:47 +04006088 memset(&ps->reserved, 0, sizeof(ps->reserved));
Saurabh Sengar2da29bc2015-10-30 12:56:11 +05306089 return 0;
Beth Kone9f42752009-07-07 11:50:38 -04006090}
6091
6092static int kvm_vm_ioctl_set_pit2(struct kvm *kvm, struct kvm_pit_state2 *ps)
6093{
Saurabh Sengar2da29bc2015-10-30 12:56:11 +05306094 int start = 0;
Andrew Honig01856042015-11-18 14:50:23 -08006095 int i;
Beth Kone9f42752009-07-07 11:50:38 -04006096 u32 prev_legacy, cur_legacy;
Radim Krčmář09edea72016-03-02 22:56:43 +01006097 struct kvm_pit *pit = kvm->arch.vpit;
6098
6099 mutex_lock(&pit->pit_state.lock);
6100 prev_legacy = pit->pit_state.flags & KVM_PIT_FLAGS_HPET_LEGACY;
Beth Kone9f42752009-07-07 11:50:38 -04006101 cur_legacy = ps->flags & KVM_PIT_FLAGS_HPET_LEGACY;
6102 if (!prev_legacy && cur_legacy)
6103 start = 1;
Radim Krčmář09edea72016-03-02 22:56:43 +01006104 memcpy(&pit->pit_state.channels, &ps->channels,
6105 sizeof(pit->pit_state.channels));
6106 pit->pit_state.flags = ps->flags;
Andrew Honig01856042015-11-18 14:50:23 -08006107 for (i = 0; i < 3; i++)
Radim Krčmář09edea72016-03-02 22:56:43 +01006108 kvm_pit_load_count(pit, i, pit->pit_state.channels[i].count,
Paolo Bonzinie5e57e72016-01-07 13:50:38 +01006109 start && i == 0);
Radim Krčmář09edea72016-03-02 22:56:43 +01006110 mutex_unlock(&pit->pit_state.lock);
Saurabh Sengar2da29bc2015-10-30 12:56:11 +05306111 return 0;
Sheng Yange0f63cb2008-03-04 00:50:59 +08006112}
6113
Marcelo Tosatti52d939a2008-12-30 15:55:06 -02006114static int kvm_vm_ioctl_reinject(struct kvm *kvm,
6115 struct kvm_reinject_control *control)
6116{
Radim Krčmář71474e22016-03-02 22:56:45 +01006117 struct kvm_pit *pit = kvm->arch.vpit;
6118
Radim Krčmář71474e22016-03-02 22:56:45 +01006119 /* pit->pit_state.lock was overloaded to prevent userspace from getting
6120 * an inconsistent state after running multiple KVM_REINJECT_CONTROL
6121 * ioctls in parallel. Use a separate lock if that ioctl isn't rare.
6122 */
6123 mutex_lock(&pit->pit_state.lock);
6124 kvm_pit_set_reinject(pit, control->pit_reinject);
6125 mutex_unlock(&pit->pit_state.lock);
Radim Krčmářb39c90b2016-03-02 22:56:44 +01006126
Marcelo Tosatti52d939a2008-12-30 15:55:06 -02006127 return 0;
6128}
6129
Sean Christopherson0dff0842020-02-18 13:07:29 -08006130void kvm_arch_sync_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot)
Zhang Xiantao5bb064d2007-11-18 20:29:43 +08006131{
Sean Christophersona018eba2021-02-12 16:50:10 -08006132
Kai Huang88178fd2015-01-28 10:54:27 +08006133 /*
Sean Christophersona018eba2021-02-12 16:50:10 -08006134 * Flush all CPUs' dirty log buffers to the dirty_bitmap. Called
6135 * before reporting dirty_bitmap to userspace. KVM flushes the buffers
6136 * on all VM-Exits, thus we only need to kick running vCPUs to force a
6137 * VM-Exit.
Kai Huang88178fd2015-01-28 10:54:27 +08006138 */
Sean Christophersona018eba2021-02-12 16:50:10 -08006139 struct kvm_vcpu *vcpu;
Marc Zyngier46808a42021-11-16 16:04:02 +00006140 unsigned long i;
Sean Christophersona018eba2021-02-12 16:50:10 -08006141
6142 kvm_for_each_vcpu(i, vcpu, kvm)
6143 kvm_vcpu_kick(vcpu);
Zhang Xiantao5bb064d2007-11-18 20:29:43 +08006144}
6145
Yang Zhangaa2fbe62013-04-11 19:21:40 +08006146int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_event,
6147 bool line_status)
Christoffer Dall23d43cf2012-07-24 08:51:20 -04006148{
6149 if (!irqchip_in_kernel(kvm))
6150 return -ENXIO;
6151
6152 irq_event->status = kvm_set_irq(kvm, KVM_USERSPACE_IRQ_SOURCE_ID,
Yang Zhangaa2fbe62013-04-11 19:21:40 +08006153 irq_event->irq, irq_event->level,
6154 line_status);
Christoffer Dall23d43cf2012-07-24 08:51:20 -04006155 return 0;
6156}
6157
Paolo Bonzinie5d83c72017-02-16 10:40:56 +01006158int kvm_vm_ioctl_enable_cap(struct kvm *kvm,
6159 struct kvm_enable_cap *cap)
Nadav Amit90de4a12015-04-13 01:53:41 +03006160{
6161 int r;
6162
6163 if (cap->flags)
6164 return -EINVAL;
6165
6166 switch (cap->cap) {
Oliver Upton6d849192022-03-01 06:03:47 +00006167 case KVM_CAP_DISABLE_QUIRKS2:
6168 r = -EINVAL;
6169 if (cap->args[0] & ~KVM_X86_VALID_QUIRKS)
6170 break;
6171 fallthrough;
Nadav Amit90de4a12015-04-13 01:53:41 +03006172 case KVM_CAP_DISABLE_QUIRKS:
6173 kvm->arch.disabled_quirks = cap->args[0];
6174 r = 0;
6175 break;
Steve Rutherford49df6392015-07-29 23:21:40 -07006176 case KVM_CAP_SPLIT_IRQCHIP: {
6177 mutex_lock(&kvm->lock);
Steve Rutherfordb053b2a2015-07-29 23:32:35 -07006178 r = -EINVAL;
6179 if (cap->args[0] > MAX_NR_RESERVED_IOAPIC_PINS)
6180 goto split_irqchip_unlock;
Steve Rutherford49df6392015-07-29 23:21:40 -07006181 r = -EEXIST;
6182 if (irqchip_in_kernel(kvm))
6183 goto split_irqchip_unlock;
Paolo Bonzini557abc42016-06-13 14:50:04 +02006184 if (kvm->created_vcpus)
Steve Rutherford49df6392015-07-29 23:21:40 -07006185 goto split_irqchip_unlock;
6186 r = kvm_setup_empty_irq_routing(kvm);
David Hildenbrand5c0aea02017-04-28 17:06:20 +02006187 if (r)
Steve Rutherford49df6392015-07-29 23:21:40 -07006188 goto split_irqchip_unlock;
6189 /* Pairs with irqchip_in_kernel. */
6190 smp_wmb();
Radim Krčmář49776fa2016-12-16 16:10:02 +01006191 kvm->arch.irqchip_mode = KVM_IRQCHIP_SPLIT;
Steve Rutherfordb053b2a2015-07-29 23:32:35 -07006192 kvm->arch.nr_reserved_ioapic_pins = cap->args[0];
Sean Christopherson320af552022-03-11 04:35:16 +00006193 kvm_clear_apicv_inhibit(kvm, APICV_INHIBIT_REASON_ABSENT);
Steve Rutherford49df6392015-07-29 23:21:40 -07006194 r = 0;
6195split_irqchip_unlock:
6196 mutex_unlock(&kvm->lock);
6197 break;
6198 }
Radim Krčmář371313132016-07-12 22:09:27 +02006199 case KVM_CAP_X2APIC_API:
6200 r = -EINVAL;
6201 if (cap->args[0] & ~KVM_X2APIC_API_VALID_FLAGS)
6202 break;
6203
6204 if (cap->args[0] & KVM_X2APIC_API_USE_32BIT_IDS)
6205 kvm->arch.x2apic_format = true;
Radim Krčmářc5192652016-07-12 22:09:28 +02006206 if (cap->args[0] & KVM_X2APIC_API_DISABLE_BROADCAST_QUIRK)
6207 kvm->arch.x2apic_broadcast_quirk_disabled = true;
Radim Krčmář371313132016-07-12 22:09:27 +02006208
6209 r = 0;
6210 break;
Wanpeng Li4d5422c2018-03-12 04:53:02 -07006211 case KVM_CAP_X86_DISABLE_EXITS:
6212 r = -EINVAL;
6213 if (cap->args[0] & ~KVM_X86_DISABLE_VALID_EXITS)
6214 break;
6215
6216 if ((cap->args[0] & KVM_X86_DISABLE_EXITS_MWAIT) &&
6217 kvm_can_mwait_in_guest())
6218 kvm->arch.mwait_in_guest = true;
Michael S. Tsirkin766d3572018-06-08 02:19:53 +03006219 if (cap->args[0] & KVM_X86_DISABLE_EXITS_HLT)
Wanpeng Licaa057a2018-03-12 04:53:03 -07006220 kvm->arch.hlt_in_guest = true;
Wanpeng Lib31c1142018-03-12 04:53:04 -07006221 if (cap->args[0] & KVM_X86_DISABLE_EXITS_PAUSE)
6222 kvm->arch.pause_in_guest = true;
Wanpeng Lib5170062019-05-21 14:06:53 +08006223 if (cap->args[0] & KVM_X86_DISABLE_EXITS_CSTATE)
6224 kvm->arch.cstate_in_guest = true;
Wanpeng Li4d5422c2018-03-12 04:53:02 -07006225 r = 0;
6226 break;
Drew Schmitt6fbbde92018-08-20 10:32:15 -07006227 case KVM_CAP_MSR_PLATFORM_INFO:
6228 kvm->arch.guest_can_read_msr_platform_info = cap->args[0];
6229 r = 0;
6230 break;
Jim Mattsonc4f55192018-10-16 14:29:24 -07006231 case KVM_CAP_EXCEPTION_PAYLOAD:
6232 kvm->arch.exception_payload_enabled = cap->args[0];
6233 r = 0;
6234 break;
Chenyi Qianged235112022-05-24 21:56:21 +08006235 case KVM_CAP_X86_TRIPLE_FAULT_EVENT:
6236 kvm->arch.triple_fault_event = cap->args[0];
6237 r = 0;
6238 break;
Alexander Graf1ae09952020-09-25 16:34:16 +02006239 case KVM_CAP_X86_USER_SPACE_MSR:
Aaron Lewiscf5029d2022-07-14 16:13:15 +00006240 r = -EINVAL;
6241 if (cap->args[0] & ~(KVM_MSR_EXIT_REASON_INVAL |
6242 KVM_MSR_EXIT_REASON_UNKNOWN |
6243 KVM_MSR_EXIT_REASON_FILTER))
6244 break;
Alexander Graf1ae09952020-09-25 16:34:16 +02006245 kvm->arch.user_space_msr_mask = cap->args[0];
6246 r = 0;
6247 break;
Chenyi Qiangfe6b6bc2020-11-06 17:03:14 +08006248 case KVM_CAP_X86_BUS_LOCK_EXIT:
6249 r = -EINVAL;
6250 if (cap->args[0] & ~KVM_BUS_LOCK_DETECTION_VALID_MODE)
6251 break;
6252
6253 if ((cap->args[0] & KVM_BUS_LOCK_DETECTION_OFF) &&
6254 (cap->args[0] & KVM_BUS_LOCK_DETECTION_EXIT))
6255 break;
6256
Sean Christopherson938c8742022-05-24 21:56:23 +08006257 if (kvm_caps.has_bus_lock_exit &&
Chenyi Qiangfe6b6bc2020-11-06 17:03:14 +08006258 cap->args[0] & KVM_BUS_LOCK_DETECTION_EXIT)
6259 kvm->arch.bus_lock_detection_enabled = true;
6260 r = 0;
6261 break;
Sean Christophersonfe7e9482021-04-12 16:21:43 +12006262#ifdef CONFIG_X86_SGX_KVM
6263 case KVM_CAP_SGX_ATTRIBUTE: {
6264 unsigned long allowed_attributes = 0;
6265
6266 r = sgx_set_attribute(&allowed_attributes, cap->args[0]);
6267 if (r)
6268 break;
6269
6270 /* KVM only supports the PROVISIONKEY privileged attribute. */
6271 if ((allowed_attributes & SGX_ATTR_PROVISIONKEY) &&
6272 !(allowed_attributes & ~SGX_ATTR_PROVISIONKEY))
6273 kvm->arch.sgx_provisioning_allowed = true;
6274 else
6275 r = -EINVAL;
6276 break;
6277 }
6278#endif
Nathan Tempelman54526d12021-04-08 22:32:14 +00006279 case KVM_CAP_VM_COPY_ENC_CONTEXT_FROM:
6280 r = -EINVAL;
Sean Christopherson7ad02ef2022-01-28 00:51:57 +00006281 if (!kvm_x86_ops.vm_copy_enc_context_from)
6282 break;
6283
6284 r = static_call(kvm_x86_vm_copy_enc_context_from)(kvm, cap->args[0]);
6285 break;
Peter Gondab5663932021-10-21 10:43:00 -07006286 case KVM_CAP_VM_MOVE_ENC_CONTEXT_FROM:
6287 r = -EINVAL;
Sean Christopherson7ad02ef2022-01-28 00:51:57 +00006288 if (!kvm_x86_ops.vm_move_enc_context_from)
6289 break;
6290
6291 r = static_call(kvm_x86_vm_move_enc_context_from)(kvm, cap->args[0]);
6292 break;
Ashish Kalra0dbb1122021-06-08 18:05:43 +00006293 case KVM_CAP_EXIT_HYPERCALL:
6294 if (cap->args[0] & ~KVM_EXIT_HYPERCALL_VALID_MASK) {
6295 r = -EINVAL;
6296 break;
6297 }
6298 kvm->arch.hypercall_exit_enabled = cap->args[0];
6299 r = 0;
6300 break;
Aaron Lewis19238e72021-05-10 07:48:33 -07006301 case KVM_CAP_EXIT_ON_EMULATION_FAILURE:
6302 r = -EINVAL;
6303 if (cap->args[0] & ~1)
6304 break;
6305 kvm->arch.exit_on_emulation_error = cap->args[0];
6306 r = 0;
6307 break;
David Dunnba7bb6632022-02-23 22:57:41 +00006308 case KVM_CAP_PMU_CAPABILITY:
6309 r = -EINVAL;
6310 if (!enable_pmu || (cap->args[0] & ~KVM_CAP_PMU_VALID_MASK))
6311 break;
6312
6313 mutex_lock(&kvm->lock);
6314 if (!kvm->created_vcpus) {
6315 kvm->arch.enable_pmu = !(cap->args[0] & KVM_PMU_CAP_DISABLE);
6316 r = 0;
6317 }
6318 mutex_unlock(&kvm->lock);
6319 break;
Zeng Guang35875312022-04-19 23:44:44 +08006320 case KVM_CAP_MAX_VCPU_ID:
6321 r = -EINVAL;
6322 if (cap->args[0] > KVM_MAX_VCPU_IDS)
6323 break;
6324
6325 mutex_lock(&kvm->lock);
6326 if (kvm->arch.max_vcpu_ids == cap->args[0]) {
6327 r = 0;
6328 } else if (!kvm->arch.max_vcpu_ids) {
6329 kvm->arch.max_vcpu_ids = cap->args[0];
6330 r = 0;
6331 }
6332 mutex_unlock(&kvm->lock);
6333 break;
Tao Xu2f4073e2022-05-24 21:56:24 +08006334 case KVM_CAP_X86_NOTIFY_VMEXIT:
6335 r = -EINVAL;
6336 if ((u32)cap->args[0] & ~KVM_X86_NOTIFY_VMEXIT_VALID_BITS)
6337 break;
6338 if (!kvm_caps.has_notify_vmexit)
6339 break;
6340 if (!((u32)cap->args[0] & KVM_X86_NOTIFY_VMEXIT_ENABLED))
6341 break;
6342 mutex_lock(&kvm->lock);
6343 if (!kvm->created_vcpus) {
6344 kvm->arch.notify_window = cap->args[0] >> 32;
6345 kvm->arch.notify_vmexit_flags = (u32)cap->args[0];
6346 r = 0;
6347 }
6348 mutex_unlock(&kvm->lock);
6349 break;
Ben Gardon084cc29f2022-06-13 21:25:21 +00006350 case KVM_CAP_VM_DISABLE_NX_HUGE_PAGES:
6351 r = -EINVAL;
6352
6353 /*
6354 * Since the risk of disabling NX hugepages is a guest crashing
6355 * the system, ensure the userspace process has permission to
6356 * reboot the system.
6357 *
6358 * Note that unlike the reboot() syscall, the process must have
6359 * this capability in the root namespace because exposing
6360 * /dev/kvm into a container does not limit the scope of the
6361 * iTLB multihit bug to that container. In other words,
6362 * this must use capable(), not ns_capable().
6363 */
6364 if (!capable(CAP_SYS_BOOT)) {
6365 r = -EPERM;
6366 break;
6367 }
6368
6369 if (cap->args[0])
6370 break;
6371
6372 mutex_lock(&kvm->lock);
6373 if (!kvm->created_vcpus) {
6374 kvm->arch.disable_nx_huge_pages = true;
6375 r = 0;
6376 }
6377 mutex_unlock(&kvm->lock);
6378 break;
Nadav Amit90de4a12015-04-13 01:53:41 +03006379 default:
6380 r = -EINVAL;
6381 break;
6382 }
6383 return r;
6384}
6385
Sean Christophersonb318e8d2021-03-16 11:44:33 -07006386static struct kvm_x86_msr_filter *kvm_alloc_msr_filter(bool default_allow)
Alexander Graf1a1552542020-09-25 16:34:21 +02006387{
Sean Christophersonb318e8d2021-03-16 11:44:33 -07006388 struct kvm_x86_msr_filter *msr_filter;
Alexander Graf1a1552542020-09-25 16:34:21 +02006389
Sean Christophersonb318e8d2021-03-16 11:44:33 -07006390 msr_filter = kzalloc(sizeof(*msr_filter), GFP_KERNEL_ACCOUNT);
6391 if (!msr_filter)
6392 return NULL;
Alexander Graf1a1552542020-09-25 16:34:21 +02006393
Sean Christophersonb318e8d2021-03-16 11:44:33 -07006394 msr_filter->default_allow = default_allow;
6395 return msr_filter;
Alexander Graf1a1552542020-09-25 16:34:21 +02006396}
6397
Sean Christophersonb318e8d2021-03-16 11:44:33 -07006398static void kvm_free_msr_filter(struct kvm_x86_msr_filter *msr_filter)
Alexander Graf1a1552542020-09-25 16:34:21 +02006399{
Sean Christophersonb318e8d2021-03-16 11:44:33 -07006400 u32 i;
6401
6402 if (!msr_filter)
6403 return;
6404
6405 for (i = 0; i < msr_filter->count; i++)
6406 kfree(msr_filter->ranges[i].bitmap);
6407
6408 kfree(msr_filter);
6409}
6410
6411static int kvm_add_msr_filter(struct kvm_x86_msr_filter *msr_filter,
6412 struct kvm_msr_filter_range *user_range)
6413{
Alexander Graf1a1552542020-09-25 16:34:21 +02006414 unsigned long *bitmap = NULL;
6415 size_t bitmap_size;
Alexander Graf1a1552542020-09-25 16:34:21 +02006416
6417 if (!user_range->nmsrs)
6418 return 0;
6419
Siddharth Chandrasekaranaca35282021-05-03 14:21:11 +02006420 if (user_range->flags & ~(KVM_MSR_FILTER_READ | KVM_MSR_FILTER_WRITE))
6421 return -EINVAL;
6422
6423 if (!user_range->flags)
6424 return -EINVAL;
6425
Alexander Graf1a1552542020-09-25 16:34:21 +02006426 bitmap_size = BITS_TO_LONGS(user_range->nmsrs) * sizeof(long);
6427 if (!bitmap_size || bitmap_size > KVM_MSR_FILTER_MAX_BITMAP_SIZE)
6428 return -EINVAL;
6429
6430 bitmap = memdup_user((__user u8*)user_range->bitmap, bitmap_size);
6431 if (IS_ERR(bitmap))
6432 return PTR_ERR(bitmap);
6433
Siddharth Chandrasekaranaca35282021-05-03 14:21:11 +02006434 msr_filter->ranges[msr_filter->count] = (struct msr_bitmap_range) {
Alexander Graf1a1552542020-09-25 16:34:21 +02006435 .flags = user_range->flags,
6436 .base = user_range->base,
6437 .nmsrs = user_range->nmsrs,
6438 .bitmap = bitmap,
6439 };
6440
Sean Christophersonb318e8d2021-03-16 11:44:33 -07006441 msr_filter->count++;
Alexander Graf1a1552542020-09-25 16:34:21 +02006442 return 0;
Alexander Graf1a1552542020-09-25 16:34:21 +02006443}
6444
Alexander Graf2e3272b2022-10-17 20:45:40 +02006445static int kvm_vm_ioctl_set_msr_filter(struct kvm *kvm,
6446 struct kvm_msr_filter *filter)
Alexander Graf1a1552542020-09-25 16:34:21 +02006447{
Sean Christophersonb318e8d2021-03-16 11:44:33 -07006448 struct kvm_x86_msr_filter *new_filter, *old_filter;
Alexander Graf1a1552542020-09-25 16:34:21 +02006449 bool default_allow;
Paolo Bonzini043248b2020-10-20 10:57:01 -04006450 bool empty = true;
Sean Christophersonb318e8d2021-03-16 11:44:33 -07006451 int r = 0;
Alexander Graf1a1552542020-09-25 16:34:21 +02006452 u32 i;
6453
Alexander Graf2e3272b2022-10-17 20:45:40 +02006454 if (filter->flags & ~KVM_MSR_FILTER_DEFAULT_DENY)
Aaron Lewiscf5029d2022-07-14 16:13:15 +00006455 return -EINVAL;
6456
Alexander Graf2e3272b2022-10-17 20:45:40 +02006457 for (i = 0; i < ARRAY_SIZE(filter->ranges); i++)
6458 empty &= !filter->ranges[i].nmsrs;
Alexander Graf1a1552542020-09-25 16:34:21 +02006459
Alexander Graf2e3272b2022-10-17 20:45:40 +02006460 default_allow = !(filter->flags & KVM_MSR_FILTER_DEFAULT_DENY);
Paolo Bonzini043248b2020-10-20 10:57:01 -04006461 if (empty && !default_allow)
6462 return -EINVAL;
6463
Sean Christophersonb318e8d2021-03-16 11:44:33 -07006464 new_filter = kvm_alloc_msr_filter(default_allow);
6465 if (!new_filter)
6466 return -ENOMEM;
Paolo Bonzini043248b2020-10-20 10:57:01 -04006467
Alexander Graf2e3272b2022-10-17 20:45:40 +02006468 for (i = 0; i < ARRAY_SIZE(filter->ranges); i++) {
6469 r = kvm_add_msr_filter(new_filter, &filter->ranges[i]);
Sean Christophersonb318e8d2021-03-16 11:44:33 -07006470 if (r) {
6471 kvm_free_msr_filter(new_filter);
6472 return r;
6473 }
Alexander Graf1a1552542020-09-25 16:34:21 +02006474 }
6475
Sean Christophersonb318e8d2021-03-16 11:44:33 -07006476 mutex_lock(&kvm->lock);
6477
6478 /* The per-VM filter is protected by kvm->lock... */
6479 old_filter = srcu_dereference_check(kvm->arch.msr_filter, &kvm->srcu, 1);
6480
6481 rcu_assign_pointer(kvm->arch.msr_filter, new_filter);
6482 synchronize_srcu(&kvm->srcu);
6483
6484 kvm_free_msr_filter(old_filter);
6485
Alexander Graf1a1552542020-09-25 16:34:21 +02006486 kvm_make_all_cpus_request(kvm, KVM_REQ_MSR_FILTER_CHANGED);
6487 mutex_unlock(&kvm->lock);
6488
Sean Christophersonb318e8d2021-03-16 11:44:33 -07006489 return 0;
Alexander Graf1a1552542020-09-25 16:34:21 +02006490}
6491
Alexander Graf1739c702022-10-17 20:45:41 +02006492#ifdef CONFIG_KVM_COMPAT
6493/* for KVM_X86_SET_MSR_FILTER */
6494struct kvm_msr_filter_range_compat {
6495 __u32 flags;
6496 __u32 nmsrs;
6497 __u32 base;
6498 __u32 bitmap;
6499};
6500
6501struct kvm_msr_filter_compat {
6502 __u32 flags;
6503 struct kvm_msr_filter_range_compat ranges[KVM_MSR_FILTER_MAX_RANGES];
6504};
6505
6506#define KVM_X86_SET_MSR_FILTER_COMPAT _IOW(KVMIO, 0xc6, struct kvm_msr_filter_compat)
6507
6508long kvm_arch_vm_compat_ioctl(struct file *filp, unsigned int ioctl,
6509 unsigned long arg)
6510{
6511 void __user *argp = (void __user *)arg;
6512 struct kvm *kvm = filp->private_data;
6513 long r = -ENOTTY;
6514
6515 switch (ioctl) {
6516 case KVM_X86_SET_MSR_FILTER_COMPAT: {
6517 struct kvm_msr_filter __user *user_msr_filter = argp;
6518 struct kvm_msr_filter_compat filter_compat;
6519 struct kvm_msr_filter filter;
6520 int i;
6521
6522 if (copy_from_user(&filter_compat, user_msr_filter,
6523 sizeof(filter_compat)))
6524 return -EFAULT;
6525
6526 filter.flags = filter_compat.flags;
6527 for (i = 0; i < ARRAY_SIZE(filter.ranges); i++) {
6528 struct kvm_msr_filter_range_compat *cr;
6529
6530 cr = &filter_compat.ranges[i];
6531 filter.ranges[i] = (struct kvm_msr_filter_range) {
6532 .flags = cr->flags,
6533 .nmsrs = cr->nmsrs,
6534 .base = cr->base,
6535 .bitmap = (__u8 *)(ulong)cr->bitmap,
6536 };
6537 }
6538
6539 r = kvm_vm_ioctl_set_msr_filter(kvm, &filter);
6540 break;
6541 }
6542 }
6543
6544 return r;
6545}
6546#endif
6547
Sergey Senozhatsky7d628742021-06-06 11:10:45 +09006548#ifdef CONFIG_HAVE_KVM_PM_NOTIFIER
6549static int kvm_arch_suspend_notifier(struct kvm *kvm)
6550{
6551 struct kvm_vcpu *vcpu;
Marc Zyngier46808a42021-11-16 16:04:02 +00006552 unsigned long i;
6553 int ret = 0;
Sergey Senozhatsky7d628742021-06-06 11:10:45 +09006554
6555 mutex_lock(&kvm->lock);
6556 kvm_for_each_vcpu(i, vcpu, kvm) {
David Woodhouse916d3602022-03-03 15:41:14 +00006557 if (!vcpu->arch.pv_time.active)
Sergey Senozhatsky7d628742021-06-06 11:10:45 +09006558 continue;
6559
6560 ret = kvm_set_guest_paused(vcpu);
6561 if (ret) {
6562 kvm_err("Failed to pause guest VCPU%d: %d\n",
6563 vcpu->vcpu_id, ret);
6564 break;
6565 }
6566 }
6567 mutex_unlock(&kvm->lock);
6568
6569 return ret ? NOTIFY_BAD : NOTIFY_DONE;
6570}
6571
6572int kvm_arch_pm_notifier(struct kvm *kvm, unsigned long state)
6573{
6574 switch (state) {
6575 case PM_HIBERNATION_PREPARE:
6576 case PM_SUSPEND_PREPARE:
6577 return kvm_arch_suspend_notifier(kvm);
6578 }
6579
6580 return NOTIFY_DONE;
6581}
6582#endif /* CONFIG_HAVE_KVM_PM_NOTIFIER */
6583
Paolo Bonzini45e6c2f2021-09-16 18:15:33 +00006584static int kvm_vm_ioctl_get_clock(struct kvm *kvm, void __user *argp)
6585{
Paolo Bonzini869b4422021-09-16 18:15:36 +00006586 struct kvm_clock_data data = { 0 };
Paolo Bonzini45e6c2f2021-09-16 18:15:33 +00006587
Oliver Upton55c0cef2021-09-16 18:15:34 +00006588 get_kvmclock(kvm, &data);
Paolo Bonzini45e6c2f2021-09-16 18:15:33 +00006589 if (copy_to_user(argp, &data, sizeof(data)))
6590 return -EFAULT;
6591
6592 return 0;
6593}
6594
6595static int kvm_vm_ioctl_set_clock(struct kvm *kvm, void __user *argp)
6596{
6597 struct kvm_arch *ka = &kvm->arch;
6598 struct kvm_clock_data data;
Oliver Uptonc68dc1b2021-09-16 18:15:35 +00006599 u64 now_raw_ns;
Paolo Bonzini45e6c2f2021-09-16 18:15:33 +00006600
6601 if (copy_from_user(&data, argp, sizeof(data)))
6602 return -EFAULT;
6603
Oliver Uptonc68dc1b2021-09-16 18:15:35 +00006604 /*
6605 * Only KVM_CLOCK_REALTIME is used, but allow passing the
6606 * result of KVM_GET_CLOCK back to KVM_SET_CLOCK.
6607 */
6608 if (data.flags & ~KVM_CLOCK_VALID_FLAGS)
Paolo Bonzini45e6c2f2021-09-16 18:15:33 +00006609 return -EINVAL;
6610
Vitaly Kuznetsov42dcbe72022-04-07 22:10:13 +02006611 kvm_hv_request_tsc_page_update(kvm);
Paolo Bonzini45e6c2f2021-09-16 18:15:33 +00006612 kvm_start_pvclock_update(kvm);
6613 pvclock_update_vm_gtod_copy(kvm);
6614
6615 /*
6616 * This pairs with kvm_guest_time_update(): when masterclock is
6617 * in use, we use master_kernel_ns + kvmclock_offset to set
6618 * unsigned 'system_time' so if we use get_kvmclock_ns() (which
6619 * is slightly ahead) here we risk going negative on unsigned
6620 * 'system_time' when 'data.clock' is very small.
6621 */
Oliver Uptonc68dc1b2021-09-16 18:15:35 +00006622 if (data.flags & KVM_CLOCK_REALTIME) {
6623 u64 now_real_ns = ktime_get_real_ns();
6624
6625 /*
6626 * Avoid stepping the kvmclock backwards.
6627 */
6628 if (now_real_ns > data.realtime)
6629 data.clock += now_real_ns - data.realtime;
6630 }
6631
6632 if (ka->use_master_clock)
6633 now_raw_ns = ka->master_kernel_ns;
Paolo Bonzini45e6c2f2021-09-16 18:15:33 +00006634 else
Oliver Uptonc68dc1b2021-09-16 18:15:35 +00006635 now_raw_ns = get_kvmclock_base_ns();
6636 ka->kvmclock_offset = data.clock - now_raw_ns;
Paolo Bonzini45e6c2f2021-09-16 18:15:33 +00006637 kvm_end_pvclock_update(kvm);
6638 return 0;
6639}
6640
Carsten Otte1fe779f2007-10-29 16:08:35 +01006641long kvm_arch_vm_ioctl(struct file *filp,
6642 unsigned int ioctl, unsigned long arg)
6643{
6644 struct kvm *kvm = filp->private_data;
6645 void __user *argp = (void __user *)arg;
Avi Kivity367e1312009-08-26 14:57:07 +03006646 int r = -ENOTTY;
Dave Hansenf0d66272008-08-11 10:01:45 -07006647 /*
6648 * This union makes it completely explicit to gcc-3.x
6649 * that these two variables' stack usage should be
6650 * combined, not added together.
6651 */
6652 union {
6653 struct kvm_pit_state ps;
Beth Kone9f42752009-07-07 11:50:38 -04006654 struct kvm_pit_state2 ps2;
Jan Kiszkac5ff41ce2009-05-14 22:42:53 +02006655 struct kvm_pit_config pit_config;
Dave Hansenf0d66272008-08-11 10:01:45 -07006656 } u;
Carsten Otte1fe779f2007-10-29 16:08:35 +01006657
6658 switch (ioctl) {
6659 case KVM_SET_TSS_ADDR:
6660 r = kvm_vm_ioctl_set_tss_addr(kvm, arg);
Carsten Otte1fe779f2007-10-29 16:08:35 +01006661 break;
Sheng Yangb927a3c2009-07-21 10:42:48 +08006662 case KVM_SET_IDENTITY_MAP_ADDR: {
6663 u64 ident_addr;
6664
David Hildenbrand1af1ac92017-08-24 20:51:36 +02006665 mutex_lock(&kvm->lock);
6666 r = -EINVAL;
6667 if (kvm->created_vcpus)
6668 goto set_identity_unlock;
Sheng Yangb927a3c2009-07-21 10:42:48 +08006669 r = -EFAULT;
Jordan Borgner0e96f312018-10-28 12:58:28 +00006670 if (copy_from_user(&ident_addr, argp, sizeof(ident_addr)))
David Hildenbrand1af1ac92017-08-24 20:51:36 +02006671 goto set_identity_unlock;
Sheng Yangb927a3c2009-07-21 10:42:48 +08006672 r = kvm_vm_ioctl_set_identity_map_addr(kvm, ident_addr);
David Hildenbrand1af1ac92017-08-24 20:51:36 +02006673set_identity_unlock:
6674 mutex_unlock(&kvm->lock);
Sheng Yangb927a3c2009-07-21 10:42:48 +08006675 break;
6676 }
Carsten Otte1fe779f2007-10-29 16:08:35 +01006677 case KVM_SET_NR_MMU_PAGES:
6678 r = kvm_vm_ioctl_set_nr_mmu_pages(kvm, arg);
Carsten Otte1fe779f2007-10-29 16:08:35 +01006679 break;
6680 case KVM_GET_NR_MMU_PAGES:
6681 r = kvm_vm_ioctl_get_nr_mmu_pages(kvm);
6682 break;
Marcelo Tosatti3ddea122009-10-29 13:44:15 -02006683 case KVM_CREATE_IRQCHIP: {
Marcelo Tosatti3ddea122009-10-29 13:44:15 -02006684 mutex_lock(&kvm->lock);
Radim Krčmář09941362016-12-16 16:10:03 +01006685
Marcelo Tosatti3ddea122009-10-29 13:44:15 -02006686 r = -EEXIST;
Radim Krčmář35e6eaa2016-12-16 16:10:01 +01006687 if (irqchip_in_kernel(kvm))
Marcelo Tosatti3ddea122009-10-29 13:44:15 -02006688 goto create_irqchip_unlock;
Radim Krčmář09941362016-12-16 16:10:03 +01006689
Avi Kivity3e515702012-03-05 14:23:29 +02006690 r = -EINVAL;
Paolo Bonzini557abc42016-06-13 14:50:04 +02006691 if (kvm->created_vcpus)
Avi Kivity3e515702012-03-05 14:23:29 +02006692 goto create_irqchip_unlock;
Radim Krčmář09941362016-12-16 16:10:03 +01006693
6694 r = kvm_pic_init(kvm);
6695 if (r)
Marcelo Tosatti3ddea122009-10-29 13:44:15 -02006696 goto create_irqchip_unlock;
Radim Krčmář09941362016-12-16 16:10:03 +01006697
6698 r = kvm_ioapic_init(kvm);
6699 if (r) {
Radim Krčmář09941362016-12-16 16:10:03 +01006700 kvm_pic_destroy(kvm);
Radim Krčmář09941362016-12-16 16:10:03 +01006701 goto create_irqchip_unlock;
6702 }
6703
Avi Kivity399ec802008-11-19 13:58:46 +02006704 r = kvm_setup_default_irq_routing(kvm);
6705 if (r) {
Wei Yongjun72bb2fc2010-02-09 10:33:03 +08006706 kvm_ioapic_destroy(kvm);
Radim Krčmář09941362016-12-16 16:10:03 +01006707 kvm_pic_destroy(kvm);
Paolo Bonzini71ba9942015-07-29 12:31:15 +02006708 goto create_irqchip_unlock;
Avi Kivity399ec802008-11-19 13:58:46 +02006709 }
Radim Krčmář49776fa2016-12-16 16:10:02 +01006710 /* Write kvm->irq_routing before enabling irqchip_in_kernel. */
Paolo Bonzini71ba9942015-07-29 12:31:15 +02006711 smp_wmb();
Radim Krčmář49776fa2016-12-16 16:10:02 +01006712 kvm->arch.irqchip_mode = KVM_IRQCHIP_KERNEL;
Sean Christopherson320af552022-03-11 04:35:16 +00006713 kvm_clear_apicv_inhibit(kvm, APICV_INHIBIT_REASON_ABSENT);
Marcelo Tosatti3ddea122009-10-29 13:44:15 -02006714 create_irqchip_unlock:
6715 mutex_unlock(&kvm->lock);
Carsten Otte1fe779f2007-10-29 16:08:35 +01006716 break;
Marcelo Tosatti3ddea122009-10-29 13:44:15 -02006717 }
Sheng Yang78376992008-01-28 05:10:22 +08006718 case KVM_CREATE_PIT:
Jan Kiszkac5ff41ce2009-05-14 22:42:53 +02006719 u.pit_config.flags = KVM_PIT_SPEAKER_DUMMY;
6720 goto create_pit;
6721 case KVM_CREATE_PIT2:
6722 r = -EFAULT;
6723 if (copy_from_user(&u.pit_config, argp,
6724 sizeof(struct kvm_pit_config)))
6725 goto out;
6726 create_pit:
Paolo Bonzini250715a2016-06-01 14:09:24 +02006727 mutex_lock(&kvm->lock);
Avi Kivity269e05e2009-01-05 15:21:42 +02006728 r = -EEXIST;
6729 if (kvm->arch.vpit)
6730 goto create_pit_unlock;
Sheng Yang78376992008-01-28 05:10:22 +08006731 r = -ENOMEM;
Jan Kiszkac5ff41ce2009-05-14 22:42:53 +02006732 kvm->arch.vpit = kvm_create_pit(kvm, u.pit_config.flags);
Sheng Yang78376992008-01-28 05:10:22 +08006733 if (kvm->arch.vpit)
6734 r = 0;
Avi Kivity269e05e2009-01-05 15:21:42 +02006735 create_pit_unlock:
Paolo Bonzini250715a2016-06-01 14:09:24 +02006736 mutex_unlock(&kvm->lock);
Sheng Yang78376992008-01-28 05:10:22 +08006737 break;
Carsten Otte1fe779f2007-10-29 16:08:35 +01006738 case KVM_GET_IRQCHIP: {
6739 /* 0: PIC master, 1: PIC slave, 2: IOAPIC */
Sasha Levinff5c2c02011-12-04 19:36:29 +02006740 struct kvm_irqchip *chip;
Carsten Otte1fe779f2007-10-29 16:08:35 +01006741
Sasha Levinff5c2c02011-12-04 19:36:29 +02006742 chip = memdup_user(argp, sizeof(*chip));
6743 if (IS_ERR(chip)) {
6744 r = PTR_ERR(chip);
Carsten Otte1fe779f2007-10-29 16:08:35 +01006745 goto out;
Sasha Levinff5c2c02011-12-04 19:36:29 +02006746 }
6747
Carsten Otte1fe779f2007-10-29 16:08:35 +01006748 r = -ENXIO;
Radim Krčmář826da322016-12-16 16:10:06 +01006749 if (!irqchip_kernel(kvm))
Dave Hansenf0d66272008-08-11 10:01:45 -07006750 goto get_irqchip_out;
6751 r = kvm_vm_ioctl_get_irqchip(kvm, chip);
6752 if (r)
6753 goto get_irqchip_out;
6754 r = -EFAULT;
Jordan Borgner0e96f312018-10-28 12:58:28 +00006755 if (copy_to_user(argp, chip, sizeof(*chip)))
Dave Hansenf0d66272008-08-11 10:01:45 -07006756 goto get_irqchip_out;
6757 r = 0;
6758 get_irqchip_out:
6759 kfree(chip);
Carsten Otte1fe779f2007-10-29 16:08:35 +01006760 break;
6761 }
6762 case KVM_SET_IRQCHIP: {
6763 /* 0: PIC master, 1: PIC slave, 2: IOAPIC */
Sasha Levinff5c2c02011-12-04 19:36:29 +02006764 struct kvm_irqchip *chip;
Carsten Otte1fe779f2007-10-29 16:08:35 +01006765
Sasha Levinff5c2c02011-12-04 19:36:29 +02006766 chip = memdup_user(argp, sizeof(*chip));
6767 if (IS_ERR(chip)) {
6768 r = PTR_ERR(chip);
Carsten Otte1fe779f2007-10-29 16:08:35 +01006769 goto out;
Sasha Levinff5c2c02011-12-04 19:36:29 +02006770 }
6771
Carsten Otte1fe779f2007-10-29 16:08:35 +01006772 r = -ENXIO;
Radim Krčmář826da322016-12-16 16:10:06 +01006773 if (!irqchip_kernel(kvm))
Dave Hansenf0d66272008-08-11 10:01:45 -07006774 goto set_irqchip_out;
6775 r = kvm_vm_ioctl_set_irqchip(kvm, chip);
Dave Hansenf0d66272008-08-11 10:01:45 -07006776 set_irqchip_out:
6777 kfree(chip);
Carsten Otte1fe779f2007-10-29 16:08:35 +01006778 break;
6779 }
Sheng Yange0f63cb2008-03-04 00:50:59 +08006780 case KVM_GET_PIT: {
Sheng Yange0f63cb2008-03-04 00:50:59 +08006781 r = -EFAULT;
Dave Hansenf0d66272008-08-11 10:01:45 -07006782 if (copy_from_user(&u.ps, argp, sizeof(struct kvm_pit_state)))
Sheng Yange0f63cb2008-03-04 00:50:59 +08006783 goto out;
6784 r = -ENXIO;
6785 if (!kvm->arch.vpit)
6786 goto out;
Dave Hansenf0d66272008-08-11 10:01:45 -07006787 r = kvm_vm_ioctl_get_pit(kvm, &u.ps);
Sheng Yange0f63cb2008-03-04 00:50:59 +08006788 if (r)
6789 goto out;
6790 r = -EFAULT;
Dave Hansenf0d66272008-08-11 10:01:45 -07006791 if (copy_to_user(argp, &u.ps, sizeof(struct kvm_pit_state)))
Sheng Yange0f63cb2008-03-04 00:50:59 +08006792 goto out;
6793 r = 0;
6794 break;
6795 }
6796 case KVM_SET_PIT: {
Sheng Yange0f63cb2008-03-04 00:50:59 +08006797 r = -EFAULT;
Jordan Borgner0e96f312018-10-28 12:58:28 +00006798 if (copy_from_user(&u.ps, argp, sizeof(u.ps)))
Sheng Yange0f63cb2008-03-04 00:50:59 +08006799 goto out;
Steve Rutherford7289fdb2020-04-16 12:11:52 -07006800 mutex_lock(&kvm->lock);
Sheng Yange0f63cb2008-03-04 00:50:59 +08006801 r = -ENXIO;
6802 if (!kvm->arch.vpit)
Steve Rutherford7289fdb2020-04-16 12:11:52 -07006803 goto set_pit_out;
Dave Hansenf0d66272008-08-11 10:01:45 -07006804 r = kvm_vm_ioctl_set_pit(kvm, &u.ps);
Steve Rutherford7289fdb2020-04-16 12:11:52 -07006805set_pit_out:
6806 mutex_unlock(&kvm->lock);
Sheng Yange0f63cb2008-03-04 00:50:59 +08006807 break;
6808 }
Beth Kone9f42752009-07-07 11:50:38 -04006809 case KVM_GET_PIT2: {
6810 r = -ENXIO;
6811 if (!kvm->arch.vpit)
6812 goto out;
6813 r = kvm_vm_ioctl_get_pit2(kvm, &u.ps2);
6814 if (r)
6815 goto out;
6816 r = -EFAULT;
6817 if (copy_to_user(argp, &u.ps2, sizeof(u.ps2)))
6818 goto out;
6819 r = 0;
6820 break;
6821 }
6822 case KVM_SET_PIT2: {
6823 r = -EFAULT;
6824 if (copy_from_user(&u.ps2, argp, sizeof(u.ps2)))
6825 goto out;
Steve Rutherford7289fdb2020-04-16 12:11:52 -07006826 mutex_lock(&kvm->lock);
Beth Kone9f42752009-07-07 11:50:38 -04006827 r = -ENXIO;
6828 if (!kvm->arch.vpit)
Steve Rutherford7289fdb2020-04-16 12:11:52 -07006829 goto set_pit2_out;
Beth Kone9f42752009-07-07 11:50:38 -04006830 r = kvm_vm_ioctl_set_pit2(kvm, &u.ps2);
Steve Rutherford7289fdb2020-04-16 12:11:52 -07006831set_pit2_out:
6832 mutex_unlock(&kvm->lock);
Beth Kone9f42752009-07-07 11:50:38 -04006833 break;
6834 }
Marcelo Tosatti52d939a2008-12-30 15:55:06 -02006835 case KVM_REINJECT_CONTROL: {
6836 struct kvm_reinject_control control;
6837 r = -EFAULT;
6838 if (copy_from_user(&control, argp, sizeof(control)))
6839 goto out;
Miaohe Lincad23e72019-12-07 17:25:22 +08006840 r = -ENXIO;
6841 if (!kvm->arch.vpit)
6842 goto out;
Marcelo Tosatti52d939a2008-12-30 15:55:06 -02006843 r = kvm_vm_ioctl_reinject(kvm, &control);
Marcelo Tosatti52d939a2008-12-30 15:55:06 -02006844 break;
6845 }
Paolo Bonzinid71ba782015-07-29 11:56:48 +02006846 case KVM_SET_BOOT_CPU_ID:
6847 r = 0;
6848 mutex_lock(&kvm->lock);
Paolo Bonzini557abc42016-06-13 14:50:04 +02006849 if (kvm->created_vcpus)
Paolo Bonzinid71ba782015-07-29 11:56:48 +02006850 r = -EBUSY;
6851 else
6852 kvm->arch.bsp_vcpu_id = arg;
6853 mutex_unlock(&kvm->lock);
6854 break;
Paolo Bonzinib59b1532021-02-26 04:54:45 -05006855#ifdef CONFIG_KVM_XEN
Ed Swierkffde22a2009-10-15 15:21:43 -07006856 case KVM_XEN_HVM_CONFIG: {
Paolo Bonzini51776042017-10-26 15:45:47 +02006857 struct kvm_xen_hvm_config xhc;
Ed Swierkffde22a2009-10-15 15:21:43 -07006858 r = -EFAULT;
Paolo Bonzini51776042017-10-26 15:45:47 +02006859 if (copy_from_user(&xhc, argp, sizeof(xhc)))
Ed Swierkffde22a2009-10-15 15:21:43 -07006860 goto out;
David Woodhouse78e98782021-02-02 13:19:35 +00006861 r = kvm_xen_hvm_config(kvm, &xhc);
Ed Swierkffde22a2009-10-15 15:21:43 -07006862 break;
6863 }
Joao Martinsa76b9642020-12-03 15:52:25 +00006864 case KVM_XEN_HVM_GET_ATTR: {
6865 struct kvm_xen_hvm_attr xha;
6866
6867 r = -EFAULT;
6868 if (copy_from_user(&xha, argp, sizeof(xha)))
Hannes Edercded19f2009-02-21 02:19:13 +01006869 goto out;
Joao Martinsa76b9642020-12-03 15:52:25 +00006870 r = kvm_xen_hvm_get_attr(kvm, &xha);
6871 if (!r && copy_to_user(argp, &xha, sizeof(xha)))
6872 r = -EFAULT;
6873 break;
6874 }
6875 case KVM_XEN_HVM_SET_ATTR: {
6876 struct kvm_xen_hvm_attr xha;
6877
6878 r = -EFAULT;
6879 if (copy_from_user(&xha, argp, sizeof(xha)))
6880 goto out;
6881 r = kvm_xen_hvm_set_attr(kvm, &xha);
Carsten Ottebbd9b642007-10-30 18:44:21 +01006882 break;
6883 }
David Woodhouse35025732022-03-03 15:41:18 +00006884 case KVM_XEN_HVM_EVTCHN_SEND: {
6885 struct kvm_irq_routing_xen_evtchn uxe;
6886
6887 r = -EFAULT;
6888 if (copy_from_user(&uxe, argp, sizeof(uxe)))
6889 goto out;
6890 r = kvm_xen_hvm_evtchn_send(kvm, &uxe);
6891 break;
6892 }
Paolo Bonzinib59b1532021-02-26 04:54:45 -05006893#endif
Paolo Bonzini45e6c2f2021-09-16 18:15:33 +00006894 case KVM_SET_CLOCK:
6895 r = kvm_vm_ioctl_set_clock(kvm, argp);
Glauber Costaafbcf7a2009-10-16 15:28:36 -04006896 break;
Paolo Bonzini45e6c2f2021-09-16 18:15:33 +00006897 case KVM_GET_CLOCK:
6898 r = kvm_vm_ioctl_get_clock(kvm, argp);
Glauber Costaafbcf7a2009-10-16 15:28:36 -04006899 break;
David Woodhouseffbb61d2022-02-25 14:53:02 +00006900 case KVM_SET_TSC_KHZ: {
6901 u32 user_tsc_khz;
6902
6903 r = -EINVAL;
6904 user_tsc_khz = (u32)arg;
6905
Sean Christopherson938c8742022-05-24 21:56:23 +08006906 if (kvm_caps.has_tsc_control &&
6907 user_tsc_khz >= kvm_caps.max_guest_tsc_khz)
David Woodhouseffbb61d2022-02-25 14:53:02 +00006908 goto out;
6909
6910 if (user_tsc_khz == 0)
6911 user_tsc_khz = tsc_khz;
6912
6913 WRITE_ONCE(kvm->arch.default_tsc_khz, user_tsc_khz);
6914 r = 0;
6915
6916 goto out;
6917 }
6918 case KVM_GET_TSC_KHZ: {
6919 r = READ_ONCE(kvm->arch.default_tsc_khz);
6920 goto out;
6921 }
Brijesh Singh5acc5c02017-12-04 10:57:26 -06006922 case KVM_MEMORY_ENCRYPT_OP: {
6923 r = -ENOTTY;
Sean Christopherson03d004c2022-01-28 00:52:05 +00006924 if (!kvm_x86_ops.mem_enc_ioctl)
6925 goto out;
6926
6927 r = static_call(kvm_x86_mem_enc_ioctl)(kvm, argp);
Brijesh Singh5acc5c02017-12-04 10:57:26 -06006928 break;
6929 }
Brijesh Singh69eaede2017-12-04 10:57:26 -06006930 case KVM_MEMORY_ENCRYPT_REG_REGION: {
6931 struct kvm_enc_region region;
6932
6933 r = -EFAULT;
6934 if (copy_from_user(&region, argp, sizeof(region)))
6935 goto out;
6936
6937 r = -ENOTTY;
Sean Christopherson03d004c2022-01-28 00:52:05 +00006938 if (!kvm_x86_ops.mem_enc_register_region)
6939 goto out;
6940
6941 r = static_call(kvm_x86_mem_enc_register_region)(kvm, &region);
Brijesh Singh69eaede2017-12-04 10:57:26 -06006942 break;
6943 }
6944 case KVM_MEMORY_ENCRYPT_UNREG_REGION: {
6945 struct kvm_enc_region region;
6946
6947 r = -EFAULT;
6948 if (copy_from_user(&region, argp, sizeof(region)))
6949 goto out;
6950
6951 r = -ENOTTY;
Sean Christopherson03d004c2022-01-28 00:52:05 +00006952 if (!kvm_x86_ops.mem_enc_unregister_region)
6953 goto out;
6954
6955 r = static_call(kvm_x86_mem_enc_unregister_region)(kvm, &region);
Brijesh Singh69eaede2017-12-04 10:57:26 -06006956 break;
6957 }
Roman Kaganfaeb7832018-02-01 16:48:32 +03006958 case KVM_HYPERV_EVENTFD: {
6959 struct kvm_hyperv_eventfd hvevfd;
6960
6961 r = -EFAULT;
6962 if (copy_from_user(&hvevfd, argp, sizeof(hvevfd)))
6963 goto out;
6964 r = kvm_vm_ioctl_hv_eventfd(kvm, &hvevfd);
6965 break;
6966 }
Eric Hankland66bb8a02019-07-10 18:25:15 -07006967 case KVM_SET_PMU_EVENT_FILTER:
6968 r = kvm_vm_ioctl_set_pmu_event_filter(kvm, argp);
6969 break;
Alexander Graf2e3272b2022-10-17 20:45:40 +02006970 case KVM_X86_SET_MSR_FILTER: {
6971 struct kvm_msr_filter __user *user_msr_filter = argp;
6972 struct kvm_msr_filter filter;
6973
6974 if (copy_from_user(&filter, user_msr_filter, sizeof(filter)))
6975 return -EFAULT;
6976
6977 r = kvm_vm_ioctl_set_msr_filter(kvm, &filter);
Alexander Graf1a1552542020-09-25 16:34:21 +02006978 break;
Alexander Graf2e3272b2022-10-17 20:45:40 +02006979 }
Carsten Otte043405e2007-10-10 17:16:19 +02006980 default:
Paolo Bonziniad6260d2017-03-27 14:30:40 +02006981 r = -ENOTTY;
Carsten Otte043405e2007-10-10 17:16:19 +02006982 }
6983out:
6984 return r;
6985}
6986
6987static void kvm_init_msr_list(void)
6988{
6989 u32 dummy[2];
Chenyi Qiang7a5ee6e2019-11-06 14:35:20 +08006990 unsigned i;
Carsten Ottebbd9b642007-10-30 18:44:21 +01006991
Wei Wang0144ba02022-02-01 13:23:22 -08006992 BUILD_BUG_ON_MSG(KVM_PMC_MAX_FIXED != 3,
Chenyi Qiang7a5ee6e2019-11-06 14:35:20 +08006993 "Please update the fixed PMCs in msrs_to_saved_all[]");
Paolo Bonzini24c29b72019-10-01 15:18:26 +02006994
Xiaoyao Li6cbee2b2019-11-13 09:15:21 +08006995 num_msrs_to_save = 0;
6996 num_emulated_msrs = 0;
6997 num_msr_based_features = 0;
6998
Chenyi Qiang7a5ee6e2019-11-06 14:35:20 +08006999 for (i = 0; i < ARRAY_SIZE(msrs_to_save_all); i++) {
7000 if (rdmsr_safe(msrs_to_save_all[i], &dummy[0], &dummy[1]) < 0)
Carsten Ottebbd9b642007-10-30 18:44:21 +01007001 continue;
Paolo Bonzini93c4adc2014-03-05 23:19:52 +01007002
7003 /*
7004 * Even MSRs that are valid in the host may not be exposed
Paolo Bonzini9dbe6cf2015-11-12 14:49:17 +01007005 * to the guests in some cases.
Paolo Bonzini93c4adc2014-03-05 23:19:52 +01007006 */
Chenyi Qiang7a5ee6e2019-11-06 14:35:20 +08007007 switch (msrs_to_save_all[i]) {
Paolo Bonzini93c4adc2014-03-05 23:19:52 +01007008 case MSR_IA32_BNDCFGS:
Liran Alon503234b2018-09-14 03:25:53 +03007009 if (!kvm_mpx_supported())
Paolo Bonzini93c4adc2014-03-05 23:19:52 +01007010 continue;
7011 break;
Paolo Bonzini9dbe6cf2015-11-12 14:49:17 +01007012 case MSR_TSC_AUX:
Sean Christopherson36fa06f2021-05-04 10:17:26 -07007013 if (!kvm_cpu_cap_has(X86_FEATURE_RDTSCP) &&
7014 !kvm_cpu_cap_has(X86_FEATURE_RDPID))
Paolo Bonzini9dbe6cf2015-11-12 14:49:17 +01007015 continue;
7016 break;
Maxim Levitskyf4cfcd22020-05-23 19:14:55 +03007017 case MSR_IA32_UMWAIT_CONTROL:
7018 if (!kvm_cpu_cap_has(X86_FEATURE_WAITPKG))
7019 continue;
7020 break;
Chao Pengbf8c55d2018-10-24 16:05:14 +08007021 case MSR_IA32_RTIT_CTL:
7022 case MSR_IA32_RTIT_STATUS:
Sean Christopherson7b874c22020-03-02 15:56:59 -08007023 if (!kvm_cpu_cap_has(X86_FEATURE_INTEL_PT))
Chao Pengbf8c55d2018-10-24 16:05:14 +08007024 continue;
7025 break;
7026 case MSR_IA32_RTIT_CR3_MATCH:
Sean Christopherson7b874c22020-03-02 15:56:59 -08007027 if (!kvm_cpu_cap_has(X86_FEATURE_INTEL_PT) ||
Chao Pengbf8c55d2018-10-24 16:05:14 +08007028 !intel_pt_validate_hw_cap(PT_CAP_cr3_filtering))
7029 continue;
7030 break;
7031 case MSR_IA32_RTIT_OUTPUT_BASE:
7032 case MSR_IA32_RTIT_OUTPUT_MASK:
Sean Christopherson7b874c22020-03-02 15:56:59 -08007033 if (!kvm_cpu_cap_has(X86_FEATURE_INTEL_PT) ||
Chao Pengbf8c55d2018-10-24 16:05:14 +08007034 (!intel_pt_validate_hw_cap(PT_CAP_topa_output) &&
7035 !intel_pt_validate_hw_cap(PT_CAP_single_range_output)))
7036 continue;
7037 break;
Sean Christopherson7cb85fc2020-02-18 15:40:10 -08007038 case MSR_IA32_RTIT_ADDR0_A ... MSR_IA32_RTIT_ADDR3_B:
Sean Christopherson7b874c22020-03-02 15:56:59 -08007039 if (!kvm_cpu_cap_has(X86_FEATURE_INTEL_PT) ||
Chenyi Qiang7a5ee6e2019-11-06 14:35:20 +08007040 msrs_to_save_all[i] - MSR_IA32_RTIT_ADDR0_A >=
Chao Pengbf8c55d2018-10-24 16:05:14 +08007041 intel_pt_validate_hw_cap(PT_CAP_num_address_ranges) * 2)
7042 continue;
7043 break;
Paolo Bonzinicf05a672019-10-01 15:33:07 +02007044 case MSR_ARCH_PERFMON_PERFCTR0 ... MSR_ARCH_PERFMON_PERFCTR0 + 17:
Chenyi Qiang7a5ee6e2019-11-06 14:35:20 +08007045 if (msrs_to_save_all[i] - MSR_ARCH_PERFMON_PERFCTR0 >=
Like Xu968635a2022-04-11 18:19:44 +08007046 min(INTEL_PMC_MAX_GENERIC, kvm_pmu_cap.num_counters_gp))
Paolo Bonzini24c29b72019-10-01 15:18:26 +02007047 continue;
7048 break;
Paolo Bonzinicf05a672019-10-01 15:33:07 +02007049 case MSR_ARCH_PERFMON_EVENTSEL0 ... MSR_ARCH_PERFMON_EVENTSEL0 + 17:
Chenyi Qiang7a5ee6e2019-11-06 14:35:20 +08007050 if (msrs_to_save_all[i] - MSR_ARCH_PERFMON_EVENTSEL0 >=
Like Xu968635a2022-04-11 18:19:44 +08007051 min(INTEL_PMC_MAX_GENERIC, kvm_pmu_cap.num_counters_gp))
Paolo Bonzini24c29b72019-10-01 15:18:26 +02007052 continue;
Sean Christopherson7cb85fc2020-02-18 15:40:10 -08007053 break;
Jing Liu820a6ee2022-01-05 04:35:21 -08007054 case MSR_IA32_XFD:
Jing Liu548e8362022-01-05 04:35:24 -08007055 case MSR_IA32_XFD_ERR:
Jing Liu820a6ee2022-01-05 04:35:21 -08007056 if (!kvm_cpu_cap_has(X86_FEATURE_XFD))
7057 continue;
7058 break;
Paolo Bonzini93c4adc2014-03-05 23:19:52 +01007059 default:
7060 break;
7061 }
7062
Chenyi Qiang7a5ee6e2019-11-06 14:35:20 +08007063 msrs_to_save[num_msrs_to_save++] = msrs_to_save_all[i];
Carsten Ottebbd9b642007-10-30 18:44:21 +01007064 }
Paolo Bonzini62ef68b2015-05-05 12:08:55 +02007065
Chenyi Qiang7a5ee6e2019-11-06 14:35:20 +08007066 for (i = 0; i < ARRAY_SIZE(emulated_msrs_all); i++) {
Jason Baronb36464772021-01-14 22:27:56 -05007067 if (!static_call(kvm_x86_has_emulated_msr)(NULL, emulated_msrs_all[i]))
Tom Lendackybc226f02018-05-10 22:06:39 +02007068 continue;
Paolo Bonzini62ef68b2015-05-05 12:08:55 +02007069
Chenyi Qiang7a5ee6e2019-11-06 14:35:20 +08007070 emulated_msrs[num_emulated_msrs++] = emulated_msrs_all[i];
Paolo Bonzini62ef68b2015-05-05 12:08:55 +02007071 }
Tom Lendacky801e4592018-02-21 13:39:51 -06007072
Chenyi Qiang7a5ee6e2019-11-06 14:35:20 +08007073 for (i = 0; i < ARRAY_SIZE(msr_based_features_all); i++) {
Tom Lendacky801e4592018-02-21 13:39:51 -06007074 struct kvm_msr_entry msr;
7075
Chenyi Qiang7a5ee6e2019-11-06 14:35:20 +08007076 msr.index = msr_based_features_all[i];
Wanpeng Li66421c12018-02-28 14:03:30 +08007077 if (kvm_get_msr_feature(&msr))
Tom Lendacky801e4592018-02-21 13:39:51 -06007078 continue;
7079
Chenyi Qiang7a5ee6e2019-11-06 14:35:20 +08007080 msr_based_features[num_msr_based_features++] = msr_based_features_all[i];
Tom Lendacky801e4592018-02-21 13:39:51 -06007081 }
Carsten Ottebbd9b642007-10-30 18:44:21 +01007082}
7083
7084static int vcpu_mmio_write(struct kvm_vcpu *vcpu, gpa_t addr, int len,
7085 const void *v)
7086{
Avi Kivity70252a12010-01-19 12:51:22 +02007087 int handled = 0;
7088 int n;
Carsten Ottebbd9b642007-10-30 18:44:21 +01007089
Avi Kivity70252a12010-01-19 12:51:22 +02007090 do {
7091 n = min(len, 8);
Paolo Bonzinibce87cc2016-01-08 13:48:51 +01007092 if (!(lapic_in_kernel(vcpu) &&
Nikolay Nikolaeve32edf42015-03-26 14:39:28 +00007093 !kvm_iodevice_write(vcpu, &vcpu->arch.apic->dev, addr, n, v))
7094 && kvm_io_bus_write(vcpu, KVM_MMIO_BUS, addr, n, v))
Avi Kivity70252a12010-01-19 12:51:22 +02007095 break;
7096 handled += n;
7097 addr += n;
7098 len -= n;
7099 v += n;
7100 } while (len);
7101
7102 return handled;
Carsten Ottebbd9b642007-10-30 18:44:21 +01007103}
7104
7105static int vcpu_mmio_read(struct kvm_vcpu *vcpu, gpa_t addr, int len, void *v)
7106{
Avi Kivity70252a12010-01-19 12:51:22 +02007107 int handled = 0;
7108 int n;
Carsten Ottebbd9b642007-10-30 18:44:21 +01007109
Avi Kivity70252a12010-01-19 12:51:22 +02007110 do {
7111 n = min(len, 8);
Paolo Bonzinibce87cc2016-01-08 13:48:51 +01007112 if (!(lapic_in_kernel(vcpu) &&
Nikolay Nikolaeve32edf42015-03-26 14:39:28 +00007113 !kvm_iodevice_read(vcpu, &vcpu->arch.apic->dev,
7114 addr, n, v))
7115 && kvm_io_bus_read(vcpu, KVM_MMIO_BUS, addr, n, v))
Avi Kivity70252a12010-01-19 12:51:22 +02007116 break;
Wanpeng Lie39d200f2017-12-14 17:40:50 -08007117 trace_kvm_mmio(KVM_TRACE_MMIO_READ, n, addr, v);
Avi Kivity70252a12010-01-19 12:51:22 +02007118 handled += n;
7119 addr += n;
7120 len -= n;
7121 v += n;
7122 } while (len);
7123
7124 return handled;
Carsten Ottebbd9b642007-10-30 18:44:21 +01007125}
7126
Gleb Natapov2dafc6c2010-03-18 15:20:16 +02007127static void kvm_set_segment(struct kvm_vcpu *vcpu,
7128 struct kvm_segment *var, int seg)
7129{
Jason Baronb36464772021-01-14 22:27:56 -05007130 static_call(kvm_x86_set_segment)(vcpu, var, seg);
Gleb Natapov2dafc6c2010-03-18 15:20:16 +02007131}
7132
7133void kvm_get_segment(struct kvm_vcpu *vcpu,
7134 struct kvm_segment *var, int seg)
7135{
Jason Baronb36464772021-01-14 22:27:56 -05007136 static_call(kvm_x86_get_segment)(vcpu, var, seg);
Gleb Natapov2dafc6c2010-03-18 15:20:16 +02007137}
7138
Lai Jiangshan5b22bbe2022-03-11 15:03:41 +08007139gpa_t translate_nested_gpa(struct kvm_vcpu *vcpu, gpa_t gpa, u64 access,
Paolo Bonzini54987b72014-09-02 13:23:06 +02007140 struct x86_exception *exception)
Joerg Roedel02f59dc2010-09-10 17:30:54 +02007141{
Lai Jiangshan1f5a21e2021-11-24 20:20:44 +08007142 struct kvm_mmu *mmu = vcpu->arch.mmu;
Joerg Roedel02f59dc2010-09-10 17:30:54 +02007143 gpa_t t_gpa;
Joerg Roedel02f59dc2010-09-10 17:30:54 +02007144
7145 BUG_ON(!mmu_is_nested(vcpu));
7146
7147 /* NPT walks are always user-walks */
7148 access |= PFERR_USER_MASK;
Lai Jiangshan1f5a21e2021-11-24 20:20:44 +08007149 t_gpa = mmu->gva_to_gpa(vcpu, mmu, gpa, access, exception);
Joerg Roedel02f59dc2010-09-10 17:30:54 +02007150
7151 return t_gpa;
7152}
7153
Avi Kivityab9ae312010-11-22 17:53:26 +02007154gpa_t kvm_mmu_gva_to_gpa_read(struct kvm_vcpu *vcpu, gva_t gva,
7155 struct x86_exception *exception)
Gleb Natapov1871c602010-02-10 14:21:32 +02007156{
Lai Jiangshan1f5a21e2021-11-24 20:20:44 +08007157 struct kvm_mmu *mmu = vcpu->arch.walk_mmu;
7158
Lai Jiangshan5b22bbe2022-03-11 15:03:41 +08007159 u64 access = (static_call(kvm_x86_get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0;
Lai Jiangshan1f5a21e2021-11-24 20:20:44 +08007160 return mmu->gva_to_gpa(vcpu, mmu, gva, access, exception);
Gleb Natapov1871c602010-02-10 14:21:32 +02007161}
Sean Christopherson54f958c2021-04-12 16:21:33 +12007162EXPORT_SYMBOL_GPL(kvm_mmu_gva_to_gpa_read);
Gleb Natapov1871c602010-02-10 14:21:32 +02007163
Avi Kivityab9ae312010-11-22 17:53:26 +02007164 gpa_t kvm_mmu_gva_to_gpa_fetch(struct kvm_vcpu *vcpu, gva_t gva,
7165 struct x86_exception *exception)
Gleb Natapov1871c602010-02-10 14:21:32 +02007166{
Lai Jiangshan1f5a21e2021-11-24 20:20:44 +08007167 struct kvm_mmu *mmu = vcpu->arch.walk_mmu;
7168
Lai Jiangshan5b22bbe2022-03-11 15:03:41 +08007169 u64 access = (static_call(kvm_x86_get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0;
Gleb Natapov1871c602010-02-10 14:21:32 +02007170 access |= PFERR_FETCH_MASK;
Lai Jiangshan1f5a21e2021-11-24 20:20:44 +08007171 return mmu->gva_to_gpa(vcpu, mmu, gva, access, exception);
Gleb Natapov1871c602010-02-10 14:21:32 +02007172}
7173
Avi Kivityab9ae312010-11-22 17:53:26 +02007174gpa_t kvm_mmu_gva_to_gpa_write(struct kvm_vcpu *vcpu, gva_t gva,
7175 struct x86_exception *exception)
Gleb Natapov1871c602010-02-10 14:21:32 +02007176{
Lai Jiangshan1f5a21e2021-11-24 20:20:44 +08007177 struct kvm_mmu *mmu = vcpu->arch.walk_mmu;
7178
Lai Jiangshan5b22bbe2022-03-11 15:03:41 +08007179 u64 access = (static_call(kvm_x86_get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0;
Gleb Natapov1871c602010-02-10 14:21:32 +02007180 access |= PFERR_WRITE_MASK;
Lai Jiangshan1f5a21e2021-11-24 20:20:44 +08007181 return mmu->gva_to_gpa(vcpu, mmu, gva, access, exception);
Gleb Natapov1871c602010-02-10 14:21:32 +02007182}
Sean Christopherson54f958c2021-04-12 16:21:33 +12007183EXPORT_SYMBOL_GPL(kvm_mmu_gva_to_gpa_write);
Gleb Natapov1871c602010-02-10 14:21:32 +02007184
7185/* uses this to access any guest's mapped memory without checking CPL */
Avi Kivityab9ae312010-11-22 17:53:26 +02007186gpa_t kvm_mmu_gva_to_gpa_system(struct kvm_vcpu *vcpu, gva_t gva,
7187 struct x86_exception *exception)
Gleb Natapov1871c602010-02-10 14:21:32 +02007188{
Lai Jiangshan1f5a21e2021-11-24 20:20:44 +08007189 struct kvm_mmu *mmu = vcpu->arch.walk_mmu;
7190
7191 return mmu->gva_to_gpa(vcpu, mmu, gva, 0, exception);
Gleb Natapov1871c602010-02-10 14:21:32 +02007192}
7193
7194static int kvm_read_guest_virt_helper(gva_t addr, void *val, unsigned int bytes,
Lai Jiangshan5b22bbe2022-03-11 15:03:41 +08007195 struct kvm_vcpu *vcpu, u64 access,
Avi Kivitybcc55cb2010-11-22 17:53:22 +02007196 struct x86_exception *exception)
Carsten Ottebbd9b642007-10-30 18:44:21 +01007197{
Lai Jiangshan1f5a21e2021-11-24 20:20:44 +08007198 struct kvm_mmu *mmu = vcpu->arch.walk_mmu;
Carsten Ottebbd9b642007-10-30 18:44:21 +01007199 void *data = val;
Marcelo Tosatti10589a42007-12-20 19:18:22 -05007200 int r = X86EMUL_CONTINUE;
Carsten Ottebbd9b642007-10-30 18:44:21 +01007201
7202 while (bytes) {
Lai Jiangshan1f5a21e2021-11-24 20:20:44 +08007203 gpa_t gpa = mmu->gva_to_gpa(vcpu, mmu, addr, access, exception);
Carsten Ottebbd9b642007-10-30 18:44:21 +01007204 unsigned offset = addr & (PAGE_SIZE-1);
Izik Eidus77c20022008-12-29 01:42:19 +02007205 unsigned toread = min(bytes, (unsigned)PAGE_SIZE - offset);
Carsten Ottebbd9b642007-10-30 18:44:21 +01007206 int ret;
7207
Hou Wenlong6e1d2a32022-07-01 17:24:13 +08007208 if (gpa == INVALID_GPA)
Avi Kivityab9ae312010-11-22 17:53:26 +02007209 return X86EMUL_PROPAGATE_FAULT;
Paolo Bonzini54bf36a2015-04-08 15:39:23 +02007210 ret = kvm_vcpu_read_guest_page(vcpu, gpa >> PAGE_SHIFT, data,
7211 offset, toread);
Marcelo Tosatti10589a42007-12-20 19:18:22 -05007212 if (ret < 0) {
Gleb Natapovc3cd7ff2010-04-28 19:15:35 +03007213 r = X86EMUL_IO_NEEDED;
Marcelo Tosatti10589a42007-12-20 19:18:22 -05007214 goto out;
7215 }
Carsten Ottebbd9b642007-10-30 18:44:21 +01007216
Izik Eidus77c20022008-12-29 01:42:19 +02007217 bytes -= toread;
7218 data += toread;
7219 addr += toread;
Carsten Ottebbd9b642007-10-30 18:44:21 +01007220 }
Marcelo Tosatti10589a42007-12-20 19:18:22 -05007221out:
Marcelo Tosatti10589a42007-12-20 19:18:22 -05007222 return r;
Carsten Ottebbd9b642007-10-30 18:44:21 +01007223}
Izik Eidus77c20022008-12-29 01:42:19 +02007224
Gleb Natapov1871c602010-02-10 14:21:32 +02007225/* used for instruction fetching */
Avi Kivity0f65dd72011-04-20 13:37:53 +03007226static int kvm_fetch_guest_virt(struct x86_emulate_ctxt *ctxt,
7227 gva_t addr, void *val, unsigned int bytes,
Avi Kivitybcc55cb2010-11-22 17:53:22 +02007228 struct x86_exception *exception)
Gleb Natapov1871c602010-02-10 14:21:32 +02007229{
Avi Kivity0f65dd72011-04-20 13:37:53 +03007230 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
Lai Jiangshan1f5a21e2021-11-24 20:20:44 +08007231 struct kvm_mmu *mmu = vcpu->arch.walk_mmu;
Lai Jiangshan5b22bbe2022-03-11 15:03:41 +08007232 u64 access = (static_call(kvm_x86_get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0;
Paolo Bonzini44583cb2014-05-13 14:02:13 +02007233 unsigned offset;
7234 int ret;
Avi Kivity0f65dd72011-04-20 13:37:53 +03007235
Paolo Bonzini44583cb2014-05-13 14:02:13 +02007236 /* Inline kvm_read_guest_virt_helper for speed. */
Lai Jiangshan1f5a21e2021-11-24 20:20:44 +08007237 gpa_t gpa = mmu->gva_to_gpa(vcpu, mmu, addr, access|PFERR_FETCH_MASK,
7238 exception);
Hou Wenlong6e1d2a32022-07-01 17:24:13 +08007239 if (unlikely(gpa == INVALID_GPA))
Paolo Bonzini44583cb2014-05-13 14:02:13 +02007240 return X86EMUL_PROPAGATE_FAULT;
7241
7242 offset = addr & (PAGE_SIZE-1);
7243 if (WARN_ON(offset + bytes > PAGE_SIZE))
7244 bytes = (unsigned)PAGE_SIZE - offset;
Paolo Bonzini54bf36a2015-04-08 15:39:23 +02007245 ret = kvm_vcpu_read_guest_page(vcpu, gpa >> PAGE_SHIFT, val,
7246 offset, bytes);
Paolo Bonzini44583cb2014-05-13 14:02:13 +02007247 if (unlikely(ret < 0))
7248 return X86EMUL_IO_NEEDED;
7249
7250 return X86EMUL_CONTINUE;
Gleb Natapov1871c602010-02-10 14:21:32 +02007251}
7252
Paolo Bonzinice14e868a2018-06-06 17:37:49 +02007253int kvm_read_guest_virt(struct kvm_vcpu *vcpu,
Avi Kivity0f65dd72011-04-20 13:37:53 +03007254 gva_t addr, void *val, unsigned int bytes,
Avi Kivitybcc55cb2010-11-22 17:53:22 +02007255 struct x86_exception *exception)
Gleb Natapov1871c602010-02-10 14:21:32 +02007256{
Lai Jiangshan5b22bbe2022-03-11 15:03:41 +08007257 u64 access = (static_call(kvm_x86_get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0;
Avi Kivity0f65dd72011-04-20 13:37:53 +03007258
Paolo Bonzini353c0952019-01-29 18:41:16 +01007259 /*
7260 * FIXME: this should call handle_emulation_failure if X86EMUL_IO_NEEDED
7261 * is returned, but our callers are not ready for that and they blindly
7262 * call kvm_inject_page_fault. Ensure that they at least do not leak
7263 * uninitialized kernel stack memory into cr2 and error code.
7264 */
7265 memset(exception, 0, sizeof(*exception));
Gleb Natapov1871c602010-02-10 14:21:32 +02007266 return kvm_read_guest_virt_helper(addr, val, bytes, vcpu, access,
Avi Kivitybcc55cb2010-11-22 17:53:22 +02007267 exception);
Gleb Natapov1871c602010-02-10 14:21:32 +02007268}
Nadav Har'El064aea72011-05-25 23:04:56 +03007269EXPORT_SYMBOL_GPL(kvm_read_guest_virt);
Gleb Natapov1871c602010-02-10 14:21:32 +02007270
Paolo Bonzinice14e868a2018-06-06 17:37:49 +02007271static int emulator_read_std(struct x86_emulate_ctxt *ctxt,
7272 gva_t addr, void *val, unsigned int bytes,
Paolo Bonzini3c9fa242018-06-06 17:38:09 +02007273 struct x86_exception *exception, bool system)
Gleb Natapov1871c602010-02-10 14:21:32 +02007274{
Avi Kivity0f65dd72011-04-20 13:37:53 +03007275 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
Lai Jiangshan5b22bbe2022-03-11 15:03:41 +08007276 u64 access = 0;
Paolo Bonzini3c9fa242018-06-06 17:38:09 +02007277
Lai Jiangshan4f4aa802022-03-11 15:03:44 +08007278 if (system)
7279 access |= PFERR_IMPLICIT_ACCESS;
7280 else if (static_call(kvm_x86_get_cpl)(vcpu) == 3)
Paolo Bonzini3c9fa242018-06-06 17:38:09 +02007281 access |= PFERR_USER_MASK;
7282
7283 return kvm_read_guest_virt_helper(addr, val, bytes, vcpu, access, exception);
Gleb Natapov1871c602010-02-10 14:21:32 +02007284}
7285
Radim Krčmář7a036a62015-10-30 16:36:24 +01007286static int kvm_read_guest_phys_system(struct x86_emulate_ctxt *ctxt,
7287 unsigned long addr, void *val, unsigned int bytes)
7288{
7289 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
7290 int r = kvm_vcpu_read_guest(vcpu, addr, val, bytes);
7291
7292 return r < 0 ? X86EMUL_IO_NEEDED : X86EMUL_CONTINUE;
7293}
7294
Paolo Bonzinice14e868a2018-06-06 17:37:49 +02007295static int kvm_write_guest_virt_helper(gva_t addr, void *val, unsigned int bytes,
Lai Jiangshan5b22bbe2022-03-11 15:03:41 +08007296 struct kvm_vcpu *vcpu, u64 access,
Paolo Bonzinice14e868a2018-06-06 17:37:49 +02007297 struct x86_exception *exception)
Izik Eidus77c20022008-12-29 01:42:19 +02007298{
Lai Jiangshan1f5a21e2021-11-24 20:20:44 +08007299 struct kvm_mmu *mmu = vcpu->arch.walk_mmu;
Izik Eidus77c20022008-12-29 01:42:19 +02007300 void *data = val;
7301 int r = X86EMUL_CONTINUE;
7302
7303 while (bytes) {
Lai Jiangshan1f5a21e2021-11-24 20:20:44 +08007304 gpa_t gpa = mmu->gva_to_gpa(vcpu, mmu, addr, access, exception);
Izik Eidus77c20022008-12-29 01:42:19 +02007305 unsigned offset = addr & (PAGE_SIZE-1);
7306 unsigned towrite = min(bytes, (unsigned)PAGE_SIZE - offset);
7307 int ret;
7308
Hou Wenlong6e1d2a32022-07-01 17:24:13 +08007309 if (gpa == INVALID_GPA)
Avi Kivityab9ae312010-11-22 17:53:26 +02007310 return X86EMUL_PROPAGATE_FAULT;
Paolo Bonzini54bf36a2015-04-08 15:39:23 +02007311 ret = kvm_vcpu_write_guest(vcpu, gpa, data, towrite);
Izik Eidus77c20022008-12-29 01:42:19 +02007312 if (ret < 0) {
Gleb Natapovc3cd7ff2010-04-28 19:15:35 +03007313 r = X86EMUL_IO_NEEDED;
Izik Eidus77c20022008-12-29 01:42:19 +02007314 goto out;
7315 }
7316
7317 bytes -= towrite;
7318 data += towrite;
7319 addr += towrite;
7320 }
7321out:
7322 return r;
7323}
Paolo Bonzinice14e868a2018-06-06 17:37:49 +02007324
7325static int emulator_write_std(struct x86_emulate_ctxt *ctxt, gva_t addr, void *val,
Paolo Bonzini3c9fa242018-06-06 17:38:09 +02007326 unsigned int bytes, struct x86_exception *exception,
7327 bool system)
Paolo Bonzinice14e868a2018-06-06 17:37:49 +02007328{
7329 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
Lai Jiangshan5b22bbe2022-03-11 15:03:41 +08007330 u64 access = PFERR_WRITE_MASK;
Paolo Bonzini3c9fa242018-06-06 17:38:09 +02007331
Lai Jiangshan4f4aa802022-03-11 15:03:44 +08007332 if (system)
7333 access |= PFERR_IMPLICIT_ACCESS;
7334 else if (static_call(kvm_x86_get_cpl)(vcpu) == 3)
Paolo Bonzini3c9fa242018-06-06 17:38:09 +02007335 access |= PFERR_USER_MASK;
Paolo Bonzinice14e868a2018-06-06 17:37:49 +02007336
7337 return kvm_write_guest_virt_helper(addr, val, bytes, vcpu,
Paolo Bonzini3c9fa242018-06-06 17:38:09 +02007338 access, exception);
Paolo Bonzinice14e868a2018-06-06 17:37:49 +02007339}
7340
7341int kvm_write_guest_virt_system(struct kvm_vcpu *vcpu, gva_t addr, void *val,
7342 unsigned int bytes, struct x86_exception *exception)
7343{
Paolo Bonzinic595cee2018-07-02 13:07:14 +02007344 /* kvm_write_guest_virt_system can pull in tons of pages. */
7345 vcpu->arch.l1tf_flush_l1d = true;
7346
Paolo Bonzinice14e868a2018-06-06 17:37:49 +02007347 return kvm_write_guest_virt_helper(addr, val, bytes, vcpu,
7348 PFERR_WRITE_MASK, exception);
7349}
Nadav Har'El6a4d7552011-05-25 23:08:00 +03007350EXPORT_SYMBOL_GPL(kvm_write_guest_virt_system);
Izik Eidus77c20022008-12-29 01:42:19 +02007351
Sean Christopherson4d31d9e2022-01-20 01:07:15 +00007352static int kvm_can_emulate_insn(struct kvm_vcpu *vcpu, int emul_type,
7353 void *insn, int insn_len)
7354{
7355 return static_call(kvm_x86_can_emulate_instruction)(vcpu, emul_type,
7356 insn, insn_len);
7357}
7358
Wanpeng Li082d06e2018-04-03 16:28:48 -07007359int handle_ud(struct kvm_vcpu *vcpu)
7360{
Masami Hiramatsub3dc0692019-09-06 22:13:59 +09007361 static const char kvm_emulate_prefix[] = { __KVM_EMULATE_PREFIX };
Sean Christopherson40aaa5b2022-08-30 23:16:14 +00007362 int fep_flags = READ_ONCE(force_emulation_prefix);
Wanpeng Li6c86eed2018-04-03 16:28:49 -07007363 int emul_type = EMULTYPE_TRAP_UD;
Wanpeng Li6c86eed2018-04-03 16:28:49 -07007364 char sig[5]; /* ud2; .ascii "kvm" */
7365 struct x86_exception e;
Wanpeng Li082d06e2018-04-03 16:28:48 -07007366
Sean Christopherson4d31d9e2022-01-20 01:07:15 +00007367 if (unlikely(!kvm_can_emulate_insn(vcpu, emul_type, NULL, 0)))
Sean Christopherson09e3e2a2020-09-15 16:27:02 -07007368 return 1;
7369
Sean Christopherson40aaa5b2022-08-30 23:16:14 +00007370 if (fep_flags &&
Paolo Bonzini3c9fa242018-06-06 17:38:09 +02007371 kvm_read_guest_virt(vcpu, kvm_get_linear_rip(vcpu),
7372 sig, sizeof(sig), &e) == 0 &&
Masami Hiramatsub3dc0692019-09-06 22:13:59 +09007373 memcmp(sig, kvm_emulate_prefix, sizeof(sig)) == 0) {
Sean Christopherson40aaa5b2022-08-30 23:16:14 +00007374 if (fep_flags & KVM_FEP_CLEAR_RFLAGS_RF)
Sean Christophersond500e1e2022-08-30 23:15:51 +00007375 kvm_set_rflags(vcpu, kvm_get_rflags(vcpu) & ~X86_EFLAGS_RF);
Wanpeng Li6c86eed2018-04-03 16:28:49 -07007376 kvm_rip_write(vcpu, kvm_rip_read(vcpu) + sizeof(sig));
Sean Christophersonb4000602019-08-27 14:40:32 -07007377 emul_type = EMULTYPE_TRAP_UD_FORCED;
Wanpeng Li6c86eed2018-04-03 16:28:49 -07007378 }
7379
Sean Christopherson60fc3d02019-08-27 14:40:38 -07007380 return kvm_emulate_instruction(vcpu, emul_type);
Wanpeng Li082d06e2018-04-03 16:28:48 -07007381}
7382EXPORT_SYMBOL_GPL(handle_ud);
7383
Tom Lendacky0f89b202016-12-14 14:59:23 -05007384static int vcpu_is_mmio_gpa(struct kvm_vcpu *vcpu, unsigned long gva,
7385 gpa_t gpa, bool write)
7386{
7387 /* For APIC access vmexit */
7388 if ((gpa & PAGE_MASK) == APIC_DEFAULT_PHYS_BASE)
7389 return 1;
7390
7391 if (vcpu_match_mmio_gpa(vcpu, gpa)) {
7392 trace_vcpu_match_mmio(gva, gpa, write, true);
7393 return 1;
7394 }
7395
7396 return 0;
7397}
7398
Xiao Guangrongaf7cc7d2011-07-12 03:22:46 +08007399static int vcpu_mmio_gva_to_gpa(struct kvm_vcpu *vcpu, unsigned long gva,
7400 gpa_t *gpa, struct x86_exception *exception,
7401 bool write)
7402{
Lai Jiangshan1f5a21e2021-11-24 20:20:44 +08007403 struct kvm_mmu *mmu = vcpu->arch.walk_mmu;
Lai Jiangshan5b22bbe2022-03-11 15:03:41 +08007404 u64 access = ((static_call(kvm_x86_get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0)
Avi Kivity97d64b72012-09-12 14:52:00 +03007405 | (write ? PFERR_WRITE_MASK : 0);
Xiao Guangrongaf7cc7d2011-07-12 03:22:46 +08007406
Huaitong Hanbe94f6b2016-03-22 16:51:20 +08007407 /*
7408 * currently PKRU is only applied to ept enabled guest so
7409 * there is no pkey in EPT page table for L1 guest or EPT
7410 * shadow page table for L2 guest.
7411 */
Sean Christopherson908b7d42021-07-13 09:33:04 -07007412 if (vcpu_match_mmio_gva(vcpu, gva) && (!is_paging(vcpu) ||
7413 !permission_fault(vcpu, vcpu->arch.walk_mmu,
7414 vcpu->arch.mmio_access, 0, access))) {
Xiao Guangrongbebb1062011-07-12 03:23:20 +08007415 *gpa = vcpu->arch.mmio_gfn << PAGE_SHIFT |
7416 (gva & (PAGE_SIZE - 1));
Xiao Guangrong4f022642011-07-12 03:34:24 +08007417 trace_vcpu_match_mmio(gva, *gpa, write, false);
Xiao Guangrongbebb1062011-07-12 03:23:20 +08007418 return 1;
7419 }
7420
Lai Jiangshan1f5a21e2021-11-24 20:20:44 +08007421 *gpa = mmu->gva_to_gpa(vcpu, mmu, gva, access, exception);
Xiao Guangrongaf7cc7d2011-07-12 03:22:46 +08007422
Hou Wenlong6e1d2a32022-07-01 17:24:13 +08007423 if (*gpa == INVALID_GPA)
Xiao Guangrongaf7cc7d2011-07-12 03:22:46 +08007424 return -1;
7425
Tom Lendacky0f89b202016-12-14 14:59:23 -05007426 return vcpu_is_mmio_gpa(vcpu, gva, *gpa, write);
Xiao Guangrongaf7cc7d2011-07-12 03:22:46 +08007427}
7428
Marcelo Tosatti3200f402008-03-29 20:17:59 -03007429int emulator_write_phys(struct kvm_vcpu *vcpu, gpa_t gpa,
Avi Kivitybcc55cb2010-11-22 17:53:22 +02007430 const void *val, int bytes)
Avi Kivity9f811282008-03-02 14:06:05 +02007431{
7432 int ret;
7433
Paolo Bonzini54bf36a2015-04-08 15:39:23 +02007434 ret = kvm_vcpu_write_guest(vcpu, gpa, val, bytes);
Avi Kivity9f811282008-03-02 14:06:05 +02007435 if (ret < 0)
7436 return 0;
Xiao Guangrong0eb05bf2016-02-24 17:51:13 +08007437 kvm_page_track_write(vcpu, gpa, val, bytes);
Avi Kivity9f811282008-03-02 14:06:05 +02007438 return 1;
7439}
7440
Xiao Guangrong77d197b2011-07-13 14:31:50 +08007441struct read_write_emulator_ops {
7442 int (*read_write_prepare)(struct kvm_vcpu *vcpu, void *val,
7443 int bytes);
7444 int (*read_write_emulate)(struct kvm_vcpu *vcpu, gpa_t gpa,
7445 void *val, int bytes);
7446 int (*read_write_mmio)(struct kvm_vcpu *vcpu, gpa_t gpa,
7447 int bytes, void *val);
7448 int (*read_write_exit_mmio)(struct kvm_vcpu *vcpu, gpa_t gpa,
7449 void *val, int bytes);
7450 bool write;
7451};
7452
7453static int read_prepare(struct kvm_vcpu *vcpu, void *val, int bytes)
7454{
7455 if (vcpu->mmio_read_completed) {
Xiao Guangrong77d197b2011-07-13 14:31:50 +08007456 trace_kvm_mmio(KVM_TRACE_MMIO_READ, bytes,
Wanpeng Lie39d200f2017-12-14 17:40:50 -08007457 vcpu->mmio_fragments[0].gpa, val);
Xiao Guangrong77d197b2011-07-13 14:31:50 +08007458 vcpu->mmio_read_completed = 0;
7459 return 1;
7460 }
7461
7462 return 0;
7463}
7464
7465static int read_emulate(struct kvm_vcpu *vcpu, gpa_t gpa,
7466 void *val, int bytes)
7467{
Paolo Bonzini54bf36a2015-04-08 15:39:23 +02007468 return !kvm_vcpu_read_guest(vcpu, gpa, val, bytes);
Xiao Guangrong77d197b2011-07-13 14:31:50 +08007469}
7470
7471static int write_emulate(struct kvm_vcpu *vcpu, gpa_t gpa,
7472 void *val, int bytes)
7473{
7474 return emulator_write_phys(vcpu, gpa, val, bytes);
7475}
7476
7477static int write_mmio(struct kvm_vcpu *vcpu, gpa_t gpa, int bytes, void *val)
7478{
Wanpeng Lie39d200f2017-12-14 17:40:50 -08007479 trace_kvm_mmio(KVM_TRACE_MMIO_WRITE, bytes, gpa, val);
Xiao Guangrong77d197b2011-07-13 14:31:50 +08007480 return vcpu_mmio_write(vcpu, gpa, bytes, val);
7481}
7482
7483static int read_exit_mmio(struct kvm_vcpu *vcpu, gpa_t gpa,
7484 void *val, int bytes)
7485{
Wanpeng Lie39d200f2017-12-14 17:40:50 -08007486 trace_kvm_mmio(KVM_TRACE_MMIO_READ_UNSATISFIED, bytes, gpa, NULL);
Xiao Guangrong77d197b2011-07-13 14:31:50 +08007487 return X86EMUL_IO_NEEDED;
7488}
7489
7490static int write_exit_mmio(struct kvm_vcpu *vcpu, gpa_t gpa,
7491 void *val, int bytes)
7492{
Avi Kivityf78146b2012-04-18 19:22:47 +03007493 struct kvm_mmio_fragment *frag = &vcpu->mmio_fragments[0];
7494
Xiao Guangrong87da7e62012-10-24 14:07:59 +08007495 memcpy(vcpu->run->mmio.data, frag->data, min(8u, frag->len));
Xiao Guangrong77d197b2011-07-13 14:31:50 +08007496 return X86EMUL_CONTINUE;
7497}
7498
Mathias Krause0fbe9b02012-08-30 01:30:17 +02007499static const struct read_write_emulator_ops read_emultor = {
Xiao Guangrong77d197b2011-07-13 14:31:50 +08007500 .read_write_prepare = read_prepare,
7501 .read_write_emulate = read_emulate,
7502 .read_write_mmio = vcpu_mmio_read,
7503 .read_write_exit_mmio = read_exit_mmio,
7504};
7505
Mathias Krause0fbe9b02012-08-30 01:30:17 +02007506static const struct read_write_emulator_ops write_emultor = {
Xiao Guangrong77d197b2011-07-13 14:31:50 +08007507 .read_write_emulate = write_emulate,
7508 .read_write_mmio = write_mmio,
7509 .read_write_exit_mmio = write_exit_mmio,
7510 .write = true,
7511};
7512
Xiao Guangrong22388a32011-07-13 14:32:31 +08007513static int emulator_read_write_onepage(unsigned long addr, void *val,
7514 unsigned int bytes,
7515 struct x86_exception *exception,
7516 struct kvm_vcpu *vcpu,
Mathias Krause0fbe9b02012-08-30 01:30:17 +02007517 const struct read_write_emulator_ops *ops)
Carsten Ottebbd9b642007-10-30 18:44:21 +01007518{
Xiao Guangrongaf7cc7d2011-07-12 03:22:46 +08007519 gpa_t gpa;
7520 int handled, ret;
Xiao Guangrong22388a32011-07-13 14:32:31 +08007521 bool write = ops->write;
Avi Kivityf78146b2012-04-18 19:22:47 +03007522 struct kvm_mmio_fragment *frag;
Sean Christophersonc9b8b072020-02-18 15:29:48 -08007523 struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;
Tom Lendacky0f89b202016-12-14 14:59:23 -05007524
7525 /*
7526 * If the exit was due to a NPF we may already have a GPA.
7527 * If the GPA is present, use it to avoid the GVA to GPA table walk.
7528 * Note, this cannot be used on string operations since string
7529 * operation using rep will only have the initial GPA from the NPF
7530 * occurred.
7531 */
Sean Christopherson744e6992020-02-18 15:03:09 -08007532 if (ctxt->gpa_available && emulator_can_use_gpa(ctxt) &&
7533 (addr & ~PAGE_MASK) == (ctxt->gpa_val & ~PAGE_MASK)) {
7534 gpa = ctxt->gpa_val;
Brijesh Singh618232e2017-08-17 18:36:57 +02007535 ret = vcpu_is_mmio_gpa(vcpu, addr, gpa, write);
7536 } else {
7537 ret = vcpu_mmio_gva_to_gpa(vcpu, addr, &gpa, exception, write);
7538 if (ret < 0)
7539 return X86EMUL_PROPAGATE_FAULT;
Tom Lendacky0f89b202016-12-14 14:59:23 -05007540 }
Xiao Guangrong22388a32011-07-13 14:32:31 +08007541
Brijesh Singh618232e2017-08-17 18:36:57 +02007542 if (!ret && ops->read_write_emulate(vcpu, gpa, val, bytes))
Carsten Ottebbd9b642007-10-30 18:44:21 +01007543 return X86EMUL_CONTINUE;
7544
Carsten Ottebbd9b642007-10-30 18:44:21 +01007545 /*
7546 * Is this MMIO handled locally?
7547 */
Xiao Guangrong22388a32011-07-13 14:32:31 +08007548 handled = ops->read_write_mmio(vcpu, gpa, bytes, val);
Avi Kivity70252a12010-01-19 12:51:22 +02007549 if (handled == bytes)
Carsten Ottebbd9b642007-10-30 18:44:21 +01007550 return X86EMUL_CONTINUE;
Carsten Ottebbd9b642007-10-30 18:44:21 +01007551
Avi Kivity70252a12010-01-19 12:51:22 +02007552 gpa += handled;
7553 bytes -= handled;
7554 val += handled;
7555
Xiao Guangrong87da7e62012-10-24 14:07:59 +08007556 WARN_ON(vcpu->mmio_nr_fragments >= KVM_MAX_MMIO_FRAGMENTS);
7557 frag = &vcpu->mmio_fragments[vcpu->mmio_nr_fragments++];
7558 frag->gpa = gpa;
7559 frag->data = val;
7560 frag->len = bytes;
Avi Kivityf78146b2012-04-18 19:22:47 +03007561 return X86EMUL_CONTINUE;
Carsten Ottebbd9b642007-10-30 18:44:21 +01007562}
7563
Xiubo Li52eb5a62015-03-13 17:39:45 +08007564static int emulator_read_write(struct x86_emulate_ctxt *ctxt,
7565 unsigned long addr,
Xiao Guangrong22388a32011-07-13 14:32:31 +08007566 void *val, unsigned int bytes,
7567 struct x86_exception *exception,
Mathias Krause0fbe9b02012-08-30 01:30:17 +02007568 const struct read_write_emulator_ops *ops)
Carsten Ottebbd9b642007-10-30 18:44:21 +01007569{
Avi Kivity0f65dd72011-04-20 13:37:53 +03007570 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
Avi Kivityf78146b2012-04-18 19:22:47 +03007571 gpa_t gpa;
7572 int rc;
7573
7574 if (ops->read_write_prepare &&
7575 ops->read_write_prepare(vcpu, val, bytes))
7576 return X86EMUL_CONTINUE;
7577
7578 vcpu->mmio_nr_fragments = 0;
Avi Kivity0f65dd72011-04-20 13:37:53 +03007579
Carsten Ottebbd9b642007-10-30 18:44:21 +01007580 /* Crossing a page boundary? */
7581 if (((addr + bytes - 1) ^ addr) & PAGE_MASK) {
Avi Kivityf78146b2012-04-18 19:22:47 +03007582 int now;
Carsten Ottebbd9b642007-10-30 18:44:21 +01007583
7584 now = -addr & ~PAGE_MASK;
Xiao Guangrong22388a32011-07-13 14:32:31 +08007585 rc = emulator_read_write_onepage(addr, val, now, exception,
7586 vcpu, ops);
7587
Carsten Ottebbd9b642007-10-30 18:44:21 +01007588 if (rc != X86EMUL_CONTINUE)
7589 return rc;
7590 addr += now;
Nadav Amitbac155312015-01-26 09:32:26 +02007591 if (ctxt->mode != X86EMUL_MODE_PROT64)
7592 addr = (u32)addr;
Carsten Ottebbd9b642007-10-30 18:44:21 +01007593 val += now;
7594 bytes -= now;
7595 }
Xiao Guangrong22388a32011-07-13 14:32:31 +08007596
Avi Kivityf78146b2012-04-18 19:22:47 +03007597 rc = emulator_read_write_onepage(addr, val, bytes, exception,
7598 vcpu, ops);
7599 if (rc != X86EMUL_CONTINUE)
7600 return rc;
7601
7602 if (!vcpu->mmio_nr_fragments)
7603 return rc;
7604
7605 gpa = vcpu->mmio_fragments[0].gpa;
7606
7607 vcpu->mmio_needed = 1;
7608 vcpu->mmio_cur_fragment = 0;
7609
Xiao Guangrong87da7e62012-10-24 14:07:59 +08007610 vcpu->run->mmio.len = min(8u, vcpu->mmio_fragments[0].len);
Avi Kivityf78146b2012-04-18 19:22:47 +03007611 vcpu->run->mmio.is_write = vcpu->mmio_is_write = ops->write;
7612 vcpu->run->exit_reason = KVM_EXIT_MMIO;
7613 vcpu->run->mmio.phys_addr = gpa;
7614
7615 return ops->read_write_exit_mmio(vcpu, gpa, val, bytes);
Xiao Guangrong22388a32011-07-13 14:32:31 +08007616}
7617
7618static int emulator_read_emulated(struct x86_emulate_ctxt *ctxt,
7619 unsigned long addr,
7620 void *val,
7621 unsigned int bytes,
7622 struct x86_exception *exception)
7623{
7624 return emulator_read_write(ctxt, addr, val, bytes,
7625 exception, &read_emultor);
7626}
7627
Xiubo Li52eb5a62015-03-13 17:39:45 +08007628static int emulator_write_emulated(struct x86_emulate_ctxt *ctxt,
Xiao Guangrong22388a32011-07-13 14:32:31 +08007629 unsigned long addr,
7630 const void *val,
7631 unsigned int bytes,
7632 struct x86_exception *exception)
7633{
7634 return emulator_read_write(ctxt, addr, (void *)val, bytes,
7635 exception, &write_emultor);
Carsten Ottebbd9b642007-10-30 18:44:21 +01007636}
Carsten Ottebbd9b642007-10-30 18:44:21 +01007637
Sean Christopherson1c2361f2022-02-02 00:49:44 +00007638#define emulator_try_cmpxchg_user(t, ptr, old, new) \
7639 (__try_cmpxchg_user((t __user *)(ptr), (t *)(old), *(t *)(new), efault ## t))
Avi Kivitydaea3e72010-03-15 13:59:54 +02007640
Avi Kivity0f65dd72011-04-20 13:37:53 +03007641static int emulator_cmpxchg_emulated(struct x86_emulate_ctxt *ctxt,
7642 unsigned long addr,
Carsten Ottebbd9b642007-10-30 18:44:21 +01007643 const void *old,
7644 const void *new,
7645 unsigned int bytes,
Avi Kivity0f65dd72011-04-20 13:37:53 +03007646 struct x86_exception *exception)
Carsten Ottebbd9b642007-10-30 18:44:21 +01007647{
Avi Kivity0f65dd72011-04-20 13:37:53 +03007648 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
Xiaoyao Li9de6fe32020-04-10 13:54:01 +02007649 u64 page_line_mask;
Sean Christopherson1c2361f2022-02-02 00:49:44 +00007650 unsigned long hva;
Avi Kivitydaea3e72010-03-15 13:59:54 +02007651 gpa_t gpa;
Sean Christopherson1c2361f2022-02-02 00:49:44 +00007652 int r;
Avi Kivitydaea3e72010-03-15 13:59:54 +02007653
Marcelo Tosatti2bacc552007-12-12 10:46:12 -05007654 /* guests cmpxchg8b have to be emulated atomically */
Avi Kivitydaea3e72010-03-15 13:59:54 +02007655 if (bytes > 8 || (bytes & (bytes - 1)))
7656 goto emul_write;
Marcelo Tosatti2bacc552007-12-12 10:46:12 -05007657
Avi Kivitydaea3e72010-03-15 13:59:54 +02007658 gpa = kvm_mmu_gva_to_gpa_write(vcpu, addr, NULL);
Marcelo Tosatti10589a42007-12-20 19:18:22 -05007659
Hou Wenlong6e1d2a32022-07-01 17:24:13 +08007660 if (gpa == INVALID_GPA ||
Avi Kivitydaea3e72010-03-15 13:59:54 +02007661 (gpa & PAGE_MASK) == APIC_DEFAULT_PHYS_BASE)
7662 goto emul_write;
Marcelo Tosatti2bacc552007-12-12 10:46:12 -05007663
Xiaoyao Li9de6fe32020-04-10 13:54:01 +02007664 /*
7665 * Emulate the atomic as a straight write to avoid #AC if SLD is
7666 * enabled in the host and the access splits a cache line.
7667 */
7668 if (boot_cpu_has(X86_FEATURE_SPLIT_LOCK_DETECT))
7669 page_line_mask = ~(cache_line_size() - 1);
7670 else
7671 page_line_mask = PAGE_MASK;
7672
7673 if (((gpa + bytes - 1) & page_line_mask) != (gpa & page_line_mask))
Avi Kivitydaea3e72010-03-15 13:59:54 +02007674 goto emul_write;
Marcelo Tosatti2bacc552007-12-12 10:46:12 -05007675
Sean Christopherson1c2361f2022-02-02 00:49:44 +00007676 hva = kvm_vcpu_gfn_to_hva(vcpu, gpa_to_gfn(gpa));
Maxim Levitsky33fbe6b2022-05-12 13:14:20 +03007677 if (kvm_is_error_hva(hva))
Wei Yongjunc19b8bd2010-07-15 08:51:58 +08007678 goto emul_write;
Izik Eidus72dc67a62008-02-10 18:04:15 +02007679
Sean Christopherson1c2361f2022-02-02 00:49:44 +00007680 hva += offset_in_page(gpa);
KarimAllah Ahmed42e35f82019-01-31 21:24:39 +01007681
Avi Kivitydaea3e72010-03-15 13:59:54 +02007682 switch (bytes) {
7683 case 1:
Sean Christopherson1c2361f2022-02-02 00:49:44 +00007684 r = emulator_try_cmpxchg_user(u8, hva, old, new);
Avi Kivitydaea3e72010-03-15 13:59:54 +02007685 break;
7686 case 2:
Sean Christopherson1c2361f2022-02-02 00:49:44 +00007687 r = emulator_try_cmpxchg_user(u16, hva, old, new);
Avi Kivitydaea3e72010-03-15 13:59:54 +02007688 break;
7689 case 4:
Sean Christopherson1c2361f2022-02-02 00:49:44 +00007690 r = emulator_try_cmpxchg_user(u32, hva, old, new);
Avi Kivitydaea3e72010-03-15 13:59:54 +02007691 break;
7692 case 8:
Sean Christopherson1c2361f2022-02-02 00:49:44 +00007693 r = emulator_try_cmpxchg_user(u64, hva, old, new);
Avi Kivitydaea3e72010-03-15 13:59:54 +02007694 break;
7695 default:
7696 BUG();
Marcelo Tosatti2bacc552007-12-12 10:46:12 -05007697 }
KarimAllah Ahmed42e35f82019-01-31 21:24:39 +01007698
Sean Christopherson1c2361f2022-02-02 00:49:44 +00007699 if (r < 0)
Sean Christopherson5d6c7de2022-02-02 00:49:45 +00007700 return X86EMUL_UNHANDLEABLE;
Sean Christopherson1c2361f2022-02-02 00:49:44 +00007701 if (r)
Avi Kivitydaea3e72010-03-15 13:59:54 +02007702 return X86EMUL_CMPXCHG_FAILED;
7703
Xiao Guangrong0eb05bf2016-02-24 17:51:13 +08007704 kvm_page_track_write(vcpu, gpa, new, bytes);
Gleb Natapov8f6abd02010-04-13 10:21:56 +03007705
7706 return X86EMUL_CONTINUE;
Avi Kivity4a5f48f2010-03-15 13:59:55 +02007707
Marcelo Tosatti3200f402008-03-29 20:17:59 -03007708emul_write:
Avi Kivitydaea3e72010-03-15 13:59:54 +02007709 printk_once(KERN_WARNING "kvm: emulating exchange as write\n");
Marcelo Tosatti2bacc552007-12-12 10:46:12 -05007710
Avi Kivity0f65dd72011-04-20 13:37:53 +03007711 return emulator_write_emulated(ctxt, addr, new, bytes, exception);
Carsten Ottebbd9b642007-10-30 18:44:21 +01007712}
7713
Xiao Guangrong6f6fbe92011-09-22 16:55:10 +08007714static int emulator_pio_in_out(struct kvm_vcpu *vcpu, int size,
Paolo Bonzini30d583f2021-10-22 08:07:19 -04007715 unsigned short port, void *data,
Xiao Guangrong6f6fbe92011-09-22 16:55:10 +08007716 unsigned int count, bool in)
7717{
Paolo Bonzini0f87ac22021-10-22 06:50:06 -04007718 unsigned i;
7719 int r;
7720
Paolo Bonzini30d583f2021-10-22 08:07:19 -04007721 WARN_ON_ONCE(vcpu->arch.pio.count);
Paolo Bonzini0f87ac22021-10-22 06:50:06 -04007722 for (i = 0; i < count; i++) {
7723 if (in)
7724 r = kvm_io_bus_read(vcpu, KVM_PIO_BUS, port, size, data);
7725 else
7726 r = kvm_io_bus_write(vcpu, KVM_PIO_BUS, port, size, data);
Paolo Bonzini35ab3b72022-06-15 11:05:06 -04007727
7728 if (r) {
7729 if (i == 0)
7730 goto userspace_io;
7731
7732 /*
7733 * Userspace must have unregistered the device while PIO
Paolo Bonzini0c05e102022-06-15 10:24:01 -04007734 * was running. Drop writes / read as 0.
Paolo Bonzini35ab3b72022-06-15 11:05:06 -04007735 */
Paolo Bonzini0c05e102022-06-15 10:24:01 -04007736 if (in)
7737 memset(data, 0, size * (count - i));
Paolo Bonzini35ab3b72022-06-15 11:05:06 -04007738 break;
7739 }
7740
Paolo Bonzini0f87ac22021-10-22 06:50:06 -04007741 data += size;
7742 }
7743 return 1;
Xiao Guangrong6f6fbe92011-09-22 16:55:10 +08007744
Paolo Bonzini0f87ac22021-10-22 06:50:06 -04007745userspace_io:
Xiao Guangrong6f6fbe92011-09-22 16:55:10 +08007746 vcpu->arch.pio.port = port;
7747 vcpu->arch.pio.in = in;
Paolo Bonzini0c05e102022-06-15 10:24:01 -04007748 vcpu->arch.pio.count = count;
Xiao Guangrong6f6fbe92011-09-22 16:55:10 +08007749 vcpu->arch.pio.size = size;
7750
Paolo Bonzini0c05e102022-06-15 10:24:01 -04007751 if (in)
7752 memset(vcpu->arch.pio_data, 0, size * count);
7753 else
7754 memcpy(vcpu->arch.pio_data, data, size * count);
Xiao Guangrong6f6fbe92011-09-22 16:55:10 +08007755
7756 vcpu->run->exit_reason = KVM_EXIT_IO;
7757 vcpu->run->io.direction = in ? KVM_EXIT_IO_IN : KVM_EXIT_IO_OUT;
7758 vcpu->run->io.size = size;
7759 vcpu->run->io.data_offset = KVM_PIO_PAGE_OFFSET * PAGE_SIZE;
7760 vcpu->run->io.count = count;
7761 vcpu->run->io.port = port;
Xiao Guangrong6f6fbe92011-09-22 16:55:10 +08007762 return 0;
7763}
Gleb Natapovcf8f70b2010-03-18 15:20:23 +02007764
Paolo Bonzinif35cee42021-10-22 08:19:48 -04007765static int emulator_pio_in(struct kvm_vcpu *vcpu, int size,
7766 unsigned short port, void *val, unsigned int count)
Paolo Bonzini3b27de22021-10-13 12:32:02 -04007767{
Paolo Bonzini0c05e102022-06-15 10:24:01 -04007768 int r = emulator_pio_in_out(vcpu, size, port, val, count, true);
7769 if (r)
7770 trace_kvm_pio(KVM_PIO_IN, port, size, count, val);
7771
7772 return r;
Paolo Bonzini3b27de22021-10-13 12:32:02 -04007773}
7774
Paolo Bonzini6b5efc92021-10-12 12:35:20 -04007775static void complete_emulator_pio_in(struct kvm_vcpu *vcpu, void *val)
Paolo Bonzini3b27de22021-10-13 12:32:02 -04007776{
Paolo Bonzini6b5efc92021-10-12 12:35:20 -04007777 int size = vcpu->arch.pio.size;
Paolo Bonzini0c05e102022-06-15 10:24:01 -04007778 unsigned int count = vcpu->arch.pio.count;
Paolo Bonzini6b5efc92021-10-12 12:35:20 -04007779 memcpy(val, vcpu->arch.pio_data, size * count);
7780 trace_kvm_pio(KVM_PIO_IN, vcpu->arch.pio.port, size, count, vcpu->arch.pio_data);
Paolo Bonzini3b27de22021-10-13 12:32:02 -04007781 vcpu->arch.pio.count = 0;
7782}
7783
Paolo Bonzinif35cee42021-10-22 08:19:48 -04007784static int emulator_pio_in_emulated(struct x86_emulate_ctxt *ctxt,
7785 int size, unsigned short port, void *val,
7786 unsigned int count)
Gleb Natapovcf8f70b2010-03-18 15:20:23 +02007787{
Paolo Bonzinif35cee42021-10-22 08:19:48 -04007788 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
Paolo Bonzini3b27de22021-10-13 12:32:02 -04007789 if (vcpu->arch.pio.count) {
Sean Christophersond07898e2021-10-25 13:13:10 -07007790 /*
7791 * Complete a previous iteration that required userspace I/O.
7792 * Note, @count isn't guaranteed to match pio.count as userspace
7793 * can modify ECX before rerunning the vCPU. Ignore any such
7794 * shenanigans as KVM doesn't support modifying the rep count,
7795 * and the emulator ensures @count doesn't overflow the buffer.
7796 */
Paolo Bonzini0c05e102022-06-15 10:24:01 -04007797 complete_emulator_pio_in(vcpu, val);
7798 return 1;
Gleb Natapovcf8f70b2010-03-18 15:20:23 +02007799 }
7800
Paolo Bonzinif35cee42021-10-22 08:19:48 -04007801 return emulator_pio_in(vcpu, size, port, val, count);
Sean Christopherson2e3bb4d2020-02-18 15:29:41 -08007802}
7803
7804static int emulator_pio_out(struct kvm_vcpu *vcpu, int size,
7805 unsigned short port, const void *val,
7806 unsigned int count)
7807{
Paolo Bonzini30d583f2021-10-22 08:07:19 -04007808 trace_kvm_pio(KVM_PIO_OUT, port, size, count, val);
Paolo Bonzini0c05e102022-06-15 10:24:01 -04007809 return emulator_pio_in_out(vcpu, size, port, (void *)val, count, false);
Gleb Natapovcf8f70b2010-03-18 15:20:23 +02007810}
7811
Carsten Ottebbd9b642007-10-30 18:44:21 +01007812static int emulator_pio_out_emulated(struct x86_emulate_ctxt *ctxt,
7813 int size, unsigned short port,
7814 const void *val, unsigned int count)
7815{
Sean Christopherson2e3bb4d2020-02-18 15:29:41 -08007816 return emulator_pio_out(emul_to_vcpu(ctxt), size, port, val, count);
Carsten Ottebbd9b642007-10-30 18:44:21 +01007817}
7818
7819static unsigned long get_segment_base(struct kvm_vcpu *vcpu, int seg)
7820{
Jason Baronb36464772021-01-14 22:27:56 -05007821 return static_call(kvm_x86_get_segment_base)(vcpu, seg);
Carsten Ottebbd9b642007-10-30 18:44:21 +01007822}
7823
Avi Kivity3cb16fe2011-04-20 15:38:44 +03007824static void emulator_invlpg(struct x86_emulate_ctxt *ctxt, ulong address)
Carsten Ottebbd9b642007-10-30 18:44:21 +01007825{
Avi Kivity3cb16fe2011-04-20 15:38:44 +03007826 kvm_mmu_invlpg(emul_to_vcpu(ctxt), address);
Carsten Ottebbd9b642007-10-30 18:44:21 +01007827}
7828
Jiang Biaoae6a2372016-11-07 08:54:51 +08007829static int kvm_emulate_wbinvd_noskip(struct kvm_vcpu *vcpu)
Sheng Yangf5f48ee2010-06-30 12:25:15 +08007830{
7831 if (!need_emulate_wbinvd(vcpu))
7832 return X86EMUL_CONTINUE;
7833
Jason Baronb36464772021-01-14 22:27:56 -05007834 if (static_call(kvm_x86_has_wbinvd_exit)()) {
Jan Kiszka2eec7342010-11-01 14:01:29 +01007835 int cpu = get_cpu();
7836
7837 cpumask_set_cpu(cpu, vcpu->arch.wbinvd_dirty_mask);
Wanpeng Lic2162e12021-03-12 10:45:51 +08007838 on_each_cpu_mask(vcpu->arch.wbinvd_dirty_mask,
Sheng Yangf5f48ee2010-06-30 12:25:15 +08007839 wbinvd_ipi, NULL, 1);
Jan Kiszka2eec7342010-11-01 14:01:29 +01007840 put_cpu();
Sheng Yangf5f48ee2010-06-30 12:25:15 +08007841 cpumask_clear(vcpu->arch.wbinvd_dirty_mask);
Jan Kiszka2eec7342010-11-01 14:01:29 +01007842 } else
7843 wbinvd();
Sheng Yangf5f48ee2010-06-30 12:25:15 +08007844 return X86EMUL_CONTINUE;
7845}
Joel Schopp5cb56052015-03-02 13:43:31 -06007846
7847int kvm_emulate_wbinvd(struct kvm_vcpu *vcpu)
7848{
Kyle Huey6affcbe2016-11-29 12:40:40 -08007849 kvm_emulate_wbinvd_noskip(vcpu);
7850 return kvm_skip_emulated_instruction(vcpu);
Joel Schopp5cb56052015-03-02 13:43:31 -06007851}
Sheng Yangf5f48ee2010-06-30 12:25:15 +08007852EXPORT_SYMBOL_GPL(kvm_emulate_wbinvd);
7853
Joel Schopp5cb56052015-03-02 13:43:31 -06007854
7855
Avi Kivitybcaf5cc2011-04-20 15:53:23 +03007856static void emulator_wbinvd(struct x86_emulate_ctxt *ctxt)
7857{
Joel Schopp5cb56052015-03-02 13:43:31 -06007858 kvm_emulate_wbinvd_noskip(emul_to_vcpu(ctxt));
Avi Kivitybcaf5cc2011-04-20 15:53:23 +03007859}
7860
Paolo Bonzini29d6ca42021-02-03 03:42:41 -05007861static void emulator_get_dr(struct x86_emulate_ctxt *ctxt, int dr,
7862 unsigned long *dest)
Carsten Ottebbd9b642007-10-30 18:44:21 +01007863{
Paolo Bonzini29d6ca42021-02-03 03:42:41 -05007864 kvm_get_dr(emul_to_vcpu(ctxt), dr, dest);
Carsten Ottebbd9b642007-10-30 18:44:21 +01007865}
7866
Xiubo Li52eb5a62015-03-13 17:39:45 +08007867static int emulator_set_dr(struct x86_emulate_ctxt *ctxt, int dr,
7868 unsigned long value)
Carsten Ottebbd9b642007-10-30 18:44:21 +01007869{
Gleb Natapov338dbc92010-04-28 19:15:32 +03007870
Paolo Bonzini996ff542020-12-14 07:49:54 -05007871 return kvm_set_dr(emul_to_vcpu(ctxt), dr, value);
Carsten Ottebbd9b642007-10-30 18:44:21 +01007872}
7873
Gleb Natapov52a46612010-03-18 15:20:03 +02007874static u64 mk_cr_64(u64 curr_cr, u32 new_val)
7875{
7876 return (curr_cr & ~((1ULL << 32) - 1)) | new_val;
7877}
7878
Avi Kivity717746e2011-04-20 13:37:53 +03007879static unsigned long emulator_get_cr(struct x86_emulate_ctxt *ctxt, int cr)
Gleb Natapov52a46612010-03-18 15:20:03 +02007880{
Avi Kivity717746e2011-04-20 13:37:53 +03007881 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
Gleb Natapov52a46612010-03-18 15:20:03 +02007882 unsigned long value;
7883
7884 switch (cr) {
7885 case 0:
7886 value = kvm_read_cr0(vcpu);
7887 break;
7888 case 2:
7889 value = vcpu->arch.cr2;
7890 break;
7891 case 3:
Avi Kivity9f8fe502010-12-05 17:30:00 +02007892 value = kvm_read_cr3(vcpu);
Gleb Natapov52a46612010-03-18 15:20:03 +02007893 break;
7894 case 4:
7895 value = kvm_read_cr4(vcpu);
7896 break;
7897 case 8:
7898 value = kvm_get_cr8(vcpu);
7899 break;
7900 default:
Christoffer Dalla737f252012-06-03 21:17:48 +03007901 kvm_err("%s: unexpected cr %u\n", __func__, cr);
Gleb Natapov52a46612010-03-18 15:20:03 +02007902 return 0;
7903 }
7904
7905 return value;
7906}
7907
Avi Kivity717746e2011-04-20 13:37:53 +03007908static int emulator_set_cr(struct x86_emulate_ctxt *ctxt, int cr, ulong val)
Gleb Natapov52a46612010-03-18 15:20:03 +02007909{
Avi Kivity717746e2011-04-20 13:37:53 +03007910 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
Gleb Natapov0f122442010-04-28 19:15:31 +03007911 int res = 0;
7912
Gleb Natapov52a46612010-03-18 15:20:03 +02007913 switch (cr) {
7914 case 0:
Avi Kivity49a9b072010-06-10 17:02:14 +03007915 res = kvm_set_cr0(vcpu, mk_cr_64(kvm_read_cr0(vcpu), val));
Gleb Natapov52a46612010-03-18 15:20:03 +02007916 break;
7917 case 2:
7918 vcpu->arch.cr2 = val;
7919 break;
7920 case 3:
Avi Kivity23902182010-06-10 17:02:16 +03007921 res = kvm_set_cr3(vcpu, val);
Gleb Natapov52a46612010-03-18 15:20:03 +02007922 break;
7923 case 4:
Avi Kivitya83b29c2010-06-10 17:02:15 +03007924 res = kvm_set_cr4(vcpu, mk_cr_64(kvm_read_cr4(vcpu), val));
Gleb Natapov52a46612010-03-18 15:20:03 +02007925 break;
7926 case 8:
Andre Przywaraeea1cff2010-12-21 11:12:00 +01007927 res = kvm_set_cr8(vcpu, val);
Gleb Natapov52a46612010-03-18 15:20:03 +02007928 break;
7929 default:
Christoffer Dalla737f252012-06-03 21:17:48 +03007930 kvm_err("%s: unexpected cr %u\n", __func__, cr);
Gleb Natapov0f122442010-04-28 19:15:31 +03007931 res = -1;
Gleb Natapov52a46612010-03-18 15:20:03 +02007932 }
Gleb Natapov0f122442010-04-28 19:15:31 +03007933
7934 return res;
Gleb Natapov52a46612010-03-18 15:20:03 +02007935}
7936
Avi Kivity717746e2011-04-20 13:37:53 +03007937static int emulator_get_cpl(struct x86_emulate_ctxt *ctxt)
Gleb Natapov9c537242010-03-18 15:20:05 +02007938{
Jason Baronb36464772021-01-14 22:27:56 -05007939 return static_call(kvm_x86_get_cpl)(emul_to_vcpu(ctxt));
Gleb Natapov9c537242010-03-18 15:20:05 +02007940}
7941
Avi Kivity4bff1e862011-04-20 13:37:53 +03007942static void emulator_get_gdt(struct x86_emulate_ctxt *ctxt, struct desc_ptr *dt)
Gleb Natapov2dafc6c2010-03-18 15:20:16 +02007943{
Jason Baronb36464772021-01-14 22:27:56 -05007944 static_call(kvm_x86_get_gdt)(emul_to_vcpu(ctxt), dt);
Gleb Natapov2dafc6c2010-03-18 15:20:16 +02007945}
7946
Avi Kivity4bff1e862011-04-20 13:37:53 +03007947static void emulator_get_idt(struct x86_emulate_ctxt *ctxt, struct desc_ptr *dt)
Mohammed Gamal160ce1f2010-08-04 05:44:24 +03007948{
Jason Baronb36464772021-01-14 22:27:56 -05007949 static_call(kvm_x86_get_idt)(emul_to_vcpu(ctxt), dt);
Mohammed Gamal160ce1f2010-08-04 05:44:24 +03007950}
7951
Avi Kivity1ac9d0c2011-04-20 15:12:00 +03007952static void emulator_set_gdt(struct x86_emulate_ctxt *ctxt, struct desc_ptr *dt)
7953{
Jason Baronb36464772021-01-14 22:27:56 -05007954 static_call(kvm_x86_set_gdt)(emul_to_vcpu(ctxt), dt);
Avi Kivity1ac9d0c2011-04-20 15:12:00 +03007955}
7956
7957static void emulator_set_idt(struct x86_emulate_ctxt *ctxt, struct desc_ptr *dt)
7958{
Jason Baronb36464772021-01-14 22:27:56 -05007959 static_call(kvm_x86_set_idt)(emul_to_vcpu(ctxt), dt);
Avi Kivity1ac9d0c2011-04-20 15:12:00 +03007960}
7961
Avi Kivity4bff1e862011-04-20 13:37:53 +03007962static unsigned long emulator_get_cached_segment_base(
7963 struct x86_emulate_ctxt *ctxt, int seg)
Gleb Natapov5951c442010-04-28 19:15:29 +03007964{
Avi Kivity4bff1e862011-04-20 13:37:53 +03007965 return get_segment_base(emul_to_vcpu(ctxt), seg);
Gleb Natapov5951c442010-04-28 19:15:29 +03007966}
7967
Avi Kivity1aa36612011-04-27 13:20:30 +03007968static bool emulator_get_segment(struct x86_emulate_ctxt *ctxt, u16 *selector,
7969 struct desc_struct *desc, u32 *base3,
7970 int seg)
Gleb Natapov2dafc6c2010-03-18 15:20:16 +02007971{
7972 struct kvm_segment var;
7973
Avi Kivity4bff1e862011-04-20 13:37:53 +03007974 kvm_get_segment(emul_to_vcpu(ctxt), &var, seg);
Avi Kivity1aa36612011-04-27 13:20:30 +03007975 *selector = var.selector;
Gleb Natapov2dafc6c2010-03-18 15:20:16 +02007976
Gleb Natapov378a8b02013-01-21 15:36:48 +02007977 if (var.unusable) {
7978 memset(desc, 0, sizeof(*desc));
Radim Krčmářf0367ee2017-05-18 19:37:30 +02007979 if (base3)
7980 *base3 = 0;
Gleb Natapov2dafc6c2010-03-18 15:20:16 +02007981 return false;
Gleb Natapov378a8b02013-01-21 15:36:48 +02007982 }
Gleb Natapov2dafc6c2010-03-18 15:20:16 +02007983
7984 if (var.g)
7985 var.limit >>= 12;
7986 set_desc_limit(desc, var.limit);
7987 set_desc_base(desc, (unsigned long)var.base);
Gleb Natapov5601d052011-03-07 14:55:06 +02007988#ifdef CONFIG_X86_64
7989 if (base3)
7990 *base3 = var.base >> 32;
7991#endif
Gleb Natapov2dafc6c2010-03-18 15:20:16 +02007992 desc->type = var.type;
7993 desc->s = var.s;
7994 desc->dpl = var.dpl;
7995 desc->p = var.present;
7996 desc->avl = var.avl;
7997 desc->l = var.l;
7998 desc->d = var.db;
7999 desc->g = var.g;
8000
8001 return true;
8002}
8003
Avi Kivity1aa36612011-04-27 13:20:30 +03008004static void emulator_set_segment(struct x86_emulate_ctxt *ctxt, u16 selector,
8005 struct desc_struct *desc, u32 base3,
8006 int seg)
Gleb Natapov2dafc6c2010-03-18 15:20:16 +02008007{
Avi Kivity4bff1e862011-04-20 13:37:53 +03008008 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
Gleb Natapov2dafc6c2010-03-18 15:20:16 +02008009 struct kvm_segment var;
8010
Avi Kivity1aa36612011-04-27 13:20:30 +03008011 var.selector = selector;
Gleb Natapov2dafc6c2010-03-18 15:20:16 +02008012 var.base = get_desc_base(desc);
Gleb Natapov5601d052011-03-07 14:55:06 +02008013#ifdef CONFIG_X86_64
8014 var.base |= ((u64)base3) << 32;
8015#endif
Gleb Natapov2dafc6c2010-03-18 15:20:16 +02008016 var.limit = get_desc_limit(desc);
8017 if (desc->g)
8018 var.limit = (var.limit << 12) | 0xfff;
8019 var.type = desc->type;
Gleb Natapov2dafc6c2010-03-18 15:20:16 +02008020 var.dpl = desc->dpl;
8021 var.db = desc->d;
8022 var.s = desc->s;
8023 var.l = desc->l;
8024 var.g = desc->g;
8025 var.avl = desc->avl;
8026 var.present = desc->p;
8027 var.unusable = !var.present;
8028 var.padding = 0;
8029
8030 kvm_set_segment(vcpu, &var, seg);
8031 return;
8032}
8033
Hou Wenlongac8d6ca2022-03-07 20:26:33 +08008034static int emulator_get_msr_with_filter(struct x86_emulate_ctxt *ctxt,
8035 u32 msr_index, u64 *pdata)
Avi Kivity717746e2011-04-20 13:37:53 +03008036{
Alexander Graf1ae09952020-09-25 16:34:16 +02008037 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
8038 int r;
8039
Hou Wenlongac8d6ca2022-03-07 20:26:33 +08008040 r = kvm_get_msr_with_filter(vcpu, msr_index, pdata);
Hou Wenlong36d546d2022-09-02 10:47:00 +08008041 if (r < 0)
8042 return X86EMUL_UNHANDLEABLE;
Alexander Graf1ae09952020-09-25 16:34:16 +02008043
Hou Wenlong36d546d2022-09-02 10:47:00 +08008044 if (r) {
8045 if (kvm_msr_user_space(vcpu, msr_index, KVM_EXIT_X86_RDMSR, 0,
8046 complete_emulated_rdmsr, r))
8047 return X86EMUL_IO_NEEDED;
Hou Wenlong794663e2022-09-02 10:47:01 +08008048
8049 trace_kvm_msr_read_ex(msr_index);
Hou Wenlong36d546d2022-09-02 10:47:00 +08008050 return X86EMUL_PROPAGATE_FAULT;
Alexander Graf1ae09952020-09-25 16:34:16 +02008051 }
8052
Hou Wenlong794663e2022-09-02 10:47:01 +08008053 trace_kvm_msr_read(msr_index, *pdata);
Hou Wenlong36d546d2022-09-02 10:47:00 +08008054 return X86EMUL_CONTINUE;
Avi Kivity717746e2011-04-20 13:37:53 +03008055}
8056
Hou Wenlongac8d6ca2022-03-07 20:26:33 +08008057static int emulator_set_msr_with_filter(struct x86_emulate_ctxt *ctxt,
8058 u32 msr_index, u64 data)
Avi Kivity717746e2011-04-20 13:37:53 +03008059{
Alexander Graf1ae09952020-09-25 16:34:16 +02008060 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
8061 int r;
8062
Hou Wenlongac8d6ca2022-03-07 20:26:33 +08008063 r = kvm_set_msr_with_filter(vcpu, msr_index, data);
Hou Wenlong36d546d2022-09-02 10:47:00 +08008064 if (r < 0)
8065 return X86EMUL_UNHANDLEABLE;
Alexander Graf1ae09952020-09-25 16:34:16 +02008066
Hou Wenlong36d546d2022-09-02 10:47:00 +08008067 if (r) {
8068 if (kvm_msr_user_space(vcpu, msr_index, KVM_EXIT_X86_WRMSR, data,
8069 complete_emulated_msr_access, r))
8070 return X86EMUL_IO_NEEDED;
Hou Wenlong794663e2022-09-02 10:47:01 +08008071
8072 trace_kvm_msr_write_ex(msr_index, data);
Hou Wenlong36d546d2022-09-02 10:47:00 +08008073 return X86EMUL_PROPAGATE_FAULT;
Alexander Graf1ae09952020-09-25 16:34:16 +02008074 }
8075
Hou Wenlong794663e2022-09-02 10:47:01 +08008076 trace_kvm_msr_write(msr_index, data);
Hou Wenlong36d546d2022-09-02 10:47:00 +08008077 return X86EMUL_CONTINUE;
Avi Kivity717746e2011-04-20 13:37:53 +03008078}
8079
Hou Wenlongac8d6ca2022-03-07 20:26:33 +08008080static int emulator_get_msr(struct x86_emulate_ctxt *ctxt,
8081 u32 msr_index, u64 *pdata)
8082{
8083 return kvm_get_msr(emul_to_vcpu(ctxt), msr_index, pdata);
8084}
8085
8086static int emulator_set_msr(struct x86_emulate_ctxt *ctxt,
8087 u32 msr_index, u64 data)
8088{
8089 return kvm_set_msr(emul_to_vcpu(ctxt), msr_index, data);
8090}
8091
Paolo Bonzini64d60672015-05-07 11:36:11 +02008092static u64 emulator_get_smbase(struct x86_emulate_ctxt *ctxt)
8093{
8094 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
8095
8096 return vcpu->arch.smbase;
8097}
8098
8099static void emulator_set_smbase(struct x86_emulate_ctxt *ctxt, u64 smbase)
8100{
8101 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
8102
8103 vcpu->arch.smbase = smbase;
8104}
8105
Nadav Amit67f4d422014-06-02 18:34:09 +03008106static int emulator_check_pmc(struct x86_emulate_ctxt *ctxt,
8107 u32 pmc)
8108{
Jim Mattsone6cd31f2021-11-05 13:20:58 -07008109 if (kvm_pmu_is_valid_rdpmc_ecx(emul_to_vcpu(ctxt), pmc))
8110 return 0;
8111 return -EINVAL;
Nadav Amit67f4d422014-06-02 18:34:09 +03008112}
8113
Avi Kivity222d21a2011-11-10 14:57:30 +02008114static int emulator_read_pmc(struct x86_emulate_ctxt *ctxt,
8115 u32 pmc, u64 *pdata)
8116{
Wei Huangc6702c92015-06-19 13:44:45 +02008117 return kvm_pmu_rdpmc(emul_to_vcpu(ctxt), pmc, pdata);
Avi Kivity222d21a2011-11-10 14:57:30 +02008118}
8119
Avi Kivity6c3287f2011-04-20 15:43:05 +03008120static void emulator_halt(struct x86_emulate_ctxt *ctxt)
8121{
8122 emul_to_vcpu(ctxt)->arch.halt_request = 1;
8123}
8124
Avi Kivity29535382011-04-20 13:37:53 +03008125static int emulator_intercept(struct x86_emulate_ctxt *ctxt,
Joerg Roedel8a76d7f2011-04-04 12:39:27 +02008126 struct x86_instruction_info *info,
Avi Kivityc4f035c2011-04-04 12:39:22 +02008127 enum x86_intercept_stage stage)
8128{
Jason Baronb36464772021-01-14 22:27:56 -05008129 return static_call(kvm_x86_check_intercept)(emul_to_vcpu(ctxt), info, stage,
Sean Christopherson21f1b8f2020-02-18 15:29:42 -08008130 &ctxt->exception);
Avi Kivityc4f035c2011-04-04 12:39:22 +02008131}
8132
Yu Zhange911eb32017-08-24 20:27:52 +08008133static bool emulator_get_cpuid(struct x86_emulate_ctxt *ctxt,
Sean Christophersonf91af512020-03-04 17:34:37 -08008134 u32 *eax, u32 *ebx, u32 *ecx, u32 *edx,
8135 bool exact_only)
Stephan Bärwolfbdb42f52012-01-12 16:43:03 +01008136{
Sean Christophersonf91af512020-03-04 17:34:37 -08008137 return kvm_cpuid(emul_to_vcpu(ctxt), eax, ebx, ecx, edx, exact_only);
Stephan Bärwolfbdb42f52012-01-12 16:43:03 +01008138}
8139
Sean Christopherson5ae78e92019-12-17 13:32:38 -08008140static bool emulator_guest_has_long_mode(struct x86_emulate_ctxt *ctxt)
8141{
8142 return guest_cpuid_has(emul_to_vcpu(ctxt), X86_FEATURE_LM);
8143}
8144
8145static bool emulator_guest_has_movbe(struct x86_emulate_ctxt *ctxt)
8146{
8147 return guest_cpuid_has(emul_to_vcpu(ctxt), X86_FEATURE_MOVBE);
8148}
8149
8150static bool emulator_guest_has_fxsr(struct x86_emulate_ctxt *ctxt)
8151{
8152 return guest_cpuid_has(emul_to_vcpu(ctxt), X86_FEATURE_FXSR);
8153}
8154
Hou Wenlonga8368392022-03-02 21:15:14 +08008155static bool emulator_guest_has_rdpid(struct x86_emulate_ctxt *ctxt)
8156{
8157 return guest_cpuid_has(emul_to_vcpu(ctxt), X86_FEATURE_RDPID);
8158}
8159
Avi Kivitydd856ef2012-08-27 23:46:17 +03008160static ulong emulator_read_gpr(struct x86_emulate_ctxt *ctxt, unsigned reg)
8161{
Sean Christopherson27b4a9c42021-04-21 19:21:28 -07008162 return kvm_register_read_raw(emul_to_vcpu(ctxt), reg);
Avi Kivitydd856ef2012-08-27 23:46:17 +03008163}
8164
8165static void emulator_write_gpr(struct x86_emulate_ctxt *ctxt, unsigned reg, ulong val)
8166{
Sean Christopherson27b4a9c42021-04-21 19:21:28 -07008167 kvm_register_write_raw(emul_to_vcpu(ctxt), reg, val);
Avi Kivitydd856ef2012-08-27 23:46:17 +03008168}
8169
Nadav Amit801806d2015-01-26 09:32:23 +02008170static void emulator_set_nmi_mask(struct x86_emulate_ctxt *ctxt, bool masked)
8171{
Jason Baronb36464772021-01-14 22:27:56 -05008172 static_call(kvm_x86_set_nmi_mask)(emul_to_vcpu(ctxt), masked);
Nadav Amit801806d2015-01-26 09:32:23 +02008173}
8174
Ladi Prosek6ed071f2017-04-25 16:42:44 +02008175static unsigned emulator_get_hflags(struct x86_emulate_ctxt *ctxt)
8176{
8177 return emul_to_vcpu(ctxt)->arch.hflags;
8178}
8179
Sean Christophersonedce4652021-06-09 11:56:13 -07008180static void emulator_exiting_smm(struct x86_emulate_ctxt *ctxt)
Ladi Prosek6ed071f2017-04-25 16:42:44 +02008181{
Sean Christopherson78fcb2c2021-06-09 11:56:11 -07008182 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
8183
Sean Christophersondc872752021-06-09 11:56:15 -07008184 kvm_smm_changed(vcpu, false);
Ladi Prosek6ed071f2017-04-25 16:42:44 +02008185}
8186
Sean Christophersonecc513e52021-06-09 11:56:19 -07008187static int emulator_leave_smm(struct x86_emulate_ctxt *ctxt,
Sean Christophersoned193212019-04-02 08:03:09 -07008188 const char *smstate)
Ladi Prosek0234bf82017-10-11 16:54:40 +02008189{
Sean Christophersonecc513e52021-06-09 11:56:19 -07008190 return static_call(kvm_x86_leave_smm)(emul_to_vcpu(ctxt), smstate);
Ladi Prosek0234bf82017-10-11 16:54:40 +02008191}
8192
Sean Christopherson25b17222021-06-09 11:56:12 -07008193static void emulator_triple_fault(struct x86_emulate_ctxt *ctxt)
8194{
8195 kvm_make_request(KVM_REQ_TRIPLE_FAULT, emul_to_vcpu(ctxt));
8196}
8197
Vitaly Kuznetsov02d41602019-08-13 15:53:32 +02008198static int emulator_set_xcr(struct x86_emulate_ctxt *ctxt, u32 index, u64 xcr)
8199{
8200 return __kvm_set_xcr(emul_to_vcpu(ctxt), index, xcr);
8201}
8202
Sean Christopherson1cca2f82022-05-26 21:08:15 +00008203static void emulator_vm_bugged(struct x86_emulate_ctxt *ctxt)
8204{
8205 struct kvm *kvm = emul_to_vcpu(ctxt)->kvm;
8206
8207 if (!kvm->vm_bugged)
8208 kvm_vm_bugged(kvm);
8209}
8210
Mathias Krause0225fb52012-08-30 01:30:16 +02008211static const struct x86_emulate_ops emulate_ops = {
Sean Christopherson1cca2f82022-05-26 21:08:15 +00008212 .vm_bugged = emulator_vm_bugged,
Avi Kivitydd856ef2012-08-27 23:46:17 +03008213 .read_gpr = emulator_read_gpr,
8214 .write_gpr = emulator_write_gpr,
Paolo Bonzinice14e868a2018-06-06 17:37:49 +02008215 .read_std = emulator_read_std,
8216 .write_std = emulator_write_std,
Radim Krčmář7a036a62015-10-30 16:36:24 +01008217 .read_phys = kvm_read_guest_phys_system,
Gleb Natapov1871c602010-02-10 14:21:32 +02008218 .fetch = kvm_fetch_guest_virt,
Carsten Ottebbd9b642007-10-30 18:44:21 +01008219 .read_emulated = emulator_read_emulated,
8220 .write_emulated = emulator_write_emulated,
8221 .cmpxchg_emulated = emulator_cmpxchg_emulated,
Avi Kivity3cb16fe2011-04-20 15:38:44 +03008222 .invlpg = emulator_invlpg,
Gleb Natapovcf8f70b2010-03-18 15:20:23 +02008223 .pio_in_emulated = emulator_pio_in_emulated,
8224 .pio_out_emulated = emulator_pio_out_emulated,
Avi Kivity1aa36612011-04-27 13:20:30 +03008225 .get_segment = emulator_get_segment,
8226 .set_segment = emulator_set_segment,
Gleb Natapov5951c442010-04-28 19:15:29 +03008227 .get_cached_segment_base = emulator_get_cached_segment_base,
Gleb Natapov2dafc6c2010-03-18 15:20:16 +02008228 .get_gdt = emulator_get_gdt,
Mohammed Gamal160ce1f2010-08-04 05:44:24 +03008229 .get_idt = emulator_get_idt,
Avi Kivity1ac9d0c2011-04-20 15:12:00 +03008230 .set_gdt = emulator_set_gdt,
8231 .set_idt = emulator_set_idt,
Gleb Natapov52a46612010-03-18 15:20:03 +02008232 .get_cr = emulator_get_cr,
8233 .set_cr = emulator_set_cr,
Gleb Natapov9c537242010-03-18 15:20:05 +02008234 .cpl = emulator_get_cpl,
Gleb Natapov35aa5372010-04-28 19:15:27 +03008235 .get_dr = emulator_get_dr,
8236 .set_dr = emulator_set_dr,
Paolo Bonzini64d60672015-05-07 11:36:11 +02008237 .get_smbase = emulator_get_smbase,
8238 .set_smbase = emulator_set_smbase,
Hou Wenlongac8d6ca2022-03-07 20:26:33 +08008239 .set_msr_with_filter = emulator_set_msr_with_filter,
8240 .get_msr_with_filter = emulator_get_msr_with_filter,
Avi Kivity717746e2011-04-20 13:37:53 +03008241 .set_msr = emulator_set_msr,
8242 .get_msr = emulator_get_msr,
Nadav Amit67f4d422014-06-02 18:34:09 +03008243 .check_pmc = emulator_check_pmc,
Avi Kivity222d21a2011-11-10 14:57:30 +02008244 .read_pmc = emulator_read_pmc,
Avi Kivity6c3287f2011-04-20 15:43:05 +03008245 .halt = emulator_halt,
Avi Kivitybcaf5cc2011-04-20 15:53:23 +03008246 .wbinvd = emulator_wbinvd,
Avi Kivityd6aa1002011-04-20 15:47:13 +03008247 .fix_hypercall = emulator_fix_hypercall,
Avi Kivityc4f035c2011-04-04 12:39:22 +02008248 .intercept = emulator_intercept,
Stephan Bärwolfbdb42f52012-01-12 16:43:03 +01008249 .get_cpuid = emulator_get_cpuid,
Sean Christopherson5ae78e92019-12-17 13:32:38 -08008250 .guest_has_long_mode = emulator_guest_has_long_mode,
8251 .guest_has_movbe = emulator_guest_has_movbe,
8252 .guest_has_fxsr = emulator_guest_has_fxsr,
Hou Wenlonga8368392022-03-02 21:15:14 +08008253 .guest_has_rdpid = emulator_guest_has_rdpid,
Nadav Amit801806d2015-01-26 09:32:23 +02008254 .set_nmi_mask = emulator_set_nmi_mask,
Ladi Prosek6ed071f2017-04-25 16:42:44 +02008255 .get_hflags = emulator_get_hflags,
Sean Christophersonedce4652021-06-09 11:56:13 -07008256 .exiting_smm = emulator_exiting_smm,
Sean Christophersonecc513e52021-06-09 11:56:19 -07008257 .leave_smm = emulator_leave_smm,
Sean Christopherson25b17222021-06-09 11:56:12 -07008258 .triple_fault = emulator_triple_fault,
Vitaly Kuznetsov02d41602019-08-13 15:53:32 +02008259 .set_xcr = emulator_set_xcr,
Carsten Ottebbd9b642007-10-30 18:44:21 +01008260};
8261
Gleb Natapov95cb2292010-04-28 19:15:43 +03008262static void toggle_interruptibility(struct kvm_vcpu *vcpu, u32 mask)
8263{
Jason Baronb36464772021-01-14 22:27:56 -05008264 u32 int_shadow = static_call(kvm_x86_get_interrupt_shadow)(vcpu);
Gleb Natapov95cb2292010-04-28 19:15:43 +03008265 /*
8266 * an sti; sti; sequence only disable interrupts for the first
8267 * instruction. So, if the last instruction, be it emulated or
8268 * not, left the system with the INT_STI flag enabled, it
8269 * means that the last instruction is an sti. We should not
8270 * leave the flag on in this case. The same goes for mov ss
8271 */
Paolo Bonzini37ccdcb2014-05-20 14:29:47 +02008272 if (int_shadow & mask)
8273 mask = 0;
Paolo Bonzini6addfc42014-03-27 11:29:28 +01008274 if (unlikely(int_shadow || mask)) {
Jason Baronb36464772021-01-14 22:27:56 -05008275 static_call(kvm_x86_set_interrupt_shadow)(vcpu, mask);
Paolo Bonzini6addfc42014-03-27 11:29:28 +01008276 if (!mask)
8277 kvm_make_request(KVM_REQ_EVENT, vcpu);
8278 }
Gleb Natapov95cb2292010-04-28 19:15:43 +03008279}
8280
Sean Christopherson7709aba2022-08-30 23:16:08 +00008281static void inject_emulated_exception(struct kvm_vcpu *vcpu)
Gleb Natapov54b84862010-04-28 19:15:44 +03008282{
Sean Christophersonc9b8b072020-02-18 15:29:48 -08008283 struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;
Paolo Bonzinief54bcf2014-09-04 19:46:15 +02008284
Sean Christopherson7709aba2022-08-30 23:16:08 +00008285 if (ctxt->exception.vector == PF_VECTOR)
8286 kvm_inject_emulated_page_fault(vcpu, &ctxt->exception);
8287 else if (ctxt->exception.error_code_valid)
Avi Kivityda9cb572010-11-22 17:53:21 +02008288 kvm_queue_exception_e(vcpu, ctxt->exception.vector,
8289 ctxt->exception.error_code);
Gleb Natapov54b84862010-04-28 19:15:44 +03008290 else
Avi Kivityda9cb572010-11-22 17:53:21 +02008291 kvm_queue_exception(vcpu, ctxt->exception.vector);
Gleb Natapov54b84862010-04-28 19:15:44 +03008292}
8293
Sean Christophersonc9b8b072020-02-18 15:29:48 -08008294static struct x86_emulate_ctxt *alloc_emulate_ctxt(struct kvm_vcpu *vcpu)
8295{
8296 struct x86_emulate_ctxt *ctxt;
8297
8298 ctxt = kmem_cache_zalloc(x86_emulator_cache, GFP_KERNEL_ACCOUNT);
8299 if (!ctxt) {
8300 pr_err("kvm: failed to allocate vcpu's emulator\n");
8301 return NULL;
8302 }
8303
8304 ctxt->vcpu = vcpu;
8305 ctxt->ops = &emulate_ops;
8306 vcpu->arch.emulate_ctxt = ctxt;
8307
8308 return ctxt;
8309}
8310
Mohammed Gamal8ec47222010-08-16 00:47:01 +03008311static void init_emulate_ctxt(struct kvm_vcpu *vcpu)
8312{
Sean Christophersonc9b8b072020-02-18 15:29:48 -08008313 struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;
Mohammed Gamal8ec47222010-08-16 00:47:01 +03008314 int cs_db, cs_l;
8315
Jason Baronb36464772021-01-14 22:27:56 -05008316 static_call(kvm_x86_get_cs_db_l_bits)(vcpu, &cs_db, &cs_l);
Mohammed Gamal8ec47222010-08-16 00:47:01 +03008317
Sean Christopherson744e6992020-02-18 15:03:09 -08008318 ctxt->gpa_available = false;
Takuya Yoshikawaadf52232011-05-25 11:06:16 +09008319 ctxt->eflags = kvm_get_rflags(vcpu);
Paolo Bonzinic8401dd2017-06-07 15:13:14 +02008320 ctxt->tf = (ctxt->eflags & X86_EFLAGS_TF) != 0;
8321
Takuya Yoshikawaadf52232011-05-25 11:06:16 +09008322 ctxt->eip = kvm_rip_read(vcpu);
8323 ctxt->mode = (!is_protmode(vcpu)) ? X86EMUL_MODE_REAL :
8324 (ctxt->eflags & X86_EFLAGS_VM) ? X86EMUL_MODE_VM86 :
Nadav Amit42bf5492014-04-18 07:11:34 +03008325 (cs_l && is_long_mode(vcpu)) ? X86EMUL_MODE_PROT64 :
Takuya Yoshikawaadf52232011-05-25 11:06:16 +09008326 cs_db ? X86EMUL_MODE_PROT32 :
8327 X86EMUL_MODE_PROT16;
Paolo Bonzinia5845392015-04-01 18:18:53 +02008328 BUILD_BUG_ON(HF_GUEST_MASK != X86EMUL_GUEST_MASK);
Paolo Bonzini64d60672015-05-07 11:36:11 +02008329 BUILD_BUG_ON(HF_SMM_MASK != X86EMUL_SMM_MASK);
8330 BUILD_BUG_ON(HF_SMM_INSIDE_NMI_MASK != X86EMUL_SMM_INSIDE_NMI_MASK);
Takuya Yoshikawaadf52232011-05-25 11:06:16 +09008331
Wanpeng Lida6393c2021-05-27 17:01:36 -07008332 ctxt->interruptibility = 0;
8333 ctxt->have_exception = false;
8334 ctxt->exception.vector = -1;
8335 ctxt->perm_ok = false;
8336
Avi Kivitydd856ef2012-08-27 23:46:17 +03008337 init_decode_cache(ctxt);
Gleb Natapov7ae441e2011-03-31 12:06:41 +02008338 vcpu->arch.emulate_regs_need_sync_from_vcpu = false;
Mohammed Gamal8ec47222010-08-16 00:47:01 +03008339}
8340
Sean Christopherson9497e1f2019-08-27 14:40:36 -07008341void kvm_inject_realmode_interrupt(struct kvm_vcpu *vcpu, int irq, int inc_eip)
Mohammed Gamal63995652010-09-19 14:34:06 +02008342{
Sean Christophersonc9b8b072020-02-18 15:29:48 -08008343 struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;
Mohammed Gamal63995652010-09-19 14:34:06 +02008344 int ret;
8345
8346 init_emulate_ctxt(vcpu);
8347
Avi Kivity9dac77f2011-06-01 15:34:25 +03008348 ctxt->op_bytes = 2;
8349 ctxt->ad_bytes = 2;
8350 ctxt->_eip = ctxt->eip + inc_eip;
Takuya Yoshikawa9d741912011-05-29 21:53:48 +09008351 ret = emulate_int_real(ctxt, irq);
Mohammed Gamal63995652010-09-19 14:34:06 +02008352
Sean Christopherson9497e1f2019-08-27 14:40:36 -07008353 if (ret != X86EMUL_CONTINUE) {
8354 kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
8355 } else {
8356 ctxt->eip = ctxt->_eip;
8357 kvm_rip_write(vcpu, ctxt->eip);
8358 kvm_set_rflags(vcpu, ctxt->eflags);
8359 }
Mohammed Gamal63995652010-09-19 14:34:06 +02008360}
8361EXPORT_SYMBOL_GPL(kvm_inject_realmode_interrupt);
8362
David Edmondsone615e352021-09-20 11:37:36 +01008363static void prepare_emulation_failure_exit(struct kvm_vcpu *vcpu, u64 *data,
8364 u8 ndata, u8 *insn_bytes, u8 insn_size)
Aaron Lewis19238e72021-05-10 07:48:33 -07008365{
Aaron Lewis19238e72021-05-10 07:48:33 -07008366 struct kvm_run *run = vcpu->run;
David Edmondsone615e352021-09-20 11:37:36 +01008367 u64 info[5];
8368 u8 info_start;
8369
8370 /*
8371 * Zero the whole array used to retrieve the exit info, as casting to
8372 * u32 for select entries will leave some chunks uninitialized.
8373 */
8374 memset(&info, 0, sizeof(info));
8375
8376 static_call(kvm_x86_get_exit_info)(vcpu, (u32 *)&info[0], &info[1],
8377 &info[2], (u32 *)&info[3],
8378 (u32 *)&info[4]);
Aaron Lewis19238e72021-05-10 07:48:33 -07008379
8380 run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
8381 run->emulation_failure.suberror = KVM_INTERNAL_ERROR_EMULATION;
David Edmondsone615e352021-09-20 11:37:36 +01008382
8383 /*
8384 * There's currently space for 13 entries, but 5 are used for the exit
8385 * reason and info. Restrict to 4 to reduce the maintenance burden
8386 * when expanding kvm_run.emulation_failure in the future.
8387 */
8388 if (WARN_ON_ONCE(ndata > 4))
8389 ndata = 4;
8390
8391 /* Always include the flags as a 'data' entry. */
8392 info_start = 1;
Aaron Lewis19238e72021-05-10 07:48:33 -07008393 run->emulation_failure.flags = 0;
8394
8395 if (insn_size) {
David Edmondsone615e352021-09-20 11:37:36 +01008396 BUILD_BUG_ON((sizeof(run->emulation_failure.insn_size) +
8397 sizeof(run->emulation_failure.insn_bytes) != 16));
8398 info_start += 2;
Aaron Lewis19238e72021-05-10 07:48:33 -07008399 run->emulation_failure.flags |=
8400 KVM_INTERNAL_ERROR_EMULATION_FLAG_INSTRUCTION_BYTES;
8401 run->emulation_failure.insn_size = insn_size;
8402 memset(run->emulation_failure.insn_bytes, 0x90,
8403 sizeof(run->emulation_failure.insn_bytes));
David Edmondsone615e352021-09-20 11:37:36 +01008404 memcpy(run->emulation_failure.insn_bytes, insn_bytes, insn_size);
Aaron Lewis19238e72021-05-10 07:48:33 -07008405 }
David Edmondsone615e352021-09-20 11:37:36 +01008406
8407 memcpy(&run->internal.data[info_start], info, sizeof(info));
8408 memcpy(&run->internal.data[info_start + ARRAY_SIZE(info)], data,
8409 ndata * sizeof(data[0]));
8410
8411 run->emulation_failure.ndata = info_start + ARRAY_SIZE(info) + ndata;
Aaron Lewis19238e72021-05-10 07:48:33 -07008412}
8413
David Edmondsone615e352021-09-20 11:37:36 +01008414static void prepare_emulation_ctxt_failure_exit(struct kvm_vcpu *vcpu)
8415{
8416 struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;
8417
8418 prepare_emulation_failure_exit(vcpu, NULL, 0, ctxt->fetch.data,
8419 ctxt->fetch.end - ctxt->fetch.data);
8420}
8421
8422void __kvm_prepare_emulation_failure_exit(struct kvm_vcpu *vcpu, u64 *data,
8423 u8 ndata)
8424{
8425 prepare_emulation_failure_exit(vcpu, data, ndata, NULL, 0);
8426}
8427EXPORT_SYMBOL_GPL(__kvm_prepare_emulation_failure_exit);
8428
8429void kvm_prepare_emulation_failure_exit(struct kvm_vcpu *vcpu)
8430{
8431 __kvm_prepare_emulation_failure_exit(vcpu, NULL, 0);
8432}
8433EXPORT_SYMBOL_GPL(kvm_prepare_emulation_failure_exit);
8434
Liran Alone2366172018-03-12 13:12:49 +02008435static int handle_emulation_failure(struct kvm_vcpu *vcpu, int emulation_type)
Gleb Natapov6d77dbf2010-05-10 11:16:56 +03008436{
Aaron Lewis19238e72021-05-10 07:48:33 -07008437 struct kvm *kvm = vcpu->kvm;
8438
Gleb Natapov6d77dbf2010-05-10 11:16:56 +03008439 ++vcpu->stat.insn_emulation_fail;
8440 trace_kvm_emulate_insn_failed(vcpu);
Liran Alone2366172018-03-12 13:12:49 +02008441
Sean Christopherson42cbf062019-08-27 14:40:31 -07008442 if (emulation_type & EMULTYPE_VMWARE_GP) {
8443 kvm_queue_exception_e(vcpu, GP_VECTOR, 0);
Sean Christopherson60fc3d02019-08-27 14:40:38 -07008444 return 1;
Sean Christopherson42cbf062019-08-27 14:40:31 -07008445 }
Liran Alone2366172018-03-12 13:12:49 +02008446
Aaron Lewis19238e72021-05-10 07:48:33 -07008447 if (kvm->arch.exit_on_emulation_error ||
8448 (emulation_type & EMULTYPE_SKIP)) {
David Edmondsone615e352021-09-20 11:37:36 +01008449 prepare_emulation_ctxt_failure_exit(vcpu);
Sean Christopherson60fc3d02019-08-27 14:40:38 -07008450 return 0;
Sean Christopherson738fece2019-08-27 14:40:34 -07008451 }
8452
Sean Christopherson22da61c2019-08-27 14:40:28 -07008453 kvm_queue_exception(vcpu, UD_VECTOR);
8454
Jason Baronb36464772021-01-14 22:27:56 -05008455 if (!is_guest_mode(vcpu) && static_call(kvm_x86_get_cpl)(vcpu) == 0) {
David Edmondsone615e352021-09-20 11:37:36 +01008456 prepare_emulation_ctxt_failure_exit(vcpu);
Sean Christopherson60fc3d02019-08-27 14:40:38 -07008457 return 0;
Joerg Roedelfc3a9152010-11-29 17:51:49 +01008458 }
Liran Alone2366172018-03-12 13:12:49 +02008459
Sean Christopherson60fc3d02019-08-27 14:40:38 -07008460 return 1;
Gleb Natapov6d77dbf2010-05-10 11:16:56 +03008461}
8462
Sean Christopherson736c2912019-12-06 15:57:14 -08008463static bool reexecute_instruction(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa,
Gleb Natapov991eebf2013-04-11 12:10:51 +03008464 bool write_fault_to_shadow_pgtable,
8465 int emulation_type)
Gleb Natapova6f177e2010-07-08 12:41:12 +03008466{
Sean Christopherson736c2912019-12-06 15:57:14 -08008467 gpa_t gpa = cr2_or_gpa;
Dan Williamsba049e92016-01-15 16:56:11 -08008468 kvm_pfn_t pfn;
Gleb Natapova6f177e2010-07-08 12:41:12 +03008469
Sean Christopherson92daa482020-02-18 15:03:08 -08008470 if (!(emulation_type & EMULTYPE_ALLOW_RETRY_PF))
Gleb Natapov991eebf2013-04-11 12:10:51 +03008471 return false;
8472
Sean Christopherson92daa482020-02-18 15:03:08 -08008473 if (WARN_ON_ONCE(is_guest_mode(vcpu)) ||
8474 WARN_ON_ONCE(!(emulation_type & EMULTYPE_PF)))
Sean Christopherson6c3dfeb2018-08-23 13:56:51 -07008475 return false;
8476
Paolo Bonzini347a0d02022-02-10 08:00:56 -05008477 if (!vcpu->arch.mmu->root_role.direct) {
Xiao Guangrong95b3cf62013-01-13 23:46:52 +08008478 /*
8479 * Write permission should be allowed since only
8480 * write access need to be emulated.
8481 */
Sean Christopherson736c2912019-12-06 15:57:14 -08008482 gpa = kvm_mmu_gva_to_gpa_write(vcpu, cr2_or_gpa, NULL);
Gleb Natapov68be0802010-07-14 19:05:45 +03008483
Xiao Guangrong95b3cf62013-01-13 23:46:52 +08008484 /*
8485 * If the mapping is invalid in guest, let cpu retry
8486 * it to generate fault.
8487 */
Hou Wenlong6e1d2a32022-07-01 17:24:13 +08008488 if (gpa == INVALID_GPA)
Xiao Guangrong95b3cf62013-01-13 23:46:52 +08008489 return true;
8490 }
Gleb Natapova6f177e2010-07-08 12:41:12 +03008491
Xiao Guangrong8e3d9d02012-08-21 10:57:42 +08008492 /*
8493 * Do not retry the unhandleable instruction if it faults on the
8494 * readonly host memory, otherwise it will goto a infinite loop:
8495 * retry instruction -> write #PF -> emulation fail -> retry
8496 * instruction -> ...
8497 */
8498 pfn = gfn_to_pfn(vcpu->kvm, gpa_to_gfn(gpa));
Xiao Guangrong95b3cf62013-01-13 23:46:52 +08008499
8500 /*
8501 * If the instruction failed on the error pfn, it can not be fixed,
8502 * report the error to userspace.
8503 */
8504 if (is_error_noslot_pfn(pfn))
8505 return false;
8506
8507 kvm_release_pfn_clean(pfn);
8508
8509 /* The instructions are well-emulated on direct mmu. */
Paolo Bonzini347a0d02022-02-10 08:00:56 -05008510 if (vcpu->arch.mmu->root_role.direct) {
Xiao Guangrong95b3cf62013-01-13 23:46:52 +08008511 unsigned int indirect_shadow_pages;
8512
Ben Gardon531810c2021-02-02 10:57:24 -08008513 write_lock(&vcpu->kvm->mmu_lock);
Xiao Guangrong95b3cf62013-01-13 23:46:52 +08008514 indirect_shadow_pages = vcpu->kvm->arch.indirect_shadow_pages;
Ben Gardon531810c2021-02-02 10:57:24 -08008515 write_unlock(&vcpu->kvm->mmu_lock);
Xiao Guangrong95b3cf62013-01-13 23:46:52 +08008516
8517 if (indirect_shadow_pages)
8518 kvm_mmu_unprotect_page(vcpu->kvm, gpa_to_gfn(gpa));
8519
Gleb Natapova6f177e2010-07-08 12:41:12 +03008520 return true;
Xiao Guangrong8e3d9d02012-08-21 10:57:42 +08008521 }
Gleb Natapova6f177e2010-07-08 12:41:12 +03008522
Xiao Guangrong95b3cf62013-01-13 23:46:52 +08008523 /*
8524 * if emulation was due to access to shadowed page table
8525 * and it failed try to unshadow page and re-enter the
8526 * guest to let CPU execute the instruction.
8527 */
8528 kvm_mmu_unprotect_page(vcpu->kvm, gpa_to_gfn(gpa));
Xiao Guangrong93c05d32013-01-13 23:49:07 +08008529
8530 /*
8531 * If the access faults on its page table, it can not
8532 * be fixed by unprotecting shadow page and it should
8533 * be reported to userspace.
8534 */
8535 return !write_fault_to_shadow_pgtable;
Gleb Natapova6f177e2010-07-08 12:41:12 +03008536}
8537
Xiao Guangrong1cb3f3a2011-09-22 17:02:48 +08008538static bool retry_instruction(struct x86_emulate_ctxt *ctxt,
Sean Christopherson736c2912019-12-06 15:57:14 -08008539 gpa_t cr2_or_gpa, int emulation_type)
Xiao Guangrong1cb3f3a2011-09-22 17:02:48 +08008540{
8541 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
Sean Christopherson736c2912019-12-06 15:57:14 -08008542 unsigned long last_retry_eip, last_retry_addr, gpa = cr2_or_gpa;
Xiao Guangrong1cb3f3a2011-09-22 17:02:48 +08008543
8544 last_retry_eip = vcpu->arch.last_retry_eip;
8545 last_retry_addr = vcpu->arch.last_retry_addr;
8546
8547 /*
8548 * If the emulation is caused by #PF and it is non-page_table
8549 * writing instruction, it means the VM-EXIT is caused by shadow
8550 * page protected, we can zap the shadow page and retry this
8551 * instruction directly.
8552 *
8553 * Note: if the guest uses a non-page-table modifying instruction
8554 * on the PDE that points to the instruction, then we will unmap
8555 * the instruction and go to an infinite loop. So, we cache the
8556 * last retried eip and the last fault address, if we meet the eip
8557 * and the address again, we can break out of the potential infinite
8558 * loop.
8559 */
8560 vcpu->arch.last_retry_eip = vcpu->arch.last_retry_addr = 0;
8561
Sean Christopherson92daa482020-02-18 15:03:08 -08008562 if (!(emulation_type & EMULTYPE_ALLOW_RETRY_PF))
Xiao Guangrong1cb3f3a2011-09-22 17:02:48 +08008563 return false;
8564
Sean Christopherson92daa482020-02-18 15:03:08 -08008565 if (WARN_ON_ONCE(is_guest_mode(vcpu)) ||
8566 WARN_ON_ONCE(!(emulation_type & EMULTYPE_PF)))
Sean Christopherson6c3dfeb2018-08-23 13:56:51 -07008567 return false;
8568
Xiao Guangrong1cb3f3a2011-09-22 17:02:48 +08008569 if (x86_page_table_writing_insn(ctxt))
8570 return false;
8571
Sean Christopherson736c2912019-12-06 15:57:14 -08008572 if (ctxt->eip == last_retry_eip && last_retry_addr == cr2_or_gpa)
Xiao Guangrong1cb3f3a2011-09-22 17:02:48 +08008573 return false;
8574
8575 vcpu->arch.last_retry_eip = ctxt->eip;
Sean Christopherson736c2912019-12-06 15:57:14 -08008576 vcpu->arch.last_retry_addr = cr2_or_gpa;
Xiao Guangrong1cb3f3a2011-09-22 17:02:48 +08008577
Paolo Bonzini347a0d02022-02-10 08:00:56 -05008578 if (!vcpu->arch.mmu->root_role.direct)
Sean Christopherson736c2912019-12-06 15:57:14 -08008579 gpa = kvm_mmu_gva_to_gpa_write(vcpu, cr2_or_gpa, NULL);
Xiao Guangrong1cb3f3a2011-09-22 17:02:48 +08008580
Xiao Guangrong22368022013-01-13 23:44:12 +08008581 kvm_mmu_unprotect_page(vcpu->kvm, gpa_to_gfn(gpa));
Xiao Guangrong1cb3f3a2011-09-22 17:02:48 +08008582
8583 return true;
8584}
8585
Gleb Natapov716d51a2012-09-03 15:24:26 +03008586static int complete_emulated_mmio(struct kvm_vcpu *vcpu);
8587static int complete_emulated_pio(struct kvm_vcpu *vcpu);
8588
Sean Christophersondc872752021-06-09 11:56:15 -07008589static void kvm_smm_changed(struct kvm_vcpu *vcpu, bool entering_smm)
Paolo Bonzinia5845392015-04-01 18:18:53 +02008590{
Sean Christopherson1270e642021-06-09 11:56:17 -07008591 trace_kvm_smm_transition(vcpu->vcpu_id, vcpu->arch.smbase, entering_smm);
Sean Christopherson0d7ee6f2021-06-09 11:56:16 -07008592
Sean Christophersondc872752021-06-09 11:56:15 -07008593 if (entering_smm) {
8594 vcpu->arch.hflags |= HF_SMM_MASK;
8595 } else {
8596 vcpu->arch.hflags &= ~(HF_SMM_MASK | HF_SMM_INSIDE_NMI_MASK);
8597
Paolo Bonzinic43203c2016-06-01 22:26:00 +02008598 /* Process a latched INIT or SMI, if any. */
8599 kvm_make_request(KVM_REQ_EVENT, vcpu);
Maxim Levitsky37687c42021-09-13 17:09:50 +03008600
8601 /*
8602 * Even if KVM_SET_SREGS2 loaded PDPTRs out of band,
8603 * on SMM exit we still need to reload them from
8604 * guest memory
8605 */
8606 vcpu->arch.pdptrs_from_userspace = false;
Paolo Bonzini64d60672015-05-07 11:36:11 +02008607 }
Paolo Bonzini699023e2015-05-18 15:03:39 +02008608
8609 kvm_mmu_reset_context(vcpu);
Paolo Bonzini64d60672015-05-07 11:36:11 +02008610}
8611
Paolo Bonzini4a1e10d2013-05-30 11:48:30 +02008612static int kvm_vcpu_check_hw_bp(unsigned long addr, u32 type, u32 dr7,
8613 unsigned long *db)
8614{
8615 u32 dr6 = 0;
8616 int i;
8617 u32 enable, rwlen;
8618
8619 enable = dr7;
8620 rwlen = dr7 >> 16;
8621 for (i = 0; i < 4; i++, enable >>= 2, rwlen >>= 4)
8622 if ((enable & 3) && (rwlen & 15) == type && db[i] == addr)
8623 dr6 |= (1 << i);
8624 return dr6;
8625}
8626
Sean Christopherson120c2c42019-08-27 14:40:29 -07008627static int kvm_vcpu_do_singlestep(struct kvm_vcpu *vcpu)
Paolo Bonzini663f4c62013-06-25 18:32:07 +02008628{
8629 struct kvm_run *kvm_run = vcpu->run;
8630
Paolo Bonzinic8401dd2017-06-07 15:13:14 +02008631 if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP) {
Chenyi Qiang9a3ecd52021-02-02 17:04:31 +08008632 kvm_run->debug.arch.dr6 = DR6_BS | DR6_ACTIVE_LOW;
Peter Xud5d260c2020-05-05 16:49:59 -04008633 kvm_run->debug.arch.pc = kvm_get_linear_rip(vcpu);
Paolo Bonzinic8401dd2017-06-07 15:13:14 +02008634 kvm_run->debug.arch.exception = DB_VECTOR;
8635 kvm_run->exit_reason = KVM_EXIT_DEBUG;
Sean Christopherson60fc3d02019-08-27 14:40:38 -07008636 return 0;
Paolo Bonzini663f4c62013-06-25 18:32:07 +02008637 }
Sean Christopherson120c2c42019-08-27 14:40:29 -07008638 kvm_queue_exception_p(vcpu, DB_VECTOR, DR6_BS);
Sean Christopherson60fc3d02019-08-27 14:40:38 -07008639 return 1;
Paolo Bonzini663f4c62013-06-25 18:32:07 +02008640}
8641
Kyle Huey6affcbe2016-11-29 12:40:40 -08008642int kvm_skip_emulated_instruction(struct kvm_vcpu *vcpu)
8643{
Jason Baronb36464772021-01-14 22:27:56 -05008644 unsigned long rflags = static_call(kvm_x86_get_rflags)(vcpu);
Vitaly Kuznetsovf8ea7c62019-08-13 15:53:30 +02008645 int r;
Kyle Huey6affcbe2016-11-29 12:40:40 -08008646
Jason Baronb36464772021-01-14 22:27:56 -05008647 r = static_call(kvm_x86_skip_emulated_instruction)(vcpu);
Sean Christopherson60fc3d02019-08-27 14:40:38 -07008648 if (unlikely(!r))
Vitaly Kuznetsovf8ea7c62019-08-13 15:53:30 +02008649 return 0;
Paolo Bonzinic8401dd2017-06-07 15:13:14 +02008650
Eric Hankland9cd803d2021-11-30 15:42:20 +08008651 kvm_pmu_trigger_event(vcpu, PERF_COUNT_HW_INSTRUCTIONS);
8652
Paolo Bonzinic8401dd2017-06-07 15:13:14 +02008653 /*
8654 * rflags is the old, "raw" value of the flags. The new value has
8655 * not been saved yet.
8656 *
8657 * This is correct even for TF set by the guest, because "the
8658 * processor will not generate this exception after the instruction
8659 * that sets the TF flag".
8660 */
8661 if (unlikely(rflags & X86_EFLAGS_TF))
Sean Christopherson120c2c42019-08-27 14:40:29 -07008662 r = kvm_vcpu_do_singlestep(vcpu);
Sean Christopherson60fc3d02019-08-27 14:40:38 -07008663 return r;
Kyle Huey6affcbe2016-11-29 12:40:40 -08008664}
8665EXPORT_SYMBOL_GPL(kvm_skip_emulated_instruction);
8666
Sean Christophersonbaf67ca2022-08-30 23:15:52 +00008667static bool kvm_is_code_breakpoint_inhibited(struct kvm_vcpu *vcpu)
Paolo Bonzini4a1e10d2013-05-30 11:48:30 +02008668{
Sean Christophersonbaf67ca2022-08-30 23:15:52 +00008669 u32 shadow;
8670
8671 if (kvm_get_rflags(vcpu) & X86_EFLAGS_RF)
8672 return true;
8673
8674 /*
8675 * Intel CPUs inhibit code #DBs when MOV/POP SS blocking is active,
8676 * but AMD CPUs do not. MOV/POP SS blocking is rare, check that first
8677 * to avoid the relatively expensive CPUID lookup.
8678 */
8679 shadow = static_call(kvm_x86_get_interrupt_shadow)(vcpu);
8680 return (shadow & KVM_X86_SHADOW_INT_MOV_SS) &&
8681 guest_cpuid_is_intel(vcpu);
8682}
8683
Sean Christopherson750f8fc2022-08-30 23:15:50 +00008684static bool kvm_vcpu_check_code_breakpoint(struct kvm_vcpu *vcpu,
8685 int emulation_type, int *r)
Paolo Bonzini4a1e10d2013-05-30 11:48:30 +02008686{
Sean Christopherson750f8fc2022-08-30 23:15:50 +00008687 WARN_ON_ONCE(emulation_type & EMULTYPE_NO_DECODE);
8688
8689 /*
8690 * Do not check for code breakpoints if hardware has already done the
8691 * checks, as inferred from the emulation type. On NO_DECODE and SKIP,
8692 * the instruction has passed all exception checks, and all intercepted
8693 * exceptions that trigger emulation have lower priority than code
8694 * breakpoints, i.e. the fact that the intercepted exception occurred
8695 * means any code breakpoints have already been serviced.
8696 *
8697 * Note, KVM needs to check for code #DBs on EMULTYPE_TRAP_UD_FORCED as
8698 * hardware has checked the RIP of the magic prefix, but not the RIP of
8699 * the instruction being emulated. The intent of forced emulation is
8700 * to behave as if KVM intercepted the instruction without an exception
8701 * and without a prefix.
8702 */
8703 if (emulation_type & (EMULTYPE_NO_DECODE | EMULTYPE_SKIP |
8704 EMULTYPE_TRAP_UD | EMULTYPE_VMWARE_GP | EMULTYPE_PF))
8705 return false;
8706
Paolo Bonzini4a1e10d2013-05-30 11:48:30 +02008707 if (unlikely(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) &&
8708 (vcpu->arch.guest_debug_dr7 & DR7_BP_EN_MASK)) {
Nadav Amit82b32772014-11-02 11:54:45 +02008709 struct kvm_run *kvm_run = vcpu->run;
8710 unsigned long eip = kvm_get_linear_rip(vcpu);
8711 u32 dr6 = kvm_vcpu_check_hw_bp(eip, 0,
Paolo Bonzini4a1e10d2013-05-30 11:48:30 +02008712 vcpu->arch.guest_debug_dr7,
8713 vcpu->arch.eff_db);
8714
8715 if (dr6 != 0) {
Chenyi Qiang9a3ecd52021-02-02 17:04:31 +08008716 kvm_run->debug.arch.dr6 = dr6 | DR6_ACTIVE_LOW;
Nadav Amit82b32772014-11-02 11:54:45 +02008717 kvm_run->debug.arch.pc = eip;
Paolo Bonzini4a1e10d2013-05-30 11:48:30 +02008718 kvm_run->debug.arch.exception = DB_VECTOR;
8719 kvm_run->exit_reason = KVM_EXIT_DEBUG;
Sean Christopherson60fc3d02019-08-27 14:40:38 -07008720 *r = 0;
Paolo Bonzini4a1e10d2013-05-30 11:48:30 +02008721 return true;
8722 }
8723 }
8724
Nadav Amit4161a562014-07-17 01:19:31 +03008725 if (unlikely(vcpu->arch.dr7 & DR7_BP_EN_MASK) &&
Sean Christophersonbaf67ca2022-08-30 23:15:52 +00008726 !kvm_is_code_breakpoint_inhibited(vcpu)) {
Nadav Amit82b32772014-11-02 11:54:45 +02008727 unsigned long eip = kvm_get_linear_rip(vcpu);
8728 u32 dr6 = kvm_vcpu_check_hw_bp(eip, 0,
Paolo Bonzini4a1e10d2013-05-30 11:48:30 +02008729 vcpu->arch.dr7,
8730 vcpu->arch.db);
8731
8732 if (dr6 != 0) {
Paolo Bonzini4d5523c2020-05-05 07:33:20 -04008733 kvm_queue_exception_p(vcpu, DB_VECTOR, dr6);
Sean Christopherson60fc3d02019-08-27 14:40:38 -07008734 *r = 1;
Paolo Bonzini4a1e10d2013-05-30 11:48:30 +02008735 return true;
8736 }
8737 }
8738
8739 return false;
8740}
8741
Liran Alon04789b62018-03-12 13:12:50 +02008742static bool is_vmware_backdoor_opcode(struct x86_emulate_ctxt *ctxt)
8743{
Arbel Moshe2d7921c2018-03-12 13:12:53 +02008744 switch (ctxt->opcode_len) {
8745 case 1:
8746 switch (ctxt->b) {
8747 case 0xe4: /* IN */
8748 case 0xe5:
8749 case 0xec:
8750 case 0xed:
8751 case 0xe6: /* OUT */
8752 case 0xe7:
8753 case 0xee:
8754 case 0xef:
8755 case 0x6c: /* INS */
8756 case 0x6d:
8757 case 0x6e: /* OUTS */
8758 case 0x6f:
8759 return true;
8760 }
8761 break;
8762 case 2:
8763 switch (ctxt->b) {
8764 case 0x33: /* RDPMC */
8765 return true;
8766 }
8767 break;
Liran Alon04789b62018-03-12 13:12:50 +02008768 }
8769
8770 return false;
8771}
8772
Wei Huang4aa26912021-01-26 03:18:28 -05008773/*
Sean Christophersonfee060c2022-03-11 03:27:41 +00008774 * Decode an instruction for emulation. The caller is responsible for handling
8775 * code breakpoints. Note, manually detecting code breakpoints is unnecessary
8776 * (and wrong) when emulating on an intercepted fault-like exception[*], as
8777 * code breakpoints have higher priority and thus have already been done by
8778 * hardware.
8779 *
8780 * [*] Except #MC, which is higher priority, but KVM should never emulate in
8781 * response to a machine check.
Wei Huang4aa26912021-01-26 03:18:28 -05008782 */
8783int x86_decode_emulated_instruction(struct kvm_vcpu *vcpu, int emulation_type,
8784 void *insn, int insn_len)
8785{
Wei Huang4aa26912021-01-26 03:18:28 -05008786 struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;
Sean Christophersonfee060c2022-03-11 03:27:41 +00008787 int r;
Wei Huang4aa26912021-01-26 03:18:28 -05008788
8789 init_emulate_ctxt(vcpu);
8790
Wanpeng Lib35491e2021-05-27 17:01:37 -07008791 r = x86_decode_insn(ctxt, insn, insn_len, emulation_type);
Wei Huang4aa26912021-01-26 03:18:28 -05008792
8793 trace_kvm_emulate_insn_start(vcpu);
8794 ++vcpu->stat.insn_emulation;
8795
8796 return r;
8797}
8798EXPORT_SYMBOL_GPL(x86_decode_emulated_instruction);
8799
Sean Christopherson736c2912019-12-06 15:57:14 -08008800int x86_emulate_instruction(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa,
8801 int emulation_type, void *insn, int insn_len)
Carsten Ottebbd9b642007-10-30 18:44:21 +01008802{
Gleb Natapov95cb2292010-04-28 19:15:43 +03008803 int r;
Sean Christophersonc9b8b072020-02-18 15:29:48 -08008804 struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;
Gleb Natapov7ae441e2011-03-31 12:06:41 +02008805 bool writeback = true;
Sean Christopherson09e3e2a2020-09-15 16:27:02 -07008806 bool write_fault_to_spt;
8807
Sean Christopherson4d31d9e2022-01-20 01:07:15 +00008808 if (unlikely(!kvm_can_emulate_insn(vcpu, emulation_type, insn, insn_len)))
Sean Christopherson09e3e2a2020-09-15 16:27:02 -07008809 return 1;
Carsten Ottebbd9b642007-10-30 18:44:21 +01008810
Paolo Bonzinic595cee2018-07-02 13:07:14 +02008811 vcpu->arch.l1tf_flush_l1d = true;
8812
Xiao Guangrong93c05d32013-01-13 23:49:07 +08008813 /*
8814 * Clear write_fault_to_shadow_pgtable here to ensure it is
8815 * never reused.
8816 */
Sean Christopherson09e3e2a2020-09-15 16:27:02 -07008817 write_fault_to_spt = vcpu->arch.write_fault_to_shadow_pgtable;
Xiao Guangrong93c05d32013-01-13 23:49:07 +08008818 vcpu->arch.write_fault_to_shadow_pgtable = false;
Gleb Natapov8d7d81022011-04-12 12:36:21 +03008819
Sheng Yang571008d2008-01-02 14:49:22 +08008820 if (!(emulation_type & EMULTYPE_NO_DECODE)) {
Wei Huang4aa26912021-01-26 03:18:28 -05008821 kvm_clear_exception_queue(vcpu);
Paolo Bonzini4a1e10d2013-05-30 11:48:30 +02008822
Sean Christophersonfee060c2022-03-11 03:27:41 +00008823 /*
8824 * Return immediately if RIP hits a code breakpoint, such #DBs
8825 * are fault-like and are higher priority than any faults on
8826 * the code fetch itself.
8827 */
Sean Christopherson750f8fc2022-08-30 23:15:50 +00008828 if (kvm_vcpu_check_code_breakpoint(vcpu, emulation_type, &r))
Sean Christophersonfee060c2022-03-11 03:27:41 +00008829 return r;
8830
Wei Huang4aa26912021-01-26 03:18:28 -05008831 r = x86_decode_emulated_instruction(vcpu, emulation_type,
8832 insn, insn_len);
Takuya Yoshikawa1d2887e2011-07-30 18:03:34 +09008833 if (r != EMULATION_OK) {
Sean Christophersonb4000602019-08-27 14:40:32 -07008834 if ((emulation_type & EMULTYPE_TRAP_UD) ||
Sean Christophersonc83fad65e2019-08-27 14:40:33 -07008835 (emulation_type & EMULTYPE_TRAP_UD_FORCED)) {
8836 kvm_queue_exception(vcpu, UD_VECTOR);
Sean Christopherson60fc3d02019-08-27 14:40:38 -07008837 return 1;
Sean Christophersonc83fad65e2019-08-27 14:40:33 -07008838 }
Sean Christopherson736c2912019-12-06 15:57:14 -08008839 if (reexecute_instruction(vcpu, cr2_or_gpa,
8840 write_fault_to_spt,
8841 emulation_type))
Sean Christopherson60fc3d02019-08-27 14:40:38 -07008842 return 1;
Jan Dakinevich8530a792019-08-27 13:07:09 +00008843 if (ctxt->have_exception) {
Jan Dakinevichc8848ce2019-08-27 13:07:08 +00008844 /*
8845 * #UD should result in just EMULATION_FAILED, and trap-like
8846 * exception should not be encountered during decode.
8847 */
8848 WARN_ON_ONCE(ctxt->exception.vector == UD_VECTOR ||
8849 exception_type(ctxt->exception.vector) == EXCPT_TRAP);
Jan Dakinevich8530a792019-08-27 13:07:09 +00008850 inject_emulated_exception(vcpu);
Sean Christopherson60fc3d02019-08-27 14:40:38 -07008851 return 1;
Jan Dakinevich8530a792019-08-27 13:07:09 +00008852 }
Liran Alone2366172018-03-12 13:12:49 +02008853 return handle_emulation_failure(vcpu, emulation_type);
Carsten Ottebbd9b642007-10-30 18:44:21 +01008854 }
8855 }
8856
Sean Christopherson42cbf062019-08-27 14:40:31 -07008857 if ((emulation_type & EMULTYPE_VMWARE_GP) &&
8858 !is_vmware_backdoor_opcode(ctxt)) {
8859 kvm_queue_exception_e(vcpu, GP_VECTOR, 0);
Sean Christopherson60fc3d02019-08-27 14:40:38 -07008860 return 1;
Sean Christopherson42cbf062019-08-27 14:40:31 -07008861 }
Liran Alon04789b62018-03-12 13:12:50 +02008862
Sean Christopherson1957aa62019-08-27 14:40:39 -07008863 /*
Hou Wenlong906fa902021-11-02 17:15:30 +08008864 * EMULTYPE_SKIP without EMULTYPE_COMPLETE_USER_EXIT is intended for
8865 * use *only* by vendor callbacks for kvm_skip_emulated_instruction().
8866 * The caller is responsible for updating interruptibility state and
8867 * injecting single-step #DBs.
Sean Christopherson1957aa62019-08-27 14:40:39 -07008868 */
Gleb Natapovba8afb62009-04-12 13:36:57 +03008869 if (emulation_type & EMULTYPE_SKIP) {
Sean Christopherson5e854862021-11-02 17:15:29 +08008870 if (ctxt->mode != X86EMUL_MODE_PROT64)
8871 ctxt->eip = (u32)ctxt->_eip;
8872 else
8873 ctxt->eip = ctxt->_eip;
8874
Hou Wenlong906fa902021-11-02 17:15:30 +08008875 if (emulation_type & EMULTYPE_COMPLETE_USER_EXIT) {
8876 r = 1;
8877 goto writeback;
8878 }
8879
Sean Christopherson5e854862021-11-02 17:15:29 +08008880 kvm_rip_write(vcpu, ctxt->eip);
Nadav Amitbb663c72014-07-21 14:37:26 +03008881 if (ctxt->eflags & X86_EFLAGS_RF)
8882 kvm_set_rflags(vcpu, ctxt->eflags & ~X86_EFLAGS_RF);
Sean Christopherson60fc3d02019-08-27 14:40:38 -07008883 return 1;
Gleb Natapovba8afb62009-04-12 13:36:57 +03008884 }
8885
Sean Christopherson736c2912019-12-06 15:57:14 -08008886 if (retry_instruction(ctxt, cr2_or_gpa, emulation_type))
Sean Christopherson60fc3d02019-08-27 14:40:38 -07008887 return 1;
Xiao Guangrong1cb3f3a2011-09-22 17:02:48 +08008888
Gleb Natapov7ae441e2011-03-31 12:06:41 +02008889 /* this is needed for vmware backdoor interface to work since it
Gleb Natapov4d2179e2010-04-28 19:15:42 +03008890 changes registers values during IO operation */
Gleb Natapov7ae441e2011-03-31 12:06:41 +02008891 if (vcpu->arch.emulate_regs_need_sync_from_vcpu) {
8892 vcpu->arch.emulate_regs_need_sync_from_vcpu = false;
Avi Kivitydd856ef2012-08-27 23:46:17 +03008893 emulator_invalidate_register_cache(ctxt);
Gleb Natapov7ae441e2011-03-31 12:06:41 +02008894 }
Gleb Natapov4d2179e2010-04-28 19:15:42 +03008895
Gleb Natapov5cd21912010-03-18 15:20:26 +02008896restart:
Sean Christopherson92daa482020-02-18 15:03:08 -08008897 if (emulation_type & EMULTYPE_PF) {
8898 /* Save the faulting GPA (cr2) in the address field */
8899 ctxt->exception.address = cr2_or_gpa;
8900
8901 /* With shadow page tables, cr2 contains a GVA or nGPA. */
Paolo Bonzini347a0d02022-02-10 08:00:56 -05008902 if (vcpu->arch.mmu->root_role.direct) {
Sean Christopherson744e6992020-02-18 15:03:09 -08008903 ctxt->gpa_available = true;
8904 ctxt->gpa_val = cr2_or_gpa;
Sean Christopherson92daa482020-02-18 15:03:08 -08008905 }
8906 } else {
8907 /* Sanitize the address out of an abundance of paranoia. */
8908 ctxt->exception.address = 0;
8909 }
Tom Lendacky0f89b202016-12-14 14:59:23 -05008910
Takuya Yoshikawa9d741912011-05-29 21:53:48 +09008911 r = x86_emulate_insn(ctxt);
Gleb Natapovc3cd7ff2010-04-28 19:15:35 +03008912
Joerg Roedel775fde82011-04-04 12:39:24 +02008913 if (r == EMULATION_INTERCEPTED)
Sean Christopherson60fc3d02019-08-27 14:40:38 -07008914 return 1;
Joerg Roedel775fde82011-04-04 12:39:24 +02008915
Gleb Natapovd2ddd1c2010-08-25 12:47:43 +03008916 if (r == EMULATION_FAILED) {
Sean Christopherson736c2912019-12-06 15:57:14 -08008917 if (reexecute_instruction(vcpu, cr2_or_gpa, write_fault_to_spt,
Gleb Natapov991eebf2013-04-11 12:10:51 +03008918 emulation_type))
Sean Christopherson60fc3d02019-08-27 14:40:38 -07008919 return 1;
Gleb Natapovc3cd7ff2010-04-28 19:15:35 +03008920
Liran Alone2366172018-03-12 13:12:49 +02008921 return handle_emulation_failure(vcpu, emulation_type);
Carsten Ottebbd9b642007-10-30 18:44:21 +01008922 }
8923
Takuya Yoshikawa9d741912011-05-29 21:53:48 +09008924 if (ctxt->have_exception) {
Sean Christopherson60fc3d02019-08-27 14:40:38 -07008925 r = 1;
Sean Christopherson7709aba2022-08-30 23:16:08 +00008926 inject_emulated_exception(vcpu);
Gleb Natapovd2ddd1c2010-08-25 12:47:43 +03008927 } else if (vcpu->arch.pio.count) {
Paolo Bonzini0912c972013-08-27 15:41:43 +02008928 if (!vcpu->arch.pio.in) {
8929 /* FIXME: return into emulator if single-stepping. */
Gleb Natapove85d28f2010-07-29 15:11:52 +03008930 vcpu->arch.pio.count = 0;
Paolo Bonzini0912c972013-08-27 15:41:43 +02008931 } else {
Gleb Natapov7ae441e2011-03-31 12:06:41 +02008932 writeback = false;
Gleb Natapov716d51a2012-09-03 15:24:26 +03008933 vcpu->arch.complete_userspace_io = complete_emulated_pio;
8934 }
Sean Christopherson60fc3d02019-08-27 14:40:38 -07008935 r = 0;
Gleb Natapov7ae441e2011-03-31 12:06:41 +02008936 } else if (vcpu->mmio_needed) {
Sean Christophersonbc8a0aa2019-08-27 14:40:27 -07008937 ++vcpu->stat.mmio_exits;
8938
Gleb Natapov7ae441e2011-03-31 12:06:41 +02008939 if (!vcpu->mmio_is_write)
8940 writeback = false;
Sean Christopherson60fc3d02019-08-27 14:40:38 -07008941 r = 0;
Gleb Natapov716d51a2012-09-03 15:24:26 +03008942 vcpu->arch.complete_userspace_io = complete_emulated_mmio;
Hou Wenlongadbfb122021-11-02 17:15:32 +08008943 } else if (vcpu->arch.complete_userspace_io) {
8944 writeback = false;
8945 r = 0;
Gleb Natapov7ae441e2011-03-31 12:06:41 +02008946 } else if (r == EMULATION_RESTART)
Gleb Natapove85d28f2010-07-29 15:11:52 +03008947 goto restart;
Gleb Natapovd2ddd1c2010-08-25 12:47:43 +03008948 else
Sean Christopherson60fc3d02019-08-27 14:40:38 -07008949 r = 1;
Gleb Natapove85d28f2010-07-29 15:11:52 +03008950
Hou Wenlong906fa902021-11-02 17:15:30 +08008951writeback:
Gleb Natapov7ae441e2011-03-31 12:06:41 +02008952 if (writeback) {
Jason Baronb36464772021-01-14 22:27:56 -05008953 unsigned long rflags = static_call(kvm_x86_get_rflags)(vcpu);
Takuya Yoshikawa9d741912011-05-29 21:53:48 +09008954 toggle_interruptibility(vcpu, ctxt->interruptibility);
Gleb Natapov7ae441e2011-03-31 12:06:41 +02008955 vcpu->arch.emulate_regs_need_sync_to_vcpu = false;
Sean Christopherson5623f752022-08-30 23:15:55 +00008956
8957 /*
8958 * Note, EXCPT_DB is assumed to be fault-like as the emulator
8959 * only supports code breakpoints and general detect #DB, both
8960 * of which are fault-like.
8961 */
Nadav Amit38827db2014-11-02 11:54:53 +02008962 if (!ctxt->have_exception ||
Sean Christopherson75ee23b2019-08-23 13:55:44 -07008963 exception_type(ctxt->exception.vector) == EXCPT_TRAP) {
Eric Hankland9cd803d2021-11-30 15:42:20 +08008964 kvm_pmu_trigger_event(vcpu, PERF_COUNT_HW_INSTRUCTIONS);
Eric Hankland018d70f2021-11-30 15:42:21 +08008965 if (ctxt->is_branch)
8966 kvm_pmu_trigger_event(vcpu, PERF_COUNT_HW_BRANCH_INSTRUCTIONS);
Sean Christopherson75ee23b2019-08-23 13:55:44 -07008967 kvm_rip_write(vcpu, ctxt->eip);
Felipe Franciosi384dea12020-05-19 08:11:22 +00008968 if (r && (ctxt->tf || (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP)))
Sean Christopherson120c2c42019-08-27 14:40:29 -07008969 r = kvm_vcpu_do_singlestep(vcpu);
Paolo Bonzini2a890612022-02-01 14:18:07 -05008970 static_call_cond(kvm_x86_update_emulated_instruction)(vcpu);
Nadav Amit38827db2014-11-02 11:54:53 +02008971 __kvm_set_rflags(vcpu, ctxt->eflags);
Sean Christopherson75ee23b2019-08-23 13:55:44 -07008972 }
Paolo Bonzini6addfc42014-03-27 11:29:28 +01008973
8974 /*
8975 * For STI, interrupts are shadowed; so KVM_REQ_EVENT will
8976 * do nothing, and it will be requested again as soon as
8977 * the shadow expires. But we still need to check here,
8978 * because POPF has no interrupt shadow.
8979 */
8980 if (unlikely((ctxt->eflags & ~rflags) & X86_EFLAGS_IF))
8981 kvm_make_request(KVM_REQ_EVENT, vcpu);
Gleb Natapov7ae441e2011-03-31 12:06:41 +02008982 } else
8983 vcpu->arch.emulate_regs_need_sync_to_vcpu = true;
Gleb Natapov3457e412010-04-28 19:15:38 +03008984
Gleb Natapove85d28f2010-07-29 15:11:52 +03008985 return r;
Carsten Ottebbd9b642007-10-30 18:44:21 +01008986}
Sean Christophersonc60658d2018-08-23 13:56:53 -07008987
8988int kvm_emulate_instruction(struct kvm_vcpu *vcpu, int emulation_type)
8989{
8990 return x86_emulate_instruction(vcpu, 0, emulation_type, NULL, 0);
8991}
8992EXPORT_SYMBOL_GPL(kvm_emulate_instruction);
8993
8994int kvm_emulate_instruction_from_buffer(struct kvm_vcpu *vcpu,
8995 void *insn, int insn_len)
8996{
8997 return x86_emulate_instruction(vcpu, 0, 0, insn, insn_len);
8998}
8999EXPORT_SYMBOL_GPL(kvm_emulate_instruction_from_buffer);
Carsten Ottebbd9b642007-10-30 18:44:21 +01009000
Sean Christopherson8764ed52019-04-29 07:04:15 -07009001static int complete_fast_pio_out_port_0x7e(struct kvm_vcpu *vcpu)
9002{
9003 vcpu->arch.pio.count = 0;
9004 return 1;
9005}
9006
Sean Christopherson45def772019-03-11 20:01:05 -07009007static int complete_fast_pio_out(struct kvm_vcpu *vcpu)
9008{
9009 vcpu->arch.pio.count = 0;
9010
9011 if (unlikely(!kvm_is_linear_rip(vcpu, vcpu->arch.pio.linear_rip)))
9012 return 1;
9013
9014 return kvm_skip_emulated_instruction(vcpu);
9015}
9016
Sean Christophersondca7f122018-03-08 08:57:27 -08009017static int kvm_fast_pio_out(struct kvm_vcpu *vcpu, int size,
9018 unsigned short port)
Carsten Ottede7d7892007-10-30 18:44:25 +01009019{
Sean Christophersonde3cd112019-04-30 10:36:17 -07009020 unsigned long val = kvm_rax_read(vcpu);
Sean Christopherson2e3bb4d2020-02-18 15:29:41 -08009021 int ret = emulator_pio_out(vcpu, size, port, &val, 1);
9022
Sean Christopherson8764ed52019-04-29 07:04:15 -07009023 if (ret)
9024 return ret;
Sean Christopherson45def772019-03-11 20:01:05 -07009025
Sean Christopherson8764ed52019-04-29 07:04:15 -07009026 /*
9027 * Workaround userspace that relies on old KVM behavior of %rip being
9028 * incremented prior to exiting to userspace to handle "OUT 0x7e".
9029 */
9030 if (port == 0x7e &&
9031 kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_OUT_7E_INC_RIP)) {
9032 vcpu->arch.complete_userspace_io =
9033 complete_fast_pio_out_port_0x7e;
9034 kvm_skip_emulated_instruction(vcpu);
9035 } else {
Sean Christopherson45def772019-03-11 20:01:05 -07009036 vcpu->arch.pio.linear_rip = kvm_get_linear_rip(vcpu);
9037 vcpu->arch.complete_userspace_io = complete_fast_pio_out;
9038 }
Sean Christopherson8764ed52019-04-29 07:04:15 -07009039 return 0;
Carsten Ottede7d7892007-10-30 18:44:25 +01009040}
Carsten Ottede7d7892007-10-30 18:44:25 +01009041
Tom Lendacky8370c3d2016-11-23 12:01:50 -05009042static int complete_fast_pio_in(struct kvm_vcpu *vcpu)
9043{
9044 unsigned long val;
9045
9046 /* We should only ever be called with arch.pio.count equal to 1 */
9047 BUG_ON(vcpu->arch.pio.count != 1);
9048
Sean Christopherson45def772019-03-11 20:01:05 -07009049 if (unlikely(!kvm_is_linear_rip(vcpu, vcpu->arch.pio.linear_rip))) {
9050 vcpu->arch.pio.count = 0;
9051 return 1;
9052 }
9053
Tom Lendacky8370c3d2016-11-23 12:01:50 -05009054 /* For size less than 4 we merge, else we zero extend */
Sean Christophersonde3cd112019-04-30 10:36:17 -07009055 val = (vcpu->arch.pio.size < 4) ? kvm_rax_read(vcpu) : 0;
Tom Lendacky8370c3d2016-11-23 12:01:50 -05009056
Paolo Bonzini7a6177d2022-06-15 13:32:29 -04009057 complete_emulator_pio_in(vcpu, &val);
Sean Christophersonde3cd112019-04-30 10:36:17 -07009058 kvm_rax_write(vcpu, val);
Tom Lendacky8370c3d2016-11-23 12:01:50 -05009059
Sean Christopherson45def772019-03-11 20:01:05 -07009060 return kvm_skip_emulated_instruction(vcpu);
Tom Lendacky8370c3d2016-11-23 12:01:50 -05009061}
9062
Sean Christophersondca7f122018-03-08 08:57:27 -08009063static int kvm_fast_pio_in(struct kvm_vcpu *vcpu, int size,
9064 unsigned short port)
Tom Lendacky8370c3d2016-11-23 12:01:50 -05009065{
9066 unsigned long val;
9067 int ret;
9068
9069 /* For size less than 4 we merge, else we zero extend */
Sean Christophersonde3cd112019-04-30 10:36:17 -07009070 val = (size < 4) ? kvm_rax_read(vcpu) : 0;
Tom Lendacky8370c3d2016-11-23 12:01:50 -05009071
Sean Christopherson2e3bb4d2020-02-18 15:29:41 -08009072 ret = emulator_pio_in(vcpu, size, port, &val, 1);
Tom Lendacky8370c3d2016-11-23 12:01:50 -05009073 if (ret) {
Sean Christophersonde3cd112019-04-30 10:36:17 -07009074 kvm_rax_write(vcpu, val);
Tom Lendacky8370c3d2016-11-23 12:01:50 -05009075 return ret;
9076 }
9077
Sean Christopherson45def772019-03-11 20:01:05 -07009078 vcpu->arch.pio.linear_rip = kvm_get_linear_rip(vcpu);
Tom Lendacky8370c3d2016-11-23 12:01:50 -05009079 vcpu->arch.complete_userspace_io = complete_fast_pio_in;
9080
9081 return 0;
9082}
Sean Christophersondca7f122018-03-08 08:57:27 -08009083
9084int kvm_fast_pio(struct kvm_vcpu *vcpu, int size, unsigned short port, int in)
9085{
Sean Christopherson45def772019-03-11 20:01:05 -07009086 int ret;
Sean Christophersondca7f122018-03-08 08:57:27 -08009087
Sean Christophersondca7f122018-03-08 08:57:27 -08009088 if (in)
Sean Christopherson45def772019-03-11 20:01:05 -07009089 ret = kvm_fast_pio_in(vcpu, size, port);
Sean Christophersondca7f122018-03-08 08:57:27 -08009090 else
Sean Christopherson45def772019-03-11 20:01:05 -07009091 ret = kvm_fast_pio_out(vcpu, size, port);
9092 return ret && kvm_skip_emulated_instruction(vcpu);
Sean Christophersondca7f122018-03-08 08:57:27 -08009093}
9094EXPORT_SYMBOL_GPL(kvm_fast_pio);
Tom Lendacky8370c3d2016-11-23 12:01:50 -05009095
Sebastian Andrzej Siewior251a5fd2016-07-13 17:16:33 +00009096static int kvmclock_cpu_down_prep(unsigned int cpu)
Gerd Hoffmannc8076602009-02-04 17:52:04 +01009097{
Tejun Heo0a3aee02010-12-18 16:28:55 +01009098 __this_cpu_write(cpu_tsc_khz, 0);
Sebastian Andrzej Siewior251a5fd2016-07-13 17:16:33 +00009099 return 0;
Zachary Amsden8cfdc002010-08-19 22:07:21 -10009100}
9101
9102static void tsc_khz_changed(void *data)
9103{
9104 struct cpufreq_freqs *freq = data;
9105 unsigned long khz = 0;
9106
9107 if (data)
9108 khz = freq->new;
9109 else if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC))
9110 khz = cpufreq_quick_get(raw_smp_processor_id());
9111 if (!khz)
9112 khz = tsc_khz;
Tejun Heo0a3aee02010-12-18 16:28:55 +01009113 __this_cpu_write(cpu_tsc_khz, khz);
Gerd Hoffmannc8076602009-02-04 17:52:04 +01009114}
9115
Thomas Gleixner5fa4ec92018-01-31 09:41:40 +01009116#ifdef CONFIG_X86_64
Vitaly Kuznetsov0092e432018-01-24 14:23:37 +01009117static void kvm_hyperv_tsc_notifier(void)
9118{
Vitaly Kuznetsov0092e432018-01-24 14:23:37 +01009119 struct kvm *kvm;
Vitaly Kuznetsov0092e432018-01-24 14:23:37 +01009120 int cpu;
9121
Junaid Shahid0d9ce162019-01-03 17:14:28 -08009122 mutex_lock(&kvm_lock);
Vitaly Kuznetsov0092e432018-01-24 14:23:37 +01009123 list_for_each_entry(kvm, &vm_list, vm_list)
9124 kvm_make_mclock_inprogress_request(kvm);
9125
Paolo Bonzini6b6fcd22021-09-16 18:15:32 +00009126 /* no guest entries from this point */
Vitaly Kuznetsov0092e432018-01-24 14:23:37 +01009127 hyperv_stop_tsc_emulation();
9128
9129 /* TSC frequency always matches when on Hyper-V */
9130 for_each_present_cpu(cpu)
9131 per_cpu(cpu_tsc_khz, cpu) = tsc_khz;
Sean Christopherson938c8742022-05-24 21:56:23 +08009132 kvm_caps.max_guest_tsc_khz = tsc_khz;
Vitaly Kuznetsov0092e432018-01-24 14:23:37 +01009133
9134 list_for_each_entry(kvm, &vm_list, vm_list) {
Paolo Bonzini869b4422021-09-16 18:15:36 +00009135 __kvm_start_pvclock_update(kvm);
Vitaly Kuznetsov0092e432018-01-24 14:23:37 +01009136 pvclock_update_vm_gtod_copy(kvm);
Paolo Bonzini6b6fcd22021-09-16 18:15:32 +00009137 kvm_end_pvclock_update(kvm);
Vitaly Kuznetsov0092e432018-01-24 14:23:37 +01009138 }
Paolo Bonzini6b6fcd22021-09-16 18:15:32 +00009139
Junaid Shahid0d9ce162019-01-03 17:14:28 -08009140 mutex_unlock(&kvm_lock);
Vitaly Kuznetsov0092e432018-01-24 14:23:37 +01009141}
Thomas Gleixner5fa4ec92018-01-31 09:41:40 +01009142#endif
Vitaly Kuznetsov0092e432018-01-24 14:23:37 +01009143
Viresh Kumardf240142019-04-29 15:03:58 +05309144static void __kvmclock_cpufreq_notifier(struct cpufreq_freqs *freq, int cpu)
Gerd Hoffmannc8076602009-02-04 17:52:04 +01009145{
Gerd Hoffmannc8076602009-02-04 17:52:04 +01009146 struct kvm *kvm;
9147 struct kvm_vcpu *vcpu;
Marc Zyngier46808a42021-11-16 16:04:02 +00009148 int send_ipi = 0;
9149 unsigned long i;
Gerd Hoffmannc8076602009-02-04 17:52:04 +01009150
Zachary Amsden8cfdc002010-08-19 22:07:21 -10009151 /*
9152 * We allow guests to temporarily run on slowing clocks,
9153 * provided we notify them after, or to run on accelerating
9154 * clocks, provided we notify them before. Thus time never
9155 * goes backwards.
9156 *
9157 * However, we have a problem. We can't atomically update
9158 * the frequency of a given CPU from this function; it is
9159 * merely a notifier, which can be called from any CPU.
9160 * Changing the TSC frequency at arbitrary points in time
9161 * requires a recomputation of local variables related to
9162 * the TSC for each VCPU. We must flag these local variables
9163 * to be updated and be sure the update takes place with the
9164 * new frequency before any guests proceed.
9165 *
9166 * Unfortunately, the combination of hotplug CPU and frequency
9167 * change creates an intractable locking scenario; the order
9168 * of when these callouts happen is undefined with respect to
9169 * CPU hotplug, and they can race with each other. As such,
9170 * merely setting per_cpu(cpu_tsc_khz) = X during a hotadd is
9171 * undefined; you can actually have a CPU frequency change take
9172 * place in between the computation of X and the setting of the
9173 * variable. To protect against this problem, all updates of
9174 * the per_cpu tsc_khz variable are done in an interrupt
9175 * protected IPI, and all callers wishing to update the value
9176 * must wait for a synchronous IPI to complete (which is trivial
9177 * if the caller is on the CPU already). This establishes the
9178 * necessary total order on variable updates.
9179 *
9180 * Note that because a guest time update may take place
9181 * anytime after the setting of the VCPU's request bit, the
9182 * correct TSC value must be set before the request. However,
9183 * to ensure the update actually makes it to any guest which
9184 * starts running in hardware virtualization between the set
9185 * and the acquisition of the spinlock, we must also ping the
9186 * CPU after setting the request bit.
9187 *
9188 */
9189
Viresh Kumardf240142019-04-29 15:03:58 +05309190 smp_call_function_single(cpu, tsc_khz_changed, freq, 1);
Gerd Hoffmannc8076602009-02-04 17:52:04 +01009191
Junaid Shahid0d9ce162019-01-03 17:14:28 -08009192 mutex_lock(&kvm_lock);
Gerd Hoffmannc8076602009-02-04 17:52:04 +01009193 list_for_each_entry(kvm, &vm_list, vm_list) {
Gleb Natapov988a2ca2009-06-09 15:56:29 +03009194 kvm_for_each_vcpu(i, vcpu, kvm) {
Viresh Kumardf240142019-04-29 15:03:58 +05309195 if (vcpu->cpu != cpu)
Gerd Hoffmannc8076602009-02-04 17:52:04 +01009196 continue;
Zachary Amsdenc2855452010-09-18 14:38:15 -10009197 kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
Junaid Shahid0d9ce162019-01-03 17:14:28 -08009198 if (vcpu->cpu != raw_smp_processor_id())
Zachary Amsden8cfdc002010-08-19 22:07:21 -10009199 send_ipi = 1;
Gerd Hoffmannc8076602009-02-04 17:52:04 +01009200 }
9201 }
Junaid Shahid0d9ce162019-01-03 17:14:28 -08009202 mutex_unlock(&kvm_lock);
Gerd Hoffmannc8076602009-02-04 17:52:04 +01009203
9204 if (freq->old < freq->new && send_ipi) {
9205 /*
9206 * We upscale the frequency. Must make the guest
9207 * doesn't see old kvmclock values while running with
9208 * the new frequency, otherwise we risk the guest sees
9209 * time go backwards.
9210 *
9211 * In case we update the frequency for another cpu
9212 * (which might be in guest context) send an interrupt
9213 * to kick the cpu out of guest context. Next time
9214 * guest context is entered kvmclock will be updated,
9215 * so the guest will not see stale values.
9216 */
Viresh Kumardf240142019-04-29 15:03:58 +05309217 smp_call_function_single(cpu, tsc_khz_changed, freq, 1);
Gerd Hoffmannc8076602009-02-04 17:52:04 +01009218 }
Viresh Kumardf240142019-04-29 15:03:58 +05309219}
9220
9221static int kvmclock_cpufreq_notifier(struct notifier_block *nb, unsigned long val,
9222 void *data)
9223{
9224 struct cpufreq_freqs *freq = data;
9225 int cpu;
9226
9227 if (val == CPUFREQ_PRECHANGE && freq->old > freq->new)
9228 return 0;
9229 if (val == CPUFREQ_POSTCHANGE && freq->old < freq->new)
9230 return 0;
9231
9232 for_each_cpu(cpu, freq->policy->cpus)
9233 __kvmclock_cpufreq_notifier(freq, cpu);
9234
Gerd Hoffmannc8076602009-02-04 17:52:04 +01009235 return 0;
9236}
9237
9238static struct notifier_block kvmclock_cpufreq_notifier_block = {
Zachary Amsden8cfdc002010-08-19 22:07:21 -10009239 .notifier_call = kvmclock_cpufreq_notifier
9240};
9241
Sebastian Andrzej Siewior251a5fd2016-07-13 17:16:33 +00009242static int kvmclock_cpu_online(unsigned int cpu)
Zachary Amsden8cfdc002010-08-19 22:07:21 -10009243{
Sebastian Andrzej Siewior251a5fd2016-07-13 17:16:33 +00009244 tsc_khz_changed(NULL);
9245 return 0;
Zachary Amsden8cfdc002010-08-19 22:07:21 -10009246}
9247
Zachary Amsdenb820cc02009-09-29 11:38:34 -10009248static void kvm_timer_init(void)
9249{
Zachary Amsdenb820cc02009-09-29 11:38:34 -10009250 if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC)) {
Sean Christopherson741e5112022-02-25 14:53:03 +00009251 max_tsc_khz = tsc_khz;
Borislav Petkov758f5882016-09-04 19:13:57 +02009252
Sean Christopherson741e5112022-02-25 14:53:03 +00009253 if (IS_ENABLED(CONFIG_CPU_FREQ)) {
9254 struct cpufreq_policy *policy;
9255 int cpu;
9256
9257 cpu = get_cpu();
9258 policy = cpufreq_cpu_get(cpu);
9259 if (policy) {
9260 if (policy->cpuinfo.max_freq)
9261 max_tsc_khz = policy->cpuinfo.max_freq;
9262 cpufreq_cpu_put(policy);
9263 }
9264 put_cpu();
Wanpeng Li9a119972020-03-02 15:15:36 +08009265 }
Zachary Amsdenb820cc02009-09-29 11:38:34 -10009266 cpufreq_register_notifier(&kvmclock_cpufreq_notifier_block,
9267 CPUFREQ_TRANSITION_NOTIFIER);
9268 }
Srivatsa S. Bhat460dd422014-03-11 02:09:01 +05309269
Thomas Gleixner73c1b412016-12-21 20:19:54 +01009270 cpuhp_setup_state(CPUHP_AP_X86_KVM_CLK_ONLINE, "x86/kvm/clk:online",
Sebastian Andrzej Siewior251a5fd2016-07-13 17:16:33 +00009271 kvmclock_cpu_online, kvmclock_cpu_down_prep);
Zachary Amsdenb820cc02009-09-29 11:38:34 -10009272}
9273
Marcelo Tosatti16e8d742012-11-27 23:29:00 -02009274#ifdef CONFIG_X86_64
9275static void pvclock_gtod_update_fn(struct work_struct *work)
9276{
Marcelo Tosattid8281992012-11-27 23:29:01 -02009277 struct kvm *kvm;
Marcelo Tosattid8281992012-11-27 23:29:01 -02009278 struct kvm_vcpu *vcpu;
Marc Zyngier46808a42021-11-16 16:04:02 +00009279 unsigned long i;
Marcelo Tosattid8281992012-11-27 23:29:01 -02009280
Junaid Shahid0d9ce162019-01-03 17:14:28 -08009281 mutex_lock(&kvm_lock);
Marcelo Tosattid8281992012-11-27 23:29:01 -02009282 list_for_each_entry(kvm, &vm_list, vm_list)
9283 kvm_for_each_vcpu(i, vcpu, kvm)
Guo Hui Liu105b21b2014-09-12 13:43:19 +08009284 kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu);
Marcelo Tosattid8281992012-11-27 23:29:01 -02009285 atomic_set(&kvm_guest_has_master_clock, 0);
Junaid Shahid0d9ce162019-01-03 17:14:28 -08009286 mutex_unlock(&kvm_lock);
Marcelo Tosatti16e8d742012-11-27 23:29:00 -02009287}
9288
9289static DECLARE_WORK(pvclock_gtod_work, pvclock_gtod_update_fn);
9290
9291/*
Thomas Gleixner3f804f62021-05-06 15:21:37 +02009292 * Indirection to move queue_work() out of the tk_core.seq write held
9293 * region to prevent possible deadlocks against time accessors which
9294 * are invoked with work related locks held.
9295 */
9296static void pvclock_irq_work_fn(struct irq_work *w)
9297{
9298 queue_work(system_long_wq, &pvclock_gtod_work);
9299}
9300
9301static DEFINE_IRQ_WORK(pvclock_irq_work, pvclock_irq_work_fn);
9302
9303/*
Marcelo Tosatti16e8d742012-11-27 23:29:00 -02009304 * Notification about pvclock gtod data update.
9305 */
9306static int pvclock_gtod_notify(struct notifier_block *nb, unsigned long unused,
9307 void *priv)
9308{
9309 struct pvclock_gtod_data *gtod = &pvclock_gtod_data;
9310 struct timekeeper *tk = priv;
9311
9312 update_pvclock_gtod(tk);
9313
Thomas Gleixner3f804f62021-05-06 15:21:37 +02009314 /*
9315 * Disable master clock if host does not trust, or does not use,
9316 * TSC based clocksource. Delegate queue_work() to irq_work as
9317 * this is invoked with tk_core.seq write held.
Marcelo Tosatti16e8d742012-11-27 23:29:00 -02009318 */
Vitaly Kuznetsovb0c39dc2018-01-24 14:23:36 +01009319 if (!gtod_is_based_on_tsc(gtod->clock.vclock_mode) &&
Marcelo Tosatti16e8d742012-11-27 23:29:00 -02009320 atomic_read(&kvm_guest_has_master_clock) != 0)
Thomas Gleixner3f804f62021-05-06 15:21:37 +02009321 irq_work_queue(&pvclock_irq_work);
Marcelo Tosatti16e8d742012-11-27 23:29:00 -02009322 return 0;
9323}
9324
9325static struct notifier_block pvclock_gtod_notifier = {
9326 .notifier_call = pvclock_gtod_notify,
9327};
9328#endif
9329
Zhang Xiantaof8c16bb2007-11-14 20:40:21 +08009330int kvm_arch_init(void *opaque)
Carsten Otte043405e2007-10-10 17:16:19 +02009331{
Sean Christophersond008dfd2020-03-21 13:25:56 -07009332 struct kvm_x86_init_ops *ops = opaque;
Sean Christopherson94bda2f2022-07-15 23:00:13 +00009333 u64 host_pat;
Zachary Amsdenb820cc02009-09-29 11:38:34 -10009334 int r;
Zhang Xiantaof8c16bb2007-11-14 20:40:21 +08009335
Sean Christophersonafaf0b22020-03-21 13:26:00 -07009336 if (kvm_x86_ops.hardware_enable) {
Sean Christopherson9dadfc42021-10-18 11:39:28 -07009337 pr_err("kvm: already loaded vendor module '%s'\n", kvm_x86_ops.name);
Sean Christopherson82ffad22022-07-15 23:00:14 +00009338 return -EEXIST;
Zhang Xiantaof8c16bb2007-11-14 20:40:21 +08009339 }
9340
9341 if (!ops->cpu_has_kvm_support()) {
Sean Christopherson9dadfc42021-10-18 11:39:28 -07009342 pr_err_ratelimited("kvm: no hardware support for '%s'\n",
9343 ops->runtime_ops->name);
Sean Christopherson82ffad22022-07-15 23:00:14 +00009344 return -EOPNOTSUPP;
Zhang Xiantaof8c16bb2007-11-14 20:40:21 +08009345 }
9346 if (ops->disabled_by_bios()) {
Sean Christopherson9dadfc42021-10-18 11:39:28 -07009347 pr_err_ratelimited("kvm: support for '%s' disabled by bios\n",
9348 ops->runtime_ops->name);
Sean Christopherson82ffad22022-07-15 23:00:14 +00009349 return -EOPNOTSUPP;
Zhang Xiantaof8c16bb2007-11-14 20:40:21 +08009350 }
9351
Marc Orrb666a4b2018-11-06 14:53:56 -08009352 /*
9353 * KVM explicitly assumes that the guest has an FPU and
9354 * FXSAVE/FXRSTOR. For example, the KVM_GET_FPU explicitly casts the
9355 * vCPU's FPU state as a fxregs_state struct.
9356 */
9357 if (!boot_cpu_has(X86_FEATURE_FPU) || !boot_cpu_has(X86_FEATURE_FXSR)) {
9358 printk(KERN_ERR "kvm: inadequate fpu\n");
Sean Christopherson82ffad22022-07-15 23:00:14 +00009359 return -EOPNOTSUPP;
Marc Orrb666a4b2018-11-06 14:53:56 -08009360 }
9361
Thomas Gleixner5e17b2e2011-11-06 12:26:18 +01009362 if (IS_ENABLED(CONFIG_PREEMPT_RT) && !boot_cpu_has(X86_FEATURE_CONSTANT_TSC)) {
9363 pr_err("RT requires X86_FEATURE_CONSTANT_TSC\n");
Sean Christopherson82ffad22022-07-15 23:00:14 +00009364 return -EOPNOTSUPP;
Thomas Gleixner5e17b2e2011-11-06 12:26:18 +01009365 }
9366
Sean Christopherson94bda2f2022-07-15 23:00:13 +00009367 /*
9368 * KVM assumes that PAT entry '0' encodes WB memtype and simply zeroes
9369 * the PAT bits in SPTEs. Bail if PAT[0] is programmed to something
9370 * other than WB. Note, EPT doesn't utilize the PAT, but don't bother
9371 * with an exception. PAT[0] is set to WB on RESET and also by the
9372 * kernel, i.e. failure indicates a kernel bug or broken firmware.
9373 */
9374 if (rdmsrl_safe(MSR_IA32_CR_PAT, &host_pat) ||
9375 (host_pat & GENMASK(2, 0)) != 6) {
9376 pr_err("kvm: host PAT[0] is not WB\n");
Sean Christopherson82ffad22022-07-15 23:00:14 +00009377 return -EIO;
Sean Christopherson94bda2f2022-07-15 23:00:13 +00009378 }
Marc Orrb666a4b2018-11-06 14:53:56 -08009379
Sean Christophersonc9b8b072020-02-18 15:29:48 -08009380 x86_emulator_cache = kvm_alloc_emulator_cache();
9381 if (!x86_emulator_cache) {
9382 pr_err("kvm: failed to allocate cache for x86 emulator\n");
Sean Christopherson82ffad22022-07-15 23:00:14 +00009383 return -ENOMEM;
Sean Christophersonc9b8b072020-02-18 15:29:48 -08009384 }
9385
Sean Christopherson7e34fbd2020-09-23 11:03:55 -07009386 user_return_msrs = alloc_percpu(struct kvm_user_return_msrs);
9387 if (!user_return_msrs) {
9388 printk(KERN_ERR "kvm: failed to allocate percpu kvm_user_return_msrs\n");
Sean Christopherson82ffad22022-07-15 23:00:14 +00009389 r = -ENOMEM;
Sean Christophersonc9b8b072020-02-18 15:29:48 -08009390 goto out_free_x86_emulator_cache;
Marcelo Tosatti013f6a52013-01-03 11:41:39 -02009391 }
Sean Christophersone5fda4b2021-05-04 10:17:32 -07009392 kvm_nr_uret_msrs = 0;
Marcelo Tosatti013f6a52013-01-03 11:41:39 -02009393
Sean Christopherson1d0e8482022-03-31 22:13:59 +00009394 r = kvm_mmu_vendor_module_init();
Avi Kivity97db56c2008-01-13 13:23:56 +02009395 if (r)
Marcelo Tosatti013f6a52013-01-03 11:41:39 -02009396 goto out_free_percpu;
Avi Kivity97db56c2008-01-13 13:23:56 +02009397
Zachary Amsdenb820cc02009-09-29 11:38:34 -10009398 kvm_timer_init();
Gerd Hoffmannc8076602009-02-04 17:52:04 +01009399
Sean Christophersoncfc48182020-03-02 15:56:23 -08009400 if (boot_cpu_has(X86_FEATURE_XSAVE)) {
Dexuan Cui2acf9232010-06-10 11:27:12 +08009401 host_xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK);
Sean Christopherson938c8742022-05-24 21:56:23 +08009402 kvm_caps.supported_xcr0 = host_xcr0 & KVM_SUPPORTED_XCR0;
Sean Christophersoncfc48182020-03-02 15:56:23 -08009403 }
Dexuan Cui2acf9232010-06-10 11:27:12 +08009404
Wanpeng Li0c5f81d2019-07-06 09:26:51 +08009405 if (pi_inject_timer == -1)
Frederic Weisbecker04d4e662022-02-07 16:59:06 +01009406 pi_inject_timer = housekeeping_enabled(HK_TYPE_TIMER);
Marcelo Tosatti16e8d742012-11-27 23:29:00 -02009407#ifdef CONFIG_X86_64
9408 pvclock_gtod_register_notifier(&pvclock_gtod_notifier);
Vitaly Kuznetsov0092e432018-01-24 14:23:37 +01009409
Thomas Gleixner5fa4ec92018-01-31 09:41:40 +01009410 if (hypervisor_is_type(X86_HYPER_MS_HYPERV))
Vitaly Kuznetsov0092e432018-01-24 14:23:37 +01009411 set_hv_tscchange_cb(kvm_hyperv_tsc_notifier);
Marcelo Tosatti16e8d742012-11-27 23:29:00 -02009412#endif
9413
Zhang Xiantaof8c16bb2007-11-14 20:40:21 +08009414 return 0;
Zhang Xiantao56c6d282007-11-18 20:43:21 +08009415
Marcelo Tosatti013f6a52013-01-03 11:41:39 -02009416out_free_percpu:
Sean Christopherson7e34fbd2020-09-23 11:03:55 -07009417 free_percpu(user_return_msrs);
Sean Christophersonc9b8b072020-02-18 15:29:48 -08009418out_free_x86_emulator_cache:
9419 kmem_cache_destroy(x86_emulator_cache);
Zhang Xiantao56c6d282007-11-18 20:43:21 +08009420 return r;
Carsten Otte043405e2007-10-10 17:16:19 +02009421}
Hollis Blanchard8776e512007-10-31 17:24:24 -05009422
Zhang Xiantaof8c16bb2007-11-14 20:40:21 +08009423void kvm_arch_exit(void)
9424{
Vitaly Kuznetsov0092e432018-01-24 14:23:37 +01009425#ifdef CONFIG_X86_64
Thomas Gleixner5fa4ec92018-01-31 09:41:40 +01009426 if (hypervisor_is_type(X86_HYPER_MS_HYPERV))
Vitaly Kuznetsov0092e432018-01-24 14:23:37 +01009427 clear_hv_tscchange_cb();
9428#endif
David Matlackcef84c32016-12-16 14:30:36 -08009429 kvm_lapic_exit();
Zhang, Yanminff9d07a2010-04-19 13:32:45 +08009430
Jan Kiszka888d2562009-04-17 19:24:58 +02009431 if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC))
9432 cpufreq_unregister_notifier(&kvmclock_cpufreq_notifier_block,
9433 CPUFREQ_TRANSITION_NOTIFIER);
Sebastian Andrzej Siewior251a5fd2016-07-13 17:16:33 +00009434 cpuhp_remove_state_nocalls(CPUHP_AP_X86_KVM_CLK_ONLINE);
Marcelo Tosatti16e8d742012-11-27 23:29:00 -02009435#ifdef CONFIG_X86_64
9436 pvclock_gtod_unregister_notifier(&pvclock_gtod_notifier);
Thomas Gleixner3f804f62021-05-06 15:21:37 +02009437 irq_work_sync(&pvclock_irq_work);
Thomas Gleixner594b27e2021-05-05 23:48:17 +02009438 cancel_work_sync(&pvclock_gtod_work);
Marcelo Tosatti16e8d742012-11-27 23:29:00 -02009439#endif
Sean Christophersonafaf0b22020-03-21 13:26:00 -07009440 kvm_x86_ops.hardware_enable = NULL;
Sean Christopherson1d0e8482022-03-31 22:13:59 +00009441 kvm_mmu_vendor_module_exit();
Sean Christopherson7e34fbd2020-09-23 11:03:55 -07009442 free_percpu(user_return_msrs);
Wanpeng Lidfdc0a72021-06-10 21:59:33 -07009443 kmem_cache_destroy(x86_emulator_cache);
Paolo Bonzinib59b1532021-02-26 04:54:45 -05009444#ifdef CONFIG_KVM_XEN
Paolo Bonzinic462f852021-02-26 04:49:06 -05009445 static_key_deferred_flush(&kvm_xen_enabled);
David Woodhouse7d6bbeb2021-02-02 15:48:05 +00009446 WARN_ON(static_branch_unlikely(&kvm_xen_enabled.key));
Paolo Bonzinib59b1532021-02-26 04:54:45 -05009447#endif
Zhang Xiantao56c6d282007-11-18 20:43:21 +08009448}
Zhang Xiantaof8c16bb2007-11-14 20:40:21 +08009449
Sean Christopherson14601792021-10-08 19:12:05 -07009450static int __kvm_emulate_halt(struct kvm_vcpu *vcpu, int state, int reason)
Hollis Blanchard8776e512007-10-31 17:24:24 -05009451{
Sean Christopherson91b99ea2021-10-08 19:12:06 -07009452 /*
9453 * The vCPU has halted, e.g. executed HLT. Update the run state if the
9454 * local APIC is in-kernel, the run loop will detect the non-runnable
9455 * state and halt the vCPU. Exit to userspace if the local APIC is
9456 * managed by userspace, in which case userspace is responsible for
9457 * handling wake events.
9458 */
Hollis Blanchard8776e512007-10-31 17:24:24 -05009459 ++vcpu->stat.halt_exits;
Paolo Bonzini35754c92015-07-29 12:05:37 +02009460 if (lapic_in_kernel(vcpu)) {
Tom Lendacky647daca2021-01-04 14:20:01 -06009461 vcpu->arch.mp_state = state;
Hollis Blanchard8776e512007-10-31 17:24:24 -05009462 return 1;
9463 } else {
Tom Lendacky647daca2021-01-04 14:20:01 -06009464 vcpu->run->exit_reason = reason;
Hollis Blanchard8776e512007-10-31 17:24:24 -05009465 return 0;
9466 }
9467}
Tom Lendacky647daca2021-01-04 14:20:01 -06009468
Sean Christopherson14601792021-10-08 19:12:05 -07009469int kvm_emulate_halt_noskip(struct kvm_vcpu *vcpu)
Tom Lendacky647daca2021-01-04 14:20:01 -06009470{
Sean Christopherson14601792021-10-08 19:12:05 -07009471 return __kvm_emulate_halt(vcpu, KVM_MP_STATE_HALTED, KVM_EXIT_HLT);
Tom Lendacky647daca2021-01-04 14:20:01 -06009472}
Sean Christopherson14601792021-10-08 19:12:05 -07009473EXPORT_SYMBOL_GPL(kvm_emulate_halt_noskip);
Joel Schopp5cb56052015-03-02 13:43:31 -06009474
9475int kvm_emulate_halt(struct kvm_vcpu *vcpu)
9476{
Kyle Huey6affcbe2016-11-29 12:40:40 -08009477 int ret = kvm_skip_emulated_instruction(vcpu);
9478 /*
9479 * TODO: we might be squashing a GUESTDBG_SINGLESTEP-triggered
9480 * KVM_EXIT_DEBUG here.
9481 */
Sean Christopherson14601792021-10-08 19:12:05 -07009482 return kvm_emulate_halt_noskip(vcpu) && ret;
Joel Schopp5cb56052015-03-02 13:43:31 -06009483}
Hollis Blanchard8776e512007-10-31 17:24:24 -05009484EXPORT_SYMBOL_GPL(kvm_emulate_halt);
9485
Tom Lendacky647daca2021-01-04 14:20:01 -06009486int kvm_emulate_ap_reset_hold(struct kvm_vcpu *vcpu)
9487{
9488 int ret = kvm_skip_emulated_instruction(vcpu);
9489
Sean Christopherson14601792021-10-08 19:12:05 -07009490 return __kvm_emulate_halt(vcpu, KVM_MP_STATE_AP_RESET_HOLD,
9491 KVM_EXIT_AP_RESET_HOLD) && ret;
Tom Lendacky647daca2021-01-04 14:20:01 -06009492}
9493EXPORT_SYMBOL_GPL(kvm_emulate_ap_reset_hold);
9494
Arnd Bergmann8ef81a92017-02-09 16:10:42 +01009495#ifdef CONFIG_X86_64
Marcelo Tosatti55dd00a2017-01-24 15:09:39 -02009496static int kvm_pv_clock_pairing(struct kvm_vcpu *vcpu, gpa_t paddr,
9497 unsigned long clock_type)
9498{
9499 struct kvm_clock_pairing clock_pairing;
Arnd Bergmann899a31f2018-04-23 10:04:26 +02009500 struct timespec64 ts;
Paolo Bonzini80fbd892017-02-08 10:57:24 +01009501 u64 cycle;
Marcelo Tosatti55dd00a2017-01-24 15:09:39 -02009502 int ret;
9503
9504 if (clock_type != KVM_CLOCK_PAIRING_WALLCLOCK)
9505 return -KVM_EOPNOTSUPP;
9506
Anton Romanov3a55f722022-02-16 18:26:54 +00009507 /*
9508 * When tsc is in permanent catchup mode guests won't be able to use
9509 * pvclock_read_retry loop to get consistent view of pvclock
9510 */
9511 if (vcpu->arch.tsc_always_catchup)
9512 return -KVM_EOPNOTSUPP;
9513
YANG LI7ca7f3b2021-01-11 17:32:58 +08009514 if (!kvm_get_walltime_and_clockread(&ts, &cycle))
Marcelo Tosatti55dd00a2017-01-24 15:09:39 -02009515 return -KVM_EOPNOTSUPP;
9516
9517 clock_pairing.sec = ts.tv_sec;
9518 clock_pairing.nsec = ts.tv_nsec;
9519 clock_pairing.tsc = kvm_read_l1_tsc(vcpu, cycle);
9520 clock_pairing.flags = 0;
Liran Alonbcbfbd82018-11-08 00:43:06 +02009521 memset(&clock_pairing.pad, 0, sizeof(clock_pairing.pad));
Marcelo Tosatti55dd00a2017-01-24 15:09:39 -02009522
9523 ret = 0;
9524 if (kvm_write_guest(vcpu->kvm, paddr, &clock_pairing,
9525 sizeof(struct kvm_clock_pairing)))
9526 ret = -KVM_EFAULT;
9527
9528 return ret;
9529}
Arnd Bergmann8ef81a92017-02-09 16:10:42 +01009530#endif
Marcelo Tosatti55dd00a2017-01-24 15:09:39 -02009531
Srivatsa Vaddagiri6aef2662013-08-26 14:18:34 +05309532/*
9533 * kvm_pv_kick_cpu_op: Kick a vcpu.
9534 *
9535 * @apicid - apicid of vcpu to be kicked.
9536 */
Jinrong Liang9d68c6f2022-01-25 17:59:09 +08009537static void kvm_pv_kick_cpu_op(struct kvm *kvm, int apicid)
Srivatsa Vaddagiri6aef2662013-08-26 14:18:34 +05309538{
Vitaly Kuznetsov8a414f92022-07-08 14:51:47 +02009539 /*
9540 * All other fields are unused for APIC_DM_REMRD, but may be consumed by
9541 * common code, e.g. for tracing. Defer initialization to the compiler.
9542 */
9543 struct kvm_lapic_irq lapic_irq = {
9544 .delivery_mode = APIC_DM_REMRD,
9545 .dest_mode = APIC_DEST_PHYSICAL,
9546 .shorthand = APIC_DEST_NOSHORT,
9547 .dest_id = apicid,
9548 };
Srivatsa Vaddagiri6aef2662013-08-26 14:18:34 +05309549
Xiubo Li795a1492015-03-13 17:39:44 +08009550 kvm_irq_delivery_to_apic(kvm, NULL, &lapic_irq, NULL);
Srivatsa Vaddagiri6aef2662013-08-26 14:18:34 +05309551}
9552
Suravee Suthikulpanit4e19c362019-11-14 14:15:05 -06009553bool kvm_apicv_activated(struct kvm *kvm)
9554{
9555 return (READ_ONCE(kvm->arch.apicv_inhibit_reasons) == 0);
9556}
9557EXPORT_SYMBOL_GPL(kvm_apicv_activated);
9558
Maxim Levitskyd5fa5972022-03-22 19:40:49 +02009559bool kvm_vcpu_apicv_activated(struct kvm_vcpu *vcpu)
9560{
9561 ulong vm_reasons = READ_ONCE(vcpu->kvm->arch.apicv_inhibit_reasons);
9562 ulong vcpu_reasons = static_call(kvm_x86_vcpu_get_apicv_inhibit_reasons)(vcpu);
9563
9564 return (vm_reasons | vcpu_reasons) == 0;
9565}
9566EXPORT_SYMBOL_GPL(kvm_vcpu_apicv_activated);
Sean Christopherson4f4c4a32022-03-11 04:35:17 +00009567
9568static void set_or_clear_apicv_inhibit(unsigned long *inhibits,
9569 enum kvm_apicv_inhibit reason, bool set)
9570{
9571 if (set)
9572 __set_bit(reason, inhibits);
9573 else
9574 __clear_bit(reason, inhibits);
9575
9576 trace_kvm_apicv_inhibit_changed(reason, set, *inhibits);
9577}
9578
Vitaly Kuznetsov4651fc52021-06-09 17:09:09 +02009579static void kvm_apicv_init(struct kvm *kvm)
Suravee Suthikulpanit4e19c362019-11-14 14:15:05 -06009580{
Sean Christopherson4f4c4a32022-03-11 04:35:17 +00009581 unsigned long *inhibits = &kvm->arch.apicv_inhibit_reasons;
9582
Sean Christopherson187c8832021-10-21 17:49:27 -07009583 init_rwsem(&kvm->arch.apicv_update_lock);
Maxim Levitskyb0a16372021-08-10 23:52:44 +03009584
Sean Christopherson4f4c4a32022-03-11 04:35:17 +00009585 set_or_clear_apicv_inhibit(inhibits, APICV_INHIBIT_REASON_ABSENT, true);
9586
Paolo Bonzinief8b4b72021-11-30 07:37:45 -05009587 if (!enable_apicv)
Sean Christopherson4f4c4a32022-03-11 04:35:17 +00009588 set_or_clear_apicv_inhibit(inhibits,
Sean Christopherson80f04972022-04-20 01:37:29 +00009589 APICV_INHIBIT_REASON_DISABLE, true);
Suravee Suthikulpanit4e19c362019-11-14 14:15:05 -06009590}
Suravee Suthikulpanit4e19c362019-11-14 14:15:05 -06009591
Wanpeng Li4a7132e2021-04-09 12:18:30 +08009592static void kvm_sched_yield(struct kvm_vcpu *vcpu, unsigned long dest_id)
Wanpeng Li71506292019-06-11 20:23:49 +08009593{
9594 struct kvm_vcpu *target = NULL;
9595 struct kvm_apic_map *map;
9596
Wanpeng Li4a7132e2021-04-09 12:18:30 +08009597 vcpu->stat.directed_yield_attempted++;
9598
Wanpeng Li72b268a2021-05-18 05:00:32 -07009599 if (single_task_running())
9600 goto no_yield;
9601
Wanpeng Li71506292019-06-11 20:23:49 +08009602 rcu_read_lock();
Wanpeng Li4a7132e2021-04-09 12:18:30 +08009603 map = rcu_dereference(vcpu->kvm->arch.apic_map);
Wanpeng Li71506292019-06-11 20:23:49 +08009604
9605 if (likely(map) && dest_id <= map->max_apic_id && map->phys_map[dest_id])
9606 target = map->phys_map[dest_id]->vcpu;
9607
9608 rcu_read_unlock();
9609
Wanpeng Li4a7132e2021-04-09 12:18:30 +08009610 if (!target || !READ_ONCE(target->ready))
9611 goto no_yield;
9612
Wanpeng Lia1fa4cb2021-04-09 12:18:31 +08009613 /* Ignore requests to yield to self */
9614 if (vcpu == target)
9615 goto no_yield;
9616
Wanpeng Li4a7132e2021-04-09 12:18:30 +08009617 if (kvm_vcpu_yield_to(target) <= 0)
9618 goto no_yield;
9619
9620 vcpu->stat.directed_yield_successful++;
9621
9622no_yield:
9623 return;
Wanpeng Li71506292019-06-11 20:23:49 +08009624}
9625
Ashish Kalra0dbb1122021-06-08 18:05:43 +00009626static int complete_hypercall_exit(struct kvm_vcpu *vcpu)
9627{
9628 u64 ret = vcpu->run->hypercall.ret;
9629
9630 if (!is_64_bit_mode(vcpu))
9631 ret = (u32)ret;
9632 kvm_rax_write(vcpu, ret);
9633 ++vcpu->stat.hypercalls;
9634 return kvm_skip_emulated_instruction(vcpu);
9635}
9636
Hollis Blanchard8776e512007-10-31 17:24:24 -05009637int kvm_emulate_hypercall(struct kvm_vcpu *vcpu)
9638{
9639 unsigned long nr, a0, a1, a2, a3, ret;
Marian Rotariu6356ee02018-04-30 12:23:01 +03009640 int op_64_bit;
Hollis Blanchard8776e512007-10-31 17:24:24 -05009641
Joao Martins23200b72018-06-13 09:55:44 -04009642 if (kvm_xen_hypercall_enabled(vcpu->kvm))
9643 return kvm_xen_hypercall(vcpu);
9644
Vitaly Kuznetsov8f014552021-01-26 14:48:14 +01009645 if (kvm_hv_hypercall_enabled(vcpu))
Radim Krčmář696ca7792018-05-24 17:50:56 +02009646 return kvm_hv_hypercall(vcpu);
Gleb Natapov55cd8e52010-01-17 15:51:22 +02009647
Sean Christophersonde3cd112019-04-30 10:36:17 -07009648 nr = kvm_rax_read(vcpu);
9649 a0 = kvm_rbx_read(vcpu);
9650 a1 = kvm_rcx_read(vcpu);
9651 a2 = kvm_rdx_read(vcpu);
9652 a3 = kvm_rsi_read(vcpu);
Hollis Blanchard8776e512007-10-31 17:24:24 -05009653
Marcelo Tosatti229456f2009-06-17 09:22:14 -03009654 trace_kvm_hypercall(nr, a0, a1, a2, a3);
Feng (Eric) Liu2714d1d2008-04-10 15:31:10 -04009655
Tom Lendackyb5aead02021-05-24 12:48:57 -05009656 op_64_bit = is_64_bit_hypercall(vcpu);
Nadav Amita449c7a2014-06-18 17:19:24 +03009657 if (!op_64_bit) {
Hollis Blanchard8776e512007-10-31 17:24:24 -05009658 nr &= 0xFFFFFFFF;
9659 a0 &= 0xFFFFFFFF;
9660 a1 &= 0xFFFFFFFF;
9661 a2 &= 0xFFFFFFFF;
9662 a3 &= 0xFFFFFFFF;
9663 }
9664
Jason Baronb36464772021-01-14 22:27:56 -05009665 if (static_call(kvm_x86_get_cpl)(vcpu) != 0) {
Jan Kiszka07708c42009-08-03 18:43:28 +02009666 ret = -KVM_EPERM;
Radim Krčmář696ca7792018-05-24 17:50:56 +02009667 goto out;
Jan Kiszka07708c42009-08-03 18:43:28 +02009668 }
9669
Oliver Upton66570e962020-08-18 15:24:28 +00009670 ret = -KVM_ENOSYS;
9671
Hollis Blanchard8776e512007-10-31 17:24:24 -05009672 switch (nr) {
Avi Kivityb93463a2007-10-25 16:52:32 +02009673 case KVM_HC_VAPIC_POLL_IRQ:
9674 ret = 0;
9675 break;
Srivatsa Vaddagiri6aef2662013-08-26 14:18:34 +05309676 case KVM_HC_KICK_CPU:
Oliver Upton66570e962020-08-18 15:24:28 +00009677 if (!guest_pv_has(vcpu, KVM_FEATURE_PV_UNHALT))
9678 break;
9679
Jinrong Liang9d68c6f2022-01-25 17:59:09 +08009680 kvm_pv_kick_cpu_op(vcpu->kvm, a1);
Wanpeng Li4a7132e2021-04-09 12:18:30 +08009681 kvm_sched_yield(vcpu, a1);
Srivatsa Vaddagiri6aef2662013-08-26 14:18:34 +05309682 ret = 0;
9683 break;
Arnd Bergmann8ef81a92017-02-09 16:10:42 +01009684#ifdef CONFIG_X86_64
Marcelo Tosatti55dd00a2017-01-24 15:09:39 -02009685 case KVM_HC_CLOCK_PAIRING:
9686 ret = kvm_pv_clock_pairing(vcpu, a0, a1);
9687 break;
Sean Christopherson1ed199a2019-01-23 09:22:39 -08009688#endif
Wanpeng Li4180bf12018-07-23 14:39:54 +08009689 case KVM_HC_SEND_IPI:
Oliver Upton66570e962020-08-18 15:24:28 +00009690 if (!guest_pv_has(vcpu, KVM_FEATURE_PV_SEND_IPI))
9691 break;
9692
Wanpeng Li4180bf12018-07-23 14:39:54 +08009693 ret = kvm_pv_send_ipi(vcpu->kvm, a0, a1, a2, a3, op_64_bit);
9694 break;
Wanpeng Li71506292019-06-11 20:23:49 +08009695 case KVM_HC_SCHED_YIELD:
Oliver Upton66570e962020-08-18 15:24:28 +00009696 if (!guest_pv_has(vcpu, KVM_FEATURE_PV_SCHED_YIELD))
9697 break;
9698
Wanpeng Li4a7132e2021-04-09 12:18:30 +08009699 kvm_sched_yield(vcpu, a0);
Wanpeng Li71506292019-06-11 20:23:49 +08009700 ret = 0;
9701 break;
Ashish Kalra0dbb1122021-06-08 18:05:43 +00009702 case KVM_HC_MAP_GPA_RANGE: {
9703 u64 gpa = a0, npages = a1, attrs = a2;
9704
9705 ret = -KVM_ENOSYS;
9706 if (!(vcpu->kvm->arch.hypercall_exit_enabled & (1 << KVM_HC_MAP_GPA_RANGE)))
9707 break;
9708
9709 if (!PAGE_ALIGNED(gpa) || !npages ||
9710 gpa_to_gfn(gpa) + npages <= gpa_to_gfn(gpa)) {
9711 ret = -KVM_EINVAL;
9712 break;
9713 }
9714
9715 vcpu->run->exit_reason = KVM_EXIT_HYPERCALL;
9716 vcpu->run->hypercall.nr = KVM_HC_MAP_GPA_RANGE;
9717 vcpu->run->hypercall.args[0] = gpa;
9718 vcpu->run->hypercall.args[1] = npages;
9719 vcpu->run->hypercall.args[2] = attrs;
9720 vcpu->run->hypercall.longmode = op_64_bit;
9721 vcpu->arch.complete_userspace_io = complete_hypercall_exit;
9722 return 0;
9723 }
Hollis Blanchard8776e512007-10-31 17:24:24 -05009724 default:
9725 ret = -KVM_ENOSYS;
9726 break;
9727 }
Radim Krčmář696ca7792018-05-24 17:50:56 +02009728out:
Nadav Amita449c7a2014-06-18 17:19:24 +03009729 if (!op_64_bit)
9730 ret = (u32)ret;
Sean Christophersonde3cd112019-04-30 10:36:17 -07009731 kvm_rax_write(vcpu, ret);
Marian Rotariu6356ee02018-04-30 12:23:01 +03009732
Amit Shahf11c3a82008-02-21 01:00:30 +05309733 ++vcpu->stat.hypercalls;
Marian Rotariu6356ee02018-04-30 12:23:01 +03009734 return kvm_skip_emulated_instruction(vcpu);
Hollis Blanchard8776e512007-10-31 17:24:24 -05009735}
9736EXPORT_SYMBOL_GPL(kvm_emulate_hypercall);
9737
Jan Kiszkab6785de2012-09-20 07:43:17 +02009738static int emulator_fix_hypercall(struct x86_emulate_ctxt *ctxt)
Hollis Blanchard8776e512007-10-31 17:24:24 -05009739{
Avi Kivityd6aa1002011-04-20 15:47:13 +03009740 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
Hollis Blanchard8776e512007-10-31 17:24:24 -05009741 char instruction[3];
Marcelo Tosatti5fdbf972008-06-27 14:58:02 -03009742 unsigned long rip = kvm_rip_read(vcpu);
Hollis Blanchard8776e512007-10-31 17:24:24 -05009743
Oliver Uptonf1a97612022-03-16 00:55:37 +00009744 /*
9745 * If the quirk is disabled, synthesize a #UD and let the guest pick up
9746 * the pieces.
9747 */
9748 if (!kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_FIX_HYPERCALL_INSN)) {
9749 ctxt->exception.error_code_valid = false;
9750 ctxt->exception.vector = UD_VECTOR;
9751 ctxt->have_exception = true;
9752 return X86EMUL_PROPAGATE_FAULT;
9753 }
9754
Jason Baronb36464772021-01-14 22:27:56 -05009755 static_call(kvm_x86_patch_hypercall)(vcpu, instruction);
Hollis Blanchard8776e512007-10-31 17:24:24 -05009756
Dmitry Vyukovce2e8522017-01-17 14:51:04 +01009757 return emulator_write_emulated(ctxt, rip, instruction, 3,
9758 &ctxt->exception);
Hollis Blanchard8776e512007-10-31 17:24:24 -05009759}
9760
Avi Kivity851ba692009-08-24 11:10:17 +03009761static int dm_request_for_irq_injection(struct kvm_vcpu *vcpu)
Hollis Blanchardb6c7a5d2007-11-01 14:16:10 -05009762{
Matt Gingell782d4222015-11-16 15:26:00 -08009763 return vcpu->run->request_interrupt_window &&
9764 likely(!pic_in_kernel(vcpu->kvm));
Hollis Blanchardb6c7a5d2007-11-01 14:16:10 -05009765}
9766
Paolo Bonzini8d25b7b2022-02-19 04:28:20 -05009767/* Called within kvm->srcu read side. */
Avi Kivity851ba692009-08-24 11:10:17 +03009768static void post_kvm_run_save(struct kvm_vcpu *vcpu)
Hollis Blanchardb6c7a5d2007-11-01 14:16:10 -05009769{
Avi Kivity851ba692009-08-24 11:10:17 +03009770 struct kvm_run *kvm_run = vcpu->run;
9771
Marc Orrc5063552021-12-09 07:52:57 -08009772 kvm_run->if_flag = static_call(kvm_x86_get_if_flag)(vcpu);
Avi Kivity2d3ad1f2008-02-24 11:20:43 +02009773 kvm_run->cr8 = kvm_get_cr8(vcpu);
Hollis Blanchardb6c7a5d2007-11-01 14:16:10 -05009774 kvm_run->apic_base = kvm_get_apic_base(vcpu);
David Woodhousef3d14362021-10-26 04:12:38 +01009775
Matt Gingell127a4572015-11-17 17:32:05 +01009776 kvm_run->ready_for_interrupt_injection =
9777 pic_in_kernel(vcpu->kvm) ||
Matt Gingell782d4222015-11-16 15:26:00 -08009778 kvm_vcpu_ready_for_interrupt_injection(vcpu);
Chenyi Qiang15aad3b2020-11-06 17:03:13 +08009779
9780 if (is_smm(vcpu))
9781 kvm_run->flags |= KVM_RUN_X86_SMM;
Hollis Blanchardb6c7a5d2007-11-01 14:16:10 -05009782}
9783
Gleb Natapov95ba8273132009-04-21 17:45:08 +03009784static void update_cr8_intercept(struct kvm_vcpu *vcpu)
9785{
9786 int max_irr, tpr;
9787
Sean Christophersonafaf0b22020-03-21 13:26:00 -07009788 if (!kvm_x86_ops.update_cr8_intercept)
Gleb Natapov95ba8273132009-04-21 17:45:08 +03009789 return;
9790
Paolo Bonzinibce87cc2016-01-08 13:48:51 +01009791 if (!lapic_in_kernel(vcpu))
Avi Kivity88c808f2009-08-17 22:49:40 +03009792 return;
9793
Sean Christophersonce0a58f42022-06-14 23:05:47 +00009794 if (vcpu->arch.apic->apicv_active)
Andrey Smetanind62caab2015-11-10 15:36:33 +03009795 return;
9796
Gleb Natapov8db3baa2009-05-11 13:35:54 +03009797 if (!vcpu->arch.apic->vapic_addr)
9798 max_irr = kvm_lapic_find_highest_irr(vcpu);
9799 else
9800 max_irr = -1;
Gleb Natapov95ba8273132009-04-21 17:45:08 +03009801
9802 if (max_irr != -1)
9803 max_irr >>= 4;
9804
9805 tpr = kvm_lapic_get_cr8(vcpu);
9806
Jason Baronb36464772021-01-14 22:27:56 -05009807 static_call(kvm_x86_update_cr8_intercept)(vcpu, tpr, max_irr);
Gleb Natapov95ba8273132009-04-21 17:45:08 +03009808}
9809
Maxim Levitskyb97f0742021-02-25 17:41:32 +02009810
Sean Christophersoncb6a32c2021-03-02 09:45:14 -08009811int kvm_check_nested_events(struct kvm_vcpu *vcpu)
9812{
Sean Christophersoncb6a32c2021-03-02 09:45:14 -08009813 if (kvm_check_request(KVM_REQ_TRIPLE_FAULT, vcpu)) {
9814 kvm_x86_ops.nested_ops->triple_fault(vcpu);
9815 return 1;
9816 }
9817
9818 return kvm_x86_ops.nested_ops->check_events(vcpu);
9819}
9820
Maxim Levitskyb97f0742021-02-25 17:41:32 +02009821static void kvm_inject_exception(struct kvm_vcpu *vcpu)
9822{
Sean Christophersond4963e32022-08-30 23:16:01 +00009823 trace_kvm_inj_exception(vcpu->arch.exception.vector,
Sean Christophersona61d7c52022-05-02 00:07:31 +02009824 vcpu->arch.exception.has_error_code,
9825 vcpu->arch.exception.error_code,
9826 vcpu->arch.exception.injected);
9827
Maxim Levitskyb97f0742021-02-25 17:41:32 +02009828 if (vcpu->arch.exception.error_code && !is_protmode(vcpu))
9829 vcpu->arch.exception.error_code = false;
Sean Christopherson6ad75c52022-08-30 23:16:00 +00009830 static_call(kvm_x86_inject_exception)(vcpu);
Maxim Levitskyb97f0742021-02-25 17:41:32 +02009831}
9832
Sean Christophersone746c1f2022-08-30 23:16:11 +00009833/*
9834 * Check for any event (interrupt or exception) that is ready to be injected,
9835 * and if there is at least one event, inject the event with the highest
9836 * priority. This handles both "pending" events, i.e. events that have never
9837 * been injected into the guest, and "injected" events, i.e. events that were
9838 * injected as part of a previous VM-Enter, but weren't successfully delivered
9839 * and need to be re-injected.
9840 *
9841 * Note, this is not guaranteed to be invoked on a guest instruction boundary,
9842 * i.e. doesn't guarantee that there's an event window in the guest. KVM must
9843 * be able to inject exceptions in the "middle" of an instruction, and so must
9844 * also be able to re-inject NMIs and IRQs in the middle of an instruction.
9845 * I.e. for exceptions and re-injected events, NOT invoking this on instruction
9846 * boundaries is necessary and correct.
9847 *
9848 * For simplicity, KVM uses a single path to inject all events (except events
9849 * that are injected directly from L1 to L2) and doesn't explicitly track
9850 * instruction boundaries for asynchronous events. However, because VM-Exits
9851 * that can occur during instruction execution typically result in KVM skipping
9852 * the instruction or injecting an exception, e.g. instruction and exception
9853 * intercepts, and because pending exceptions have higher priority than pending
9854 * interrupts, KVM still honors instruction boundaries in most scenarios.
9855 *
9856 * But, if a VM-Exit occurs during instruction execution, and KVM does NOT skip
9857 * the instruction or inject an exception, then KVM can incorrecty inject a new
9858 * asynchrounous event if the event became pending after the CPU fetched the
9859 * instruction (in the guest). E.g. if a page fault (#PF, #NPF, EPT violation)
9860 * occurs and is resolved by KVM, a coincident NMI, SMI, IRQ, etc... can be
9861 * injected on the restarted instruction instead of being deferred until the
9862 * instruction completes.
9863 *
9864 * In practice, this virtualization hole is unlikely to be observed by the
9865 * guest, and even less likely to cause functional problems. To detect the
9866 * hole, the guest would have to trigger an event on a side effect of an early
9867 * phase of instruction execution, e.g. on the instruction fetch from memory.
9868 * And for it to be a functional problem, the guest would need to depend on the
9869 * ordering between that side effect, the instruction completing, _and_ the
9870 * delivery of the asynchronous event.
9871 */
9872static int kvm_check_and_inject_events(struct kvm_vcpu *vcpu,
9873 bool *req_immediate_exit)
Gleb Natapov95ba8273132009-04-21 17:45:08 +03009874{
Sean Christopherson28360f82022-08-30 23:16:05 +00009875 bool can_inject;
Jan Kiszkab6b8a142014-03-07 20:03:12 +01009876 int r;
9877
Wanpeng Li664f8e22017-08-24 03:35:09 -07009878 /*
Sean Christopherson6c593b52022-08-30 23:16:04 +00009879 * Process nested events first, as nested VM-Exit supercedes event
9880 * re-injection. If there's an event queued for re-injection, it will
9881 * be saved into the appropriate vmc{b,s}12 fields on nested VM-Exit.
Wanpeng Li664f8e22017-08-24 03:35:09 -07009882 */
Sean Christopherson6c593b52022-08-30 23:16:04 +00009883 if (is_guest_mode(vcpu))
9884 r = kvm_check_nested_events(vcpu);
9885 else
9886 r = 0;
Wanpeng Li664f8e22017-08-24 03:35:09 -07009887
Sean Christopherson6c593b52022-08-30 23:16:04 +00009888 /*
9889 * Re-inject exceptions and events *especially* if immediate entry+exit
9890 * to/from L2 is needed, as any event that has already been injected
9891 * into L2 needs to complete its lifecycle before injecting a new event.
9892 *
9893 * Don't re-inject an NMI or interrupt if there is a pending exception.
9894 * This collision arises if an exception occurred while vectoring the
9895 * injected event, KVM intercepted said exception, and KVM ultimately
9896 * determined the fault belongs to the guest and queues the exception
9897 * for injection back into the guest.
9898 *
9899 * "Injected" interrupts can also collide with pending exceptions if
9900 * userspace ignores the "ready for injection" flag and blindly queues
9901 * an interrupt. In that case, prioritizing the exception is correct,
9902 * as the exception "occurred" before the exit to userspace. Trap-like
9903 * exceptions, e.g. most #DBs, have higher priority than interrupts.
9904 * And while fault-like exceptions, e.g. #GP and #PF, are the lowest
9905 * priority, they're only generated (pended) during instruction
9906 * execution, and interrupts are recognized at instruction boundaries.
9907 * Thus a pending fault-like exception means the fault occurred on the
9908 * *previous* instruction and must be serviced prior to recognizing any
9909 * new events in order to fully complete the previous instruction.
9910 */
9911 if (vcpu->arch.exception.injected)
9912 kvm_inject_exception(vcpu);
Sean Christopherson7709aba2022-08-30 23:16:08 +00009913 else if (kvm_is_exception_pending(vcpu))
Sean Christopherson6c593b52022-08-30 23:16:04 +00009914 ; /* see above */
9915 else if (vcpu->arch.nmi_injected)
9916 static_call(kvm_x86_inject_nmi)(vcpu);
9917 else if (vcpu->arch.interrupt.injected)
9918 static_call(kvm_x86_inject_irq)(vcpu, true);
9919
9920 /*
9921 * Exceptions that morph to VM-Exits are handled above, and pending
9922 * exceptions on top of injected exceptions that do not VM-Exit should
9923 * either morph to #DF or, sadly, override the injected exception.
9924 */
Sean Christopherson3b82b8d2020-04-22 19:25:47 -07009925 WARN_ON_ONCE(vcpu->arch.exception.injected &&
9926 vcpu->arch.exception.pending);
9927
Liran Alon1a680e32018-03-23 03:01:33 +03009928 /*
Sean Christopherson6c593b52022-08-30 23:16:04 +00009929 * Bail if immediate entry+exit to/from the guest is needed to complete
9930 * nested VM-Enter or event re-injection so that a different pending
9931 * event can be serviced (or if KVM needs to exit to userspace).
9932 *
9933 * Otherwise, continue processing events even if VM-Exit occurred. The
9934 * VM-Exit will have cleared exceptions that were meant for L2, but
9935 * there may now be events that can be injected into L1.
Liran Alon1a680e32018-03-23 03:01:33 +03009936 */
Sean Christopherson6c593b52022-08-30 23:16:04 +00009937 if (r < 0)
9938 goto out;
Wanpeng Li664f8e22017-08-24 03:35:09 -07009939
Sean Christopherson28360f82022-08-30 23:16:05 +00009940 /*
Sean Christopherson7709aba2022-08-30 23:16:08 +00009941 * A pending exception VM-Exit should either result in nested VM-Exit
9942 * or force an immediate re-entry and exit to/from L2, and exception
9943 * VM-Exits cannot be injected (flag should _never_ be set).
9944 */
9945 WARN_ON_ONCE(vcpu->arch.exception_vmexit.injected ||
9946 vcpu->arch.exception_vmexit.pending);
9947
9948 /*
Sean Christopherson28360f82022-08-30 23:16:05 +00009949 * New events, other than exceptions, cannot be injected if KVM needs
9950 * to re-inject a previous event. See above comments on re-injecting
9951 * for why pending exceptions get priority.
9952 */
9953 can_inject = !kvm_event_needs_reinjection(vcpu);
9954
Gleb Natapovb59bb7b2009-07-09 15:33:51 +03009955 if (vcpu->arch.exception.pending) {
Sean Christopherson5623f752022-08-30 23:15:55 +00009956 /*
9957 * Fault-class exceptions, except #DBs, set RF=1 in the RFLAGS
9958 * value pushed on the stack. Trap-like exception and all #DBs
9959 * leave RF as-is (KVM follows Intel's behavior in this regard;
9960 * AMD states that code breakpoint #DBs excplitly clear RF=0).
9961 *
9962 * Note, most versions of Intel's SDM and AMD's APM incorrectly
9963 * describe the behavior of General Detect #DBs, which are
9964 * fault-like. They do _not_ set RF, a la code breakpoints.
9965 */
Sean Christophersond4963e32022-08-30 23:16:01 +00009966 if (exception_type(vcpu->arch.exception.vector) == EXCPT_FAULT)
Nadav Amitd6e8c852014-07-24 14:51:24 +03009967 __kvm_set_rflags(vcpu, kvm_get_rflags(vcpu) |
9968 X86_EFLAGS_RF);
9969
Sean Christophersond4963e32022-08-30 23:16:01 +00009970 if (vcpu->arch.exception.vector == DB_VECTOR) {
9971 kvm_deliver_exception_payload(vcpu, &vcpu->arch.exception);
Jim Mattsonf10c7292018-10-16 14:29:23 -07009972 if (vcpu->arch.dr7 & DR7_GD) {
9973 vcpu->arch.dr7 &= ~DR7_GD;
9974 kvm_update_dr7(vcpu);
9975 }
Nadav Amit6bdf0662014-09-30 20:49:14 +03009976 }
9977
Maxim Levitskyb97f0742021-02-25 17:41:32 +02009978 kvm_inject_exception(vcpu);
Sean Christophersona61d7c52022-05-02 00:07:31 +02009979
9980 vcpu->arch.exception.pending = false;
9981 vcpu->arch.exception.injected = true;
9982
Paolo Bonzinic6b22f52020-05-26 09:05:27 -04009983 can_inject = false;
Liran Alon1a680e32018-03-23 03:01:33 +03009984 }
9985
Maxim Levitsky61e5f692021-08-11 15:29:26 +03009986 /* Don't inject interrupts if the user asked to avoid doing so */
9987 if (vcpu->guest_debug & KVM_GUESTDBG_BLOCKIRQ)
9988 return 0;
9989
Paolo Bonzinic9d40912020-05-22 11:21:49 -04009990 /*
9991 * Finally, inject interrupt events. If an event cannot be injected
9992 * due to architectural conditions (e.g. IF=0) a window-open exit
9993 * will re-request KVM_REQ_EVENT. Sometimes however an event is pending
9994 * and can architecturally be injected, but we cannot do it right now:
9995 * an interrupt could have arrived just now and we have to inject it
9996 * as a vmexit, or there could already an event in the queue, which is
9997 * indicated by can_inject. In that case we request an immediate exit
9998 * in order to make progress and get back here for another iteration.
9999 * The kvm_x86_ops hooks communicate this by returning -EBUSY.
10000 */
10001 if (vcpu->arch.smi_pending) {
Jason Baronb36464772021-01-14 22:27:56 -050010002 r = can_inject ? static_call(kvm_x86_smi_allowed)(vcpu, true) : -EBUSY;
Paolo Bonzinic9d40912020-05-22 11:21:49 -040010003 if (r < 0)
Jim Mattsona5f69092021-06-04 10:26:03 -070010004 goto out;
Paolo Bonzinic9d40912020-05-22 11:21:49 -040010005 if (r) {
10006 vcpu->arch.smi_pending = false;
10007 ++vcpu->arch.smi_count;
10008 enter_smm(vcpu);
10009 can_inject = false;
10010 } else
Jason Baronb36464772021-01-14 22:27:56 -050010011 static_call(kvm_x86_enable_smi_window)(vcpu);
Gleb Natapov95ba8273132009-04-21 17:45:08 +030010012 }
Paolo Bonziniee2cd4b2016-06-01 22:26:01 +020010013
Paolo Bonzinic9d40912020-05-22 11:21:49 -040010014 if (vcpu->arch.nmi_pending) {
Jason Baronb36464772021-01-14 22:27:56 -050010015 r = can_inject ? static_call(kvm_x86_nmi_allowed)(vcpu, true) : -EBUSY;
Paolo Bonzinic9d40912020-05-22 11:21:49 -040010016 if (r < 0)
Jim Mattsona5f69092021-06-04 10:26:03 -070010017 goto out;
Paolo Bonzinic9d40912020-05-22 11:21:49 -040010018 if (r) {
10019 --vcpu->arch.nmi_pending;
10020 vcpu->arch.nmi_injected = true;
Sean Christophersone27bc042022-01-28 00:51:50 +000010021 static_call(kvm_x86_inject_nmi)(vcpu);
Paolo Bonzinic9d40912020-05-22 11:21:49 -040010022 can_inject = false;
Jason Baronb36464772021-01-14 22:27:56 -050010023 WARN_ON(static_call(kvm_x86_nmi_allowed)(vcpu, true) < 0);
Paolo Bonzinic9d40912020-05-22 11:21:49 -040010024 }
10025 if (vcpu->arch.nmi_pending)
Jason Baronb36464772021-01-14 22:27:56 -050010026 static_call(kvm_x86_enable_nmi_window)(vcpu);
Paolo Bonzinic9d40912020-05-22 11:21:49 -040010027 }
10028
10029 if (kvm_cpu_has_injectable_intr(vcpu)) {
Jason Baronb36464772021-01-14 22:27:56 -050010030 r = can_inject ? static_call(kvm_x86_interrupt_allowed)(vcpu, true) : -EBUSY;
Paolo Bonzinic9d40912020-05-22 11:21:49 -040010031 if (r < 0)
Jim Mattsona5f69092021-06-04 10:26:03 -070010032 goto out;
Paolo Bonzinic9d40912020-05-22 11:21:49 -040010033 if (r) {
10034 kvm_queue_interrupt(vcpu, kvm_cpu_get_interrupt(vcpu), false);
Sean Christopherson2d613912022-05-02 00:07:33 +020010035 static_call(kvm_x86_inject_irq)(vcpu, false);
Jason Baronb36464772021-01-14 22:27:56 -050010036 WARN_ON(static_call(kvm_x86_interrupt_allowed)(vcpu, true) < 0);
Paolo Bonzinic9d40912020-05-22 11:21:49 -040010037 }
10038 if (kvm_cpu_has_injectable_intr(vcpu))
Jason Baronb36464772021-01-14 22:27:56 -050010039 static_call(kvm_x86_enable_irq_window)(vcpu);
Paolo Bonzinic9d40912020-05-22 11:21:49 -040010040 }
10041
10042 if (is_guest_mode(vcpu) &&
Paolo Bonzini5b4ac1a2022-09-21 00:31:50 +000010043 kvm_x86_ops.nested_ops->has_events &&
10044 kvm_x86_ops.nested_ops->has_events(vcpu))
Paolo Bonzinic9d40912020-05-22 11:21:49 -040010045 *req_immediate_exit = true;
10046
Sean Christophersondea0d5a2022-09-30 23:00:08 +000010047 /*
10048 * KVM must never queue a new exception while injecting an event; KVM
10049 * is done emulating and should only propagate the to-be-injected event
10050 * to the VMCS/VMCB. Queueing a new exception can put the vCPU into an
10051 * infinite loop as KVM will bail from VM-Enter to inject the pending
10052 * exception and start the cycle all over.
10053 *
10054 * Exempt triple faults as they have special handling and won't put the
10055 * vCPU into an infinite loop. Triple fault can be queued when running
10056 * VMX without unrestricted guest, as that requires KVM to emulate Real
10057 * Mode events (see kvm_inject_realmode_interrupt()).
10058 */
10059 WARN_ON_ONCE(vcpu->arch.exception.pending ||
10060 vcpu->arch.exception_vmexit.pending);
Jim Mattsona5f69092021-06-04 10:26:03 -070010061 return 0;
Paolo Bonzinic9d40912020-05-22 11:21:49 -040010062
Jim Mattsona5f69092021-06-04 10:26:03 -070010063out:
10064 if (r == -EBUSY) {
10065 *req_immediate_exit = true;
10066 r = 0;
10067 }
10068 return r;
Gleb Natapov95ba8273132009-04-21 17:45:08 +030010069}
10070
Avi Kivity7460fb4a2011-09-20 13:43:14 +030010071static void process_nmi(struct kvm_vcpu *vcpu)
10072{
10073 unsigned limit = 2;
10074
10075 /*
10076 * x86 is limited to one NMI running, and one NMI pending after it.
10077 * If an NMI is already in progress, limit further NMIs to just one.
10078 * Otherwise, allow two (and we'll inject the first one immediately).
10079 */
Jason Baronb36464772021-01-14 22:27:56 -050010080 if (static_call(kvm_x86_get_nmi_mask)(vcpu) || vcpu->arch.nmi_injected)
Avi Kivity7460fb4a2011-09-20 13:43:14 +030010081 limit = 1;
10082
10083 vcpu->arch.nmi_pending += atomic_xchg(&vcpu->arch.nmi_queued, 0);
10084 vcpu->arch.nmi_pending = min(vcpu->arch.nmi_pending, limit);
10085 kvm_make_request(KVM_REQ_EVENT, vcpu);
10086}
10087
Paolo Bonziniee2cd4b2016-06-01 22:26:01 +020010088static u32 enter_smm_get_segment_flags(struct kvm_segment *seg)
Paolo Bonzini660a5d52015-05-05 11:50:23 +020010089{
10090 u32 flags = 0;
10091 flags |= seg->g << 23;
10092 flags |= seg->db << 22;
10093 flags |= seg->l << 21;
10094 flags |= seg->avl << 20;
10095 flags |= seg->present << 15;
10096 flags |= seg->dpl << 13;
10097 flags |= seg->s << 12;
10098 flags |= seg->type << 8;
10099 return flags;
10100}
10101
Paolo Bonziniee2cd4b2016-06-01 22:26:01 +020010102static void enter_smm_save_seg_32(struct kvm_vcpu *vcpu, char *buf, int n)
Paolo Bonzini660a5d52015-05-05 11:50:23 +020010103{
10104 struct kvm_segment seg;
10105 int offset;
10106
10107 kvm_get_segment(vcpu, &seg, n);
10108 put_smstate(u32, buf, 0x7fa8 + n * 4, seg.selector);
10109
10110 if (n < 3)
10111 offset = 0x7f84 + n * 12;
10112 else
10113 offset = 0x7f2c + (n - 3) * 12;
10114
10115 put_smstate(u32, buf, offset + 8, seg.base);
10116 put_smstate(u32, buf, offset + 4, seg.limit);
Paolo Bonziniee2cd4b2016-06-01 22:26:01 +020010117 put_smstate(u32, buf, offset, enter_smm_get_segment_flags(&seg));
Paolo Bonzini660a5d52015-05-05 11:50:23 +020010118}
10119
Alexander Kuleshovefbb2882015-09-06 19:35:41 +060010120#ifdef CONFIG_X86_64
Paolo Bonziniee2cd4b2016-06-01 22:26:01 +020010121static void enter_smm_save_seg_64(struct kvm_vcpu *vcpu, char *buf, int n)
Paolo Bonzini660a5d52015-05-05 11:50:23 +020010122{
10123 struct kvm_segment seg;
10124 int offset;
10125 u16 flags;
10126
10127 kvm_get_segment(vcpu, &seg, n);
10128 offset = 0x7e00 + n * 16;
10129
Paolo Bonziniee2cd4b2016-06-01 22:26:01 +020010130 flags = enter_smm_get_segment_flags(&seg) >> 8;
Paolo Bonzini660a5d52015-05-05 11:50:23 +020010131 put_smstate(u16, buf, offset, seg.selector);
10132 put_smstate(u16, buf, offset + 2, flags);
10133 put_smstate(u32, buf, offset + 4, seg.limit);
10134 put_smstate(u64, buf, offset + 8, seg.base);
10135}
Alexander Kuleshovefbb2882015-09-06 19:35:41 +060010136#endif
Paolo Bonzini660a5d52015-05-05 11:50:23 +020010137
Paolo Bonziniee2cd4b2016-06-01 22:26:01 +020010138static void enter_smm_save_state_32(struct kvm_vcpu *vcpu, char *buf)
Paolo Bonzini660a5d52015-05-05 11:50:23 +020010139{
10140 struct desc_ptr dt;
10141 struct kvm_segment seg;
10142 unsigned long val;
10143 int i;
10144
10145 put_smstate(u32, buf, 0x7ffc, kvm_read_cr0(vcpu));
10146 put_smstate(u32, buf, 0x7ff8, kvm_read_cr3(vcpu));
10147 put_smstate(u32, buf, 0x7ff4, kvm_get_rflags(vcpu));
10148 put_smstate(u32, buf, 0x7ff0, kvm_rip_read(vcpu));
10149
10150 for (i = 0; i < 8; i++)
Sean Christopherson27b4a9c42021-04-21 19:21:28 -070010151 put_smstate(u32, buf, 0x7fd0 + i * 4, kvm_register_read_raw(vcpu, i));
Paolo Bonzini660a5d52015-05-05 11:50:23 +020010152
10153 kvm_get_dr(vcpu, 6, &val);
10154 put_smstate(u32, buf, 0x7fcc, (u32)val);
10155 kvm_get_dr(vcpu, 7, &val);
10156 put_smstate(u32, buf, 0x7fc8, (u32)val);
10157
10158 kvm_get_segment(vcpu, &seg, VCPU_SREG_TR);
10159 put_smstate(u32, buf, 0x7fc4, seg.selector);
10160 put_smstate(u32, buf, 0x7f64, seg.base);
10161 put_smstate(u32, buf, 0x7f60, seg.limit);
Paolo Bonziniee2cd4b2016-06-01 22:26:01 +020010162 put_smstate(u32, buf, 0x7f5c, enter_smm_get_segment_flags(&seg));
Paolo Bonzini660a5d52015-05-05 11:50:23 +020010163
10164 kvm_get_segment(vcpu, &seg, VCPU_SREG_LDTR);
10165 put_smstate(u32, buf, 0x7fc0, seg.selector);
10166 put_smstate(u32, buf, 0x7f80, seg.base);
10167 put_smstate(u32, buf, 0x7f7c, seg.limit);
Paolo Bonziniee2cd4b2016-06-01 22:26:01 +020010168 put_smstate(u32, buf, 0x7f78, enter_smm_get_segment_flags(&seg));
Paolo Bonzini660a5d52015-05-05 11:50:23 +020010169
Jason Baronb36464772021-01-14 22:27:56 -050010170 static_call(kvm_x86_get_gdt)(vcpu, &dt);
Paolo Bonzini660a5d52015-05-05 11:50:23 +020010171 put_smstate(u32, buf, 0x7f74, dt.address);
10172 put_smstate(u32, buf, 0x7f70, dt.size);
10173
Jason Baronb36464772021-01-14 22:27:56 -050010174 static_call(kvm_x86_get_idt)(vcpu, &dt);
Paolo Bonzini660a5d52015-05-05 11:50:23 +020010175 put_smstate(u32, buf, 0x7f58, dt.address);
10176 put_smstate(u32, buf, 0x7f54, dt.size);
10177
10178 for (i = 0; i < 6; i++)
Paolo Bonziniee2cd4b2016-06-01 22:26:01 +020010179 enter_smm_save_seg_32(vcpu, buf, i);
Paolo Bonzini660a5d52015-05-05 11:50:23 +020010180
10181 put_smstate(u32, buf, 0x7f14, kvm_read_cr4(vcpu));
10182
10183 /* revision id */
10184 put_smstate(u32, buf, 0x7efc, 0x00020000);
10185 put_smstate(u32, buf, 0x7ef8, vcpu->arch.smbase);
10186}
10187
Sean Christophersonb68f3cc2019-04-02 08:10:48 -070010188#ifdef CONFIG_X86_64
Paolo Bonziniee2cd4b2016-06-01 22:26:01 +020010189static void enter_smm_save_state_64(struct kvm_vcpu *vcpu, char *buf)
Paolo Bonzini660a5d52015-05-05 11:50:23 +020010190{
Paolo Bonzini660a5d52015-05-05 11:50:23 +020010191 struct desc_ptr dt;
10192 struct kvm_segment seg;
10193 unsigned long val;
10194 int i;
10195
10196 for (i = 0; i < 16; i++)
Sean Christopherson27b4a9c42021-04-21 19:21:28 -070010197 put_smstate(u64, buf, 0x7ff8 - i * 8, kvm_register_read_raw(vcpu, i));
Paolo Bonzini660a5d52015-05-05 11:50:23 +020010198
10199 put_smstate(u64, buf, 0x7f78, kvm_rip_read(vcpu));
10200 put_smstate(u32, buf, 0x7f70, kvm_get_rflags(vcpu));
10201
10202 kvm_get_dr(vcpu, 6, &val);
10203 put_smstate(u64, buf, 0x7f68, val);
10204 kvm_get_dr(vcpu, 7, &val);
10205 put_smstate(u64, buf, 0x7f60, val);
10206
10207 put_smstate(u64, buf, 0x7f58, kvm_read_cr0(vcpu));
10208 put_smstate(u64, buf, 0x7f50, kvm_read_cr3(vcpu));
10209 put_smstate(u64, buf, 0x7f48, kvm_read_cr4(vcpu));
10210
10211 put_smstate(u32, buf, 0x7f00, vcpu->arch.smbase);
10212
10213 /* revision id */
10214 put_smstate(u32, buf, 0x7efc, 0x00020064);
10215
10216 put_smstate(u64, buf, 0x7ed0, vcpu->arch.efer);
10217
10218 kvm_get_segment(vcpu, &seg, VCPU_SREG_TR);
10219 put_smstate(u16, buf, 0x7e90, seg.selector);
Paolo Bonziniee2cd4b2016-06-01 22:26:01 +020010220 put_smstate(u16, buf, 0x7e92, enter_smm_get_segment_flags(&seg) >> 8);
Paolo Bonzini660a5d52015-05-05 11:50:23 +020010221 put_smstate(u32, buf, 0x7e94, seg.limit);
10222 put_smstate(u64, buf, 0x7e98, seg.base);
10223
Jason Baronb36464772021-01-14 22:27:56 -050010224 static_call(kvm_x86_get_idt)(vcpu, &dt);
Paolo Bonzini660a5d52015-05-05 11:50:23 +020010225 put_smstate(u32, buf, 0x7e84, dt.size);
10226 put_smstate(u64, buf, 0x7e88, dt.address);
10227
10228 kvm_get_segment(vcpu, &seg, VCPU_SREG_LDTR);
10229 put_smstate(u16, buf, 0x7e70, seg.selector);
Paolo Bonziniee2cd4b2016-06-01 22:26:01 +020010230 put_smstate(u16, buf, 0x7e72, enter_smm_get_segment_flags(&seg) >> 8);
Paolo Bonzini660a5d52015-05-05 11:50:23 +020010231 put_smstate(u32, buf, 0x7e74, seg.limit);
10232 put_smstate(u64, buf, 0x7e78, seg.base);
10233
Jason Baronb36464772021-01-14 22:27:56 -050010234 static_call(kvm_x86_get_gdt)(vcpu, &dt);
Paolo Bonzini660a5d52015-05-05 11:50:23 +020010235 put_smstate(u32, buf, 0x7e64, dt.size);
10236 put_smstate(u64, buf, 0x7e68, dt.address);
10237
10238 for (i = 0; i < 6; i++)
Paolo Bonziniee2cd4b2016-06-01 22:26:01 +020010239 enter_smm_save_seg_64(vcpu, buf, i);
Paolo Bonzini660a5d52015-05-05 11:50:23 +020010240}
Sean Christophersonb68f3cc2019-04-02 08:10:48 -070010241#endif
Paolo Bonzini660a5d52015-05-05 11:50:23 +020010242
Paolo Bonziniee2cd4b2016-06-01 22:26:01 +020010243static void enter_smm(struct kvm_vcpu *vcpu)
Paolo Bonzini64d60672015-05-07 11:36:11 +020010244{
Paolo Bonzini660a5d52015-05-05 11:50:23 +020010245 struct kvm_segment cs, ds;
Paolo Bonzini18c36262015-08-07 12:27:54 +020010246 struct desc_ptr dt;
Sean Christophersondbc47392021-06-22 10:56:59 -070010247 unsigned long cr0;
Paolo Bonzini660a5d52015-05-05 11:50:23 +020010248 char buf[512];
Paolo Bonzini660a5d52015-05-05 11:50:23 +020010249
Paolo Bonzini660a5d52015-05-05 11:50:23 +020010250 memset(buf, 0, 512);
Sean Christophersonb68f3cc2019-04-02 08:10:48 -070010251#ifdef CONFIG_X86_64
Radim Krčmářd6321d42017-08-05 00:12:49 +020010252 if (guest_cpuid_has(vcpu, X86_FEATURE_LM))
Paolo Bonziniee2cd4b2016-06-01 22:26:01 +020010253 enter_smm_save_state_64(vcpu, buf);
Paolo Bonzini660a5d52015-05-05 11:50:23 +020010254 else
Sean Christophersonb68f3cc2019-04-02 08:10:48 -070010255#endif
Paolo Bonziniee2cd4b2016-06-01 22:26:01 +020010256 enter_smm_save_state_32(vcpu, buf);
Paolo Bonzini660a5d52015-05-05 11:50:23 +020010257
Ladi Prosek0234bf82017-10-11 16:54:40 +020010258 /*
Sean Christophersonecc513e52021-06-09 11:56:19 -070010259 * Give enter_smm() a chance to make ISA-specific changes to the vCPU
10260 * state (e.g. leave guest mode) after we've saved the state into the
10261 * SMM state-save area.
Ladi Prosek0234bf82017-10-11 16:54:40 +020010262 */
Sean Christophersonecc513e52021-06-09 11:56:19 -070010263 static_call(kvm_x86_enter_smm)(vcpu, buf);
Ladi Prosek0234bf82017-10-11 16:54:40 +020010264
Sean Christophersondc872752021-06-09 11:56:15 -070010265 kvm_smm_changed(vcpu, true);
Paolo Bonzini54bf36a2015-04-08 15:39:23 +020010266 kvm_vcpu_write_guest(vcpu, vcpu->arch.smbase + 0xfe00, buf, sizeof(buf));
Paolo Bonzini660a5d52015-05-05 11:50:23 +020010267
Jason Baronb36464772021-01-14 22:27:56 -050010268 if (static_call(kvm_x86_get_nmi_mask)(vcpu))
Paolo Bonzini660a5d52015-05-05 11:50:23 +020010269 vcpu->arch.hflags |= HF_SMM_INSIDE_NMI_MASK;
10270 else
Jason Baronb36464772021-01-14 22:27:56 -050010271 static_call(kvm_x86_set_nmi_mask)(vcpu, true);
Paolo Bonzini660a5d52015-05-05 11:50:23 +020010272
10273 kvm_set_rflags(vcpu, X86_EFLAGS_FIXED);
10274 kvm_rip_write(vcpu, 0x8000);
10275
10276 cr0 = vcpu->arch.cr0 & ~(X86_CR0_PE | X86_CR0_EM | X86_CR0_TS | X86_CR0_PG);
Jason Baronb36464772021-01-14 22:27:56 -050010277 static_call(kvm_x86_set_cr0)(vcpu, cr0);
Paolo Bonzini660a5d52015-05-05 11:50:23 +020010278 vcpu->arch.cr0 = cr0;
10279
Jason Baronb36464772021-01-14 22:27:56 -050010280 static_call(kvm_x86_set_cr4)(vcpu, 0);
Paolo Bonzini660a5d52015-05-05 11:50:23 +020010281
Paolo Bonzini18c36262015-08-07 12:27:54 +020010282 /* Undocumented: IDT limit is set to zero on entry to SMM. */
10283 dt.address = dt.size = 0;
Jason Baronb36464772021-01-14 22:27:56 -050010284 static_call(kvm_x86_set_idt)(vcpu, &dt);
Paolo Bonzini18c36262015-08-07 12:27:54 +020010285
Paolo Bonzini996ff542020-12-14 07:49:54 -050010286 kvm_set_dr(vcpu, 7, DR7_FIXED_1);
Paolo Bonzini660a5d52015-05-05 11:50:23 +020010287
10288 cs.selector = (vcpu->arch.smbase >> 4) & 0xffff;
10289 cs.base = vcpu->arch.smbase;
10290
10291 ds.selector = 0;
10292 ds.base = 0;
10293
10294 cs.limit = ds.limit = 0xffffffff;
10295 cs.type = ds.type = 0x3;
10296 cs.dpl = ds.dpl = 0;
10297 cs.db = ds.db = 0;
10298 cs.s = ds.s = 1;
10299 cs.l = ds.l = 0;
10300 cs.g = ds.g = 1;
10301 cs.avl = ds.avl = 0;
10302 cs.present = ds.present = 1;
10303 cs.unusable = ds.unusable = 0;
10304 cs.padding = ds.padding = 0;
10305
10306 kvm_set_segment(vcpu, &cs, VCPU_SREG_CS);
10307 kvm_set_segment(vcpu, &ds, VCPU_SREG_DS);
10308 kvm_set_segment(vcpu, &ds, VCPU_SREG_ES);
10309 kvm_set_segment(vcpu, &ds, VCPU_SREG_FS);
10310 kvm_set_segment(vcpu, &ds, VCPU_SREG_GS);
10311 kvm_set_segment(vcpu, &ds, VCPU_SREG_SS);
10312
Sean Christophersonb68f3cc2019-04-02 08:10:48 -070010313#ifdef CONFIG_X86_64
Radim Krčmářd6321d42017-08-05 00:12:49 +020010314 if (guest_cpuid_has(vcpu, X86_FEATURE_LM))
Jason Baronb36464772021-01-14 22:27:56 -050010315 static_call(kvm_x86_set_efer)(vcpu, 0);
Sean Christophersonb68f3cc2019-04-02 08:10:48 -070010316#endif
Paolo Bonzini660a5d52015-05-05 11:50:23 +020010317
Xiaoyao Liaedbaf42020-07-09 12:34:23 +080010318 kvm_update_cpuid_runtime(vcpu);
Paolo Bonzini660a5d52015-05-05 11:50:23 +020010319 kvm_mmu_reset_context(vcpu);
Paolo Bonzini64d60672015-05-07 11:36:11 +020010320}
10321
Paolo Bonziniee2cd4b2016-06-01 22:26:01 +020010322static void process_smi(struct kvm_vcpu *vcpu)
Paolo Bonzinic43203c2016-06-01 22:26:00 +020010323{
10324 vcpu->arch.smi_pending = true;
10325 kvm_make_request(KVM_REQ_EVENT, vcpu);
10326}
10327
Nitesh Narayan Lal7ee30bc2019-11-07 07:53:43 -050010328void kvm_make_scan_ioapic_request_mask(struct kvm *kvm,
10329 unsigned long *vcpu_bitmap)
10330{
Vitaly Kuznetsov620b2432021-09-03 09:51:41 +020010331 kvm_make_vcpus_request_mask(kvm, KVM_REQ_SCAN_IOAPIC, vcpu_bitmap);
Nitesh Narayan Lal7ee30bc2019-11-07 07:53:43 -050010332}
10333
Paolo Bonzini2860c4b2016-01-07 15:05:10 +010010334void kvm_make_scan_ioapic_request(struct kvm *kvm)
10335{
10336 kvm_make_all_cpus_request(kvm, KVM_REQ_SCAN_IOAPIC);
10337}
10338
Suravee Suthikulpanit8df14af2019-11-14 14:15:06 -060010339void kvm_vcpu_update_apicv(struct kvm_vcpu *vcpu)
10340{
Sean Christophersonce0a58f42022-06-14 23:05:47 +000010341 struct kvm_lapic *apic = vcpu->arch.apic;
Maxim Levitsky06ef8132021-08-10 23:52:48 +030010342 bool activate;
10343
Suravee Suthikulpanit8df14af2019-11-14 14:15:06 -060010344 if (!lapic_in_kernel(vcpu))
10345 return;
10346
Sean Christopherson187c8832021-10-21 17:49:27 -070010347 down_read(&vcpu->kvm->arch.apicv_update_lock);
Maxim Levitsky66c768d2022-06-06 21:08:27 +030010348 preempt_disable();
Maxim Levitskyb0a16372021-08-10 23:52:44 +030010349
Suravee Suthikulpanit8fc9c7a2022-05-19 05:27:00 -050010350 /* Do not activate APICV when APIC is disabled */
10351 activate = kvm_vcpu_apicv_activated(vcpu) &&
10352 (kvm_get_apic_mode(vcpu) != LAPIC_MODE_DISABLED);
Maxim Levitskyd5fa5972022-03-22 19:40:49 +020010353
Sean Christophersonce0a58f42022-06-14 23:05:47 +000010354 if (apic->apicv_active == activate)
Maxim Levitsky06ef8132021-08-10 23:52:48 +030010355 goto out;
10356
Sean Christophersonce0a58f42022-06-14 23:05:47 +000010357 apic->apicv_active = activate;
Suravee Suthikulpanit8df14af2019-11-14 14:15:06 -060010358 kvm_apic_update_apicv(vcpu);
Jason Baronb36464772021-01-14 22:27:56 -050010359 static_call(kvm_x86_refresh_apicv_exec_ctrl)(vcpu);
Vitaly Kuznetsovbca66db2021-06-09 17:09:10 +020010360
10361 /*
10362 * When APICv gets disabled, we may still have injected interrupts
10363 * pending. At the same time, KVM_REQ_EVENT may not be set as APICv was
10364 * still active when the interrupt got accepted. Make sure
Sean Christophersone746c1f2022-08-30 23:16:11 +000010365 * kvm_check_and_inject_events() is called to check for that.
Vitaly Kuznetsovbca66db2021-06-09 17:09:10 +020010366 */
Sean Christophersonce0a58f42022-06-14 23:05:47 +000010367 if (!apic->apicv_active)
Vitaly Kuznetsovbca66db2021-06-09 17:09:10 +020010368 kvm_make_request(KVM_REQ_EVENT, vcpu);
Maxim Levitskyb0a16372021-08-10 23:52:44 +030010369
Maxim Levitsky06ef8132021-08-10 23:52:48 +030010370out:
Maxim Levitsky66c768d2022-06-06 21:08:27 +030010371 preempt_enable();
Sean Christopherson187c8832021-10-21 17:49:27 -070010372 up_read(&vcpu->kvm->arch.apicv_update_lock);
Suravee Suthikulpanit8df14af2019-11-14 14:15:06 -060010373}
10374EXPORT_SYMBOL_GPL(kvm_vcpu_update_apicv);
10375
Sean Christopherson320af552022-03-11 04:35:16 +000010376void __kvm_set_or_clear_apicv_inhibit(struct kvm *kvm,
10377 enum kvm_apicv_inhibit reason, bool set)
Suravee Suthikulpanit8df14af2019-11-14 14:15:06 -060010378{
Maxim Levitskyb0a16372021-08-10 23:52:44 +030010379 unsigned long old, new;
Paolo Bonzini8e205a62020-03-14 12:29:23 +010010380
Sean Christopherson187c8832021-10-21 17:49:27 -070010381 lockdep_assert_held_write(&kvm->arch.apicv_update_lock);
10382
Sean Christopherson7491b7b2022-03-11 04:35:15 +000010383 if (!static_call(kvm_x86_check_apicv_inhibit_reasons)(reason))
Suravee Suthikulpanitef8efd72019-11-14 14:15:10 -060010384 return;
10385
Maxim Levitskyb0a16372021-08-10 23:52:44 +030010386 old = new = kvm->arch.apicv_inhibit_reasons;
10387
Sean Christopherson4f4c4a32022-03-11 04:35:17 +000010388 set_or_clear_apicv_inhibit(&new, reason, set);
Paolo Bonzini8e205a62020-03-14 12:29:23 +010010389
Maxim Levitsky36222b12021-08-10 23:52:43 +030010390 if (!!old != !!new) {
Sean Christophersonee49a892021-10-21 17:49:25 -070010391 /*
10392 * Kick all vCPUs before setting apicv_inhibit_reasons to avoid
10393 * false positives in the sanity check WARN in svm_vcpu_run().
10394 * This task will wait for all vCPUs to ack the kick IRQ before
10395 * updating apicv_inhibit_reasons, and all other vCPUs will
10396 * block on acquiring apicv_update_lock so that vCPUs can't
10397 * redo svm_vcpu_run() without seeing the new inhibit state.
10398 *
10399 * Note, holding apicv_update_lock and taking it in the read
10400 * side (handling the request) also prevents other vCPUs from
10401 * servicing the request with a stale apicv_inhibit_reasons.
10402 */
Maxim Levitsky36222b12021-08-10 23:52:43 +030010403 kvm_make_all_cpus_request(kvm, KVM_REQ_APICV_UPDATE);
Maxim Levitskyb0a16372021-08-10 23:52:44 +030010404 kvm->arch.apicv_inhibit_reasons = new;
Maxim Levitsky36222b12021-08-10 23:52:43 +030010405 if (new) {
10406 unsigned long gfn = gpa_to_gfn(APIC_DEFAULT_PHYS_BASE);
Ben Gardon074c0082022-11-02 20:53:59 +000010407 int idx = srcu_read_lock(&kvm->srcu);
10408
Maxim Levitsky36222b12021-08-10 23:52:43 +030010409 kvm_zap_gfn_range(kvm, gfn, gfn+1);
Ben Gardon074c0082022-11-02 20:53:59 +000010410 srcu_read_unlock(&kvm->srcu, idx);
Maxim Levitsky36222b12021-08-10 23:52:43 +030010411 }
Sean Christopherson7491b7b2022-03-11 04:35:15 +000010412 } else {
Maxim Levitskyb0a16372021-08-10 23:52:44 +030010413 kvm->arch.apicv_inhibit_reasons = new;
Sean Christopherson7491b7b2022-03-11 04:35:15 +000010414 }
Maxim Levitskyb0a16372021-08-10 23:52:44 +030010415}
Suravee Suthikulpanit7d611232020-05-06 21:35:39 -050010416
Sean Christopherson320af552022-03-11 04:35:16 +000010417void kvm_set_or_clear_apicv_inhibit(struct kvm *kvm,
10418 enum kvm_apicv_inhibit reason, bool set)
Maxim Levitskyb0a16372021-08-10 23:52:44 +030010419{
Sean Christophersonf1575642021-12-08 01:52:34 +000010420 if (!enable_apicv)
10421 return;
10422
Sean Christopherson187c8832021-10-21 17:49:27 -070010423 down_write(&kvm->arch.apicv_update_lock);
Sean Christopherson320af552022-03-11 04:35:16 +000010424 __kvm_set_or_clear_apicv_inhibit(kvm, reason, set);
Sean Christopherson187c8832021-10-21 17:49:27 -070010425 up_write(&kvm->arch.apicv_update_lock);
Suravee Suthikulpanit8df14af2019-11-14 14:15:06 -060010426}
Sean Christopherson320af552022-03-11 04:35:16 +000010427EXPORT_SYMBOL_GPL(kvm_set_or_clear_apicv_inhibit);
Suravee Suthikulpanit8df14af2019-11-14 14:15:06 -060010428
Yang Zhang3d81bc72013-04-11 19:25:13 +080010429static void vcpu_scan_ioapic(struct kvm_vcpu *vcpu)
Yang Zhangc7c9c562013-01-25 10:18:51 +080010430{
Wanpeng Lidcbd3e42018-12-17 10:43:23 +080010431 if (!kvm_apic_present(vcpu))
Yang Zhang3d81bc72013-04-11 19:25:13 +080010432 return;
Yang Zhangc7c9c562013-01-25 10:18:51 +080010433
Andrey Smetanin63086302015-11-10 15:36:32 +030010434 bitmap_zero(vcpu->arch.ioapic_handled_vectors, 256);
Yang Zhangc7c9c562013-01-25 10:18:51 +080010435
Steve Rutherfordb053b2a2015-07-29 23:32:35 -070010436 if (irqchip_split(vcpu->kvm))
Andrey Smetanin63086302015-11-10 15:36:32 +030010437 kvm_scan_ioapic_routes(vcpu, vcpu->arch.ioapic_handled_vectors);
Radim Krčmářdb2bdcb2015-10-08 20:23:34 +020010438 else {
Paolo Bonzini37c4dbf2021-11-22 19:43:10 -050010439 static_call_cond(kvm_x86_sync_pir_to_irr)(vcpu);
Wanpeng Lie97f8522018-11-20 16:34:18 +080010440 if (ioapic_in_kernel(vcpu->kvm))
10441 kvm_ioapic_scan_entry(vcpu, vcpu->arch.ioapic_handled_vectors);
Radim Krčmářdb2bdcb2015-10-08 20:23:34 +020010442 }
Liran Alone40ff1d2018-03-21 02:50:31 +020010443
10444 if (is_guest_mode(vcpu))
10445 vcpu->arch.load_eoi_exitmap_pending = true;
10446 else
10447 kvm_make_request(KVM_REQ_LOAD_EOI_EXITMAP, vcpu);
10448}
10449
10450static void vcpu_load_eoi_exitmap(struct kvm_vcpu *vcpu)
10451{
10452 u64 eoi_exit_bitmap[4];
10453
10454 if (!kvm_apic_hw_enabled(vcpu->arch.apic))
10455 return;
10456
黄乐c5adbb3a2021-11-15 14:08:29 +000010457 if (to_hv_vcpu(vcpu)) {
Vitaly Kuznetsovf2bc14b2021-01-26 14:48:12 +010010458 bitmap_or((ulong *)eoi_exit_bitmap,
10459 vcpu->arch.ioapic_handled_vectors,
10460 to_hv_synic(vcpu)->vec_bitmap, 256);
Paolo Bonziniabb6d472022-02-08 13:08:19 -050010461 static_call_cond(kvm_x86_load_eoi_exitmap)(vcpu, eoi_exit_bitmap);
黄乐c5adbb3a2021-11-15 14:08:29 +000010462 return;
10463 }
Vitaly Kuznetsovf2bc14b2021-01-26 14:48:12 +010010464
Paolo Bonziniabb6d472022-02-08 13:08:19 -050010465 static_call_cond(kvm_x86_load_eoi_exitmap)(
黄乐c5adbb3a2021-11-15 14:08:29 +000010466 vcpu, (u64 *)vcpu->arch.ioapic_handled_vectors);
Yang Zhangc7c9c562013-01-25 10:18:51 +080010467}
10468
Eiichi Tsukatae649b3f2020-06-06 13:26:27 +090010469void kvm_arch_mmu_notifier_invalidate_range(struct kvm *kvm,
10470 unsigned long start, unsigned long end)
Radim Krčmářb1394e72017-11-30 19:05:45 +010010471{
10472 unsigned long apic_address;
10473
10474 /*
10475 * The physical address of apic access page is stored in the VMCS.
10476 * Update it when it becomes invalid.
10477 */
10478 apic_address = gfn_to_hva(kvm, APIC_DEFAULT_PHYS_BASE >> PAGE_SHIFT);
10479 if (start <= apic_address && apic_address < end)
10480 kvm_make_all_cpus_request(kvm, KVM_REQ_APIC_PAGE_RELOAD);
10481}
10482
Mingwei Zhang683412c2022-04-21 03:14:07 +000010483void kvm_arch_guest_memory_reclaimed(struct kvm *kvm)
10484{
10485 static_call_cond(kvm_x86_guest_memory_reclaimed)(kvm);
10486}
10487
Quanfa Fud081a342021-12-19 17:14:46 +080010488static void kvm_vcpu_reload_apic_access_page(struct kvm_vcpu *vcpu)
Tang Chen4256f432014-09-24 15:57:54 +080010489{
Paolo Bonzini35754c92015-07-29 12:05:37 +020010490 if (!lapic_in_kernel(vcpu))
Paolo Bonzinif439ed22014-10-02 13:53:24 +020010491 return;
10492
Paolo Bonzini2a890612022-02-01 14:18:07 -050010493 static_call_cond(kvm_x86_set_apic_access_page_addr)(vcpu);
Tang Chen4256f432014-09-24 15:57:54 +080010494}
Tang Chen4256f432014-09-24 15:57:54 +080010495
Sean Christophersond264ee02018-08-27 15:21:12 -070010496void __kvm_request_immediate_exit(struct kvm_vcpu *vcpu)
10497{
10498 smp_send_reschedule(vcpu->cpu);
10499}
10500EXPORT_SYMBOL_GPL(__kvm_request_immediate_exit);
10501
Takuya Yoshikawa9357d932013-12-13 15:08:38 +090010502/*
Paolo Bonzini8d25b7b2022-02-19 04:28:20 -050010503 * Called within kvm->srcu read side.
Paolo Bonzini362c6982015-02-06 12:48:04 +010010504 * Returns 1 to let vcpu_run() continue the guest execution loop without
Takuya Yoshikawa9357d932013-12-13 15:08:38 +090010505 * exiting to the userspace. Otherwise, the value will be returned to the
10506 * userspace.
10507 */
Avi Kivity851ba692009-08-24 11:10:17 +030010508static int vcpu_enter_guest(struct kvm_vcpu *vcpu)
Hollis Blanchardb6c7a5d2007-11-01 14:16:10 -050010509{
10510 int r;
Matt Gingell62a193e2015-11-16 15:26:07 -080010511 bool req_int_win =
10512 dm_request_for_irq_injection(vcpu) &&
10513 kvm_cpu_accept_dm_intr(vcpu);
Wanpeng Li404d5d72020-04-28 14:23:25 +080010514 fastpath_t exit_fastpath;
Matt Gingell62a193e2015-11-16 15:26:07 -080010515
Jan Kiszka730dca42013-04-28 10:50:52 +020010516 bool req_immediate_exit = false;
Hollis Blanchardb6c7a5d2007-11-01 14:16:10 -050010517
Peter Xufb04a1e2020-09-30 21:22:22 -040010518 /* Forbid vmenter if vcpu dirty ring is soft-full */
10519 if (unlikely(vcpu->kvm->dirty_ring_size &&
10520 kvm_dirty_ring_soft_full(&vcpu->dirty_ring))) {
10521 vcpu->run->exit_reason = KVM_EXIT_DIRTY_RING_FULL;
10522 trace_kvm_dirty_ring_exit(vcpu);
10523 r = 0;
10524 goto out;
10525 }
10526
Radim Krčmář2fa6e1e2017-06-04 14:43:52 +020010527 if (kvm_request_pending(vcpu)) {
Paolo Bonzinif4d31652021-11-11 10:13:38 -050010528 if (kvm_check_request(KVM_REQ_VM_DEAD, vcpu)) {
Sean Christopherson67369272021-07-02 15:04:25 -070010529 r = -EIO;
10530 goto out;
10531 }
Paolo Bonzini729c15c2020-09-22 06:53:57 -040010532 if (kvm_check_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu)) {
Paolo Bonzini9a78e152021-01-08 11:43:08 -050010533 if (unlikely(!kvm_x86_ops.nested_ops->get_nested_state_pages(vcpu))) {
Jim Mattson671ddc72019-10-15 10:44:05 -070010534 r = 0;
10535 goto out;
10536 }
10537 }
Sean Christopherson527d5cd2022-02-25 18:22:45 +000010538 if (kvm_check_request(KVM_REQ_MMU_FREE_OBSOLETE_ROOTS, vcpu))
10539 kvm_mmu_free_obsolete_roots(vcpu);
Avi Kivitya8eeb042010-05-10 12:34:53 +030010540 if (kvm_check_request(KVM_REQ_MIGRATE_TIMER, vcpu))
Marcelo Tosatti2f599712008-05-27 12:10:20 -030010541 __kvm_migrate_timers(vcpu);
Marcelo Tosattid8281992012-11-27 23:29:01 -020010542 if (kvm_check_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu))
Paolo Bonzini6b6fcd22021-09-16 18:15:32 +000010543 kvm_update_masterclock(vcpu->kvm);
Marcelo Tosatti0061d53d2013-05-09 20:21:41 -030010544 if (kvm_check_request(KVM_REQ_GLOBAL_CLOCK_UPDATE, vcpu))
10545 kvm_gen_kvmclock_update(vcpu);
Zachary Amsden34c238a2010-09-18 14:38:14 -100010546 if (kvm_check_request(KVM_REQ_CLOCK_UPDATE, vcpu)) {
10547 r = kvm_guest_time_update(vcpu);
Zachary Amsden8cfdc002010-08-19 22:07:21 -100010548 if (unlikely(r))
10549 goto out;
10550 }
Avi Kivitya8eeb042010-05-10 12:34:53 +030010551 if (kvm_check_request(KVM_REQ_MMU_SYNC, vcpu))
Marcelo Tosatti4731d4c2008-09-23 13:18:39 -030010552 kvm_mmu_sync_roots(vcpu);
Paolo Bonzini727a7e22020-03-05 03:52:50 -050010553 if (kvm_check_request(KVM_REQ_LOAD_MMU_PGD, vcpu))
10554 kvm_mmu_load_pgd(vcpu);
Sean Christophersoneeeb4f62020-03-20 14:28:20 -070010555 if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) {
Sean Christopherson77809382020-03-20 14:28:18 -070010556 kvm_vcpu_flush_tlb_all(vcpu);
Sean Christophersoneeeb4f62020-03-20 14:28:20 -070010557
10558 /* Flushing all ASIDs flushes the current ASID... */
10559 kvm_clear_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu);
10560 }
Sean Christopherson40e5f9082021-11-25 01:49:43 +000010561 kvm_service_local_tlb_flush_requests(vcpu);
Sean Christophersoneeeb4f62020-03-20 14:28:20 -070010562
Avi Kivitya8eeb042010-05-10 12:34:53 +030010563 if (kvm_check_request(KVM_REQ_REPORT_TPR_ACCESS, vcpu)) {
Avi Kivity851ba692009-08-24 11:10:17 +030010564 vcpu->run->exit_reason = KVM_EXIT_TPR_ACCESS;
Avi Kivityb93463a2007-10-25 16:52:32 +020010565 r = 0;
10566 goto out;
10567 }
Avi Kivitya8eeb042010-05-10 12:34:53 +030010568 if (kvm_check_request(KVM_REQ_TRIPLE_FAULT, vcpu)) {
Sean Christophersoncb6a32c2021-03-02 09:45:14 -080010569 if (is_guest_mode(vcpu)) {
10570 kvm_x86_ops.nested_ops->triple_fault(vcpu);
10571 } else {
10572 vcpu->run->exit_reason = KVM_EXIT_SHUTDOWN;
10573 vcpu->mmio_needed = 0;
10574 r = 0;
10575 goto out;
10576 }
Joerg Roedel71c4dfa2008-02-26 16:49:16 +010010577 }
Gleb Natapovaf585b92010-10-14 11:22:46 +020010578 if (kvm_check_request(KVM_REQ_APF_HALT, vcpu)) {
10579 /* Page is swapped out. Do synthetic halt */
10580 vcpu->arch.apf.halted = true;
10581 r = 1;
10582 goto out;
10583 }
Glauber Costac9aaa892011-07-11 15:28:14 -040010584 if (kvm_check_request(KVM_REQ_STEAL_UPDATE, vcpu))
10585 record_steal_time(vcpu);
Paolo Bonzini64d60672015-05-07 11:36:11 +020010586 if (kvm_check_request(KVM_REQ_SMI, vcpu))
10587 process_smi(vcpu);
Avi Kivity7460fb4a2011-09-20 13:43:14 +030010588 if (kvm_check_request(KVM_REQ_NMI, vcpu))
10589 process_nmi(vcpu);
Gleb Natapovf5132b02011-11-10 14:57:22 +020010590 if (kvm_check_request(KVM_REQ_PMU, vcpu))
Wei Huangc6702c92015-06-19 13:44:45 +020010591 kvm_pmu_handle_event(vcpu);
Gleb Natapovf5132b02011-11-10 14:57:22 +020010592 if (kvm_check_request(KVM_REQ_PMI, vcpu))
Wei Huangc6702c92015-06-19 13:44:45 +020010593 kvm_pmu_deliver_pmi(vcpu);
Steve Rutherford7543a632015-07-29 23:21:41 -070010594 if (kvm_check_request(KVM_REQ_IOAPIC_EOI_EXIT, vcpu)) {
10595 BUG_ON(vcpu->arch.pending_ioapic_eoi > 255);
10596 if (test_bit(vcpu->arch.pending_ioapic_eoi,
Andrey Smetanin63086302015-11-10 15:36:32 +030010597 vcpu->arch.ioapic_handled_vectors)) {
Steve Rutherford7543a632015-07-29 23:21:41 -070010598 vcpu->run->exit_reason = KVM_EXIT_IOAPIC_EOI;
10599 vcpu->run->eoi.vector =
10600 vcpu->arch.pending_ioapic_eoi;
10601 r = 0;
10602 goto out;
10603 }
10604 }
Yang Zhang3d81bc72013-04-11 19:25:13 +080010605 if (kvm_check_request(KVM_REQ_SCAN_IOAPIC, vcpu))
10606 vcpu_scan_ioapic(vcpu);
Liran Alone40ff1d2018-03-21 02:50:31 +020010607 if (kvm_check_request(KVM_REQ_LOAD_EOI_EXITMAP, vcpu))
10608 vcpu_load_eoi_exitmap(vcpu);
Tang Chen4256f432014-09-24 15:57:54 +080010609 if (kvm_check_request(KVM_REQ_APIC_PAGE_RELOAD, vcpu))
10610 kvm_vcpu_reload_apic_access_page(vcpu);
Andrey Smetanin2ce79182015-07-03 15:01:41 +030010611 if (kvm_check_request(KVM_REQ_HV_CRASH, vcpu)) {
10612 vcpu->run->exit_reason = KVM_EXIT_SYSTEM_EVENT;
10613 vcpu->run->system_event.type = KVM_SYSTEM_EVENT_CRASH;
Paolo Bonzinid495f942022-04-22 12:30:13 +020010614 vcpu->run->system_event.ndata = 0;
Andrey Smetanin2ce79182015-07-03 15:01:41 +030010615 r = 0;
10616 goto out;
10617 }
Andrey Smetanine516ceb2015-09-16 12:29:48 +030010618 if (kvm_check_request(KVM_REQ_HV_RESET, vcpu)) {
10619 vcpu->run->exit_reason = KVM_EXIT_SYSTEM_EVENT;
10620 vcpu->run->system_event.type = KVM_SYSTEM_EVENT_RESET;
Paolo Bonzinid495f942022-04-22 12:30:13 +020010621 vcpu->run->system_event.ndata = 0;
Andrey Smetanine516ceb2015-09-16 12:29:48 +030010622 r = 0;
10623 goto out;
10624 }
Andrey Smetanindb3975712015-11-10 15:36:35 +030010625 if (kvm_check_request(KVM_REQ_HV_EXIT, vcpu)) {
Vitaly Kuznetsov9ff5e032021-01-26 14:48:11 +010010626 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu);
10627
Andrey Smetanindb3975712015-11-10 15:36:35 +030010628 vcpu->run->exit_reason = KVM_EXIT_HYPERV;
Vitaly Kuznetsov9ff5e032021-01-26 14:48:11 +010010629 vcpu->run->hyperv = hv_vcpu->exit;
Andrey Smetanindb3975712015-11-10 15:36:35 +030010630 r = 0;
10631 goto out;
10632 }
Andrey Smetaninf3b138c2015-12-28 18:27:24 +030010633
10634 /*
10635 * KVM_REQ_HV_STIMER has to be processed after
10636 * KVM_REQ_CLOCK_UPDATE, because Hyper-V SynIC timers
10637 * depend on the guest clock being up-to-date
10638 */
Andrey Smetanin1f4b34f2015-11-30 19:22:21 +030010639 if (kvm_check_request(KVM_REQ_HV_STIMER, vcpu))
10640 kvm_hv_process_stimers(vcpu);
Suravee Suthikulpanit8df14af2019-11-14 14:15:06 -060010641 if (kvm_check_request(KVM_REQ_APICV_UPDATE, vcpu))
10642 kvm_vcpu_update_apicv(vcpu);
Vitaly Kuznetsov557a9612020-05-25 16:41:21 +020010643 if (kvm_check_request(KVM_REQ_APF_READY, vcpu))
10644 kvm_check_async_pf_completion(vcpu);
Alexander Graf1a1552542020-09-25 16:34:21 +020010645 if (kvm_check_request(KVM_REQ_MSR_FILTER_CHANGED, vcpu))
Jason Baronb36464772021-01-14 22:27:56 -050010646 static_call(kvm_x86_msr_filter_changed)(vcpu);
Makarand Sonarea85863c2021-02-12 16:50:12 -080010647
10648 if (kvm_check_request(KVM_REQ_UPDATE_CPU_DIRTY_LOGGING, vcpu))
10649 static_call(kvm_x86_update_cpu_dirty_logging)(vcpu);
Avi Kivity2f52d582008-01-16 12:49:30 +020010650 }
Avi Kivityb93463a2007-10-25 16:52:32 +020010651
David Woodhouse40da8cc2020-12-09 20:08:30 +000010652 if (kvm_check_request(KVM_REQ_EVENT, vcpu) || req_int_win ||
10653 kvm_xen_has_interrupt(vcpu)) {
Paolo Bonzini0f1e2612016-12-17 16:05:19 +010010654 ++vcpu->stat.req_event;
Jim Mattson4fe09bc2021-06-04 10:26:04 -070010655 r = kvm_apic_accept_events(vcpu);
10656 if (r < 0) {
10657 r = 0;
10658 goto out;
10659 }
Jan Kiszka66450a22013-03-13 12:42:34 +010010660 if (vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED) {
10661 r = 1;
10662 goto out;
10663 }
10664
Sean Christophersone746c1f2022-08-30 23:16:11 +000010665 r = kvm_check_and_inject_events(vcpu, &req_immediate_exit);
Jim Mattsona5f69092021-06-04 10:26:03 -070010666 if (r < 0) {
10667 r = 0;
10668 goto out;
10669 }
Paolo Bonzinic9d40912020-05-22 11:21:49 -040010670 if (req_int_win)
Jason Baronb36464772021-01-14 22:27:56 -050010671 static_call(kvm_x86_enable_irq_window)(vcpu);
Gleb Natapov6a8b1d12009-05-11 13:35:51 +030010672
Avi Kivity3842d132010-07-27 12:30:24 +030010673 if (kvm_lapic_enabled(vcpu)) {
10674 update_cr8_intercept(vcpu);
10675 kvm_lapic_sync_to_vapic(vcpu);
10676 }
Gleb Natapov95ba8273132009-04-21 17:45:08 +030010677 }
Avi Kivityb93463a2007-10-25 16:52:32 +020010678
Avi Kivityd8368af2012-05-14 18:07:56 +030010679 r = kvm_mmu_reload(vcpu);
10680 if (unlikely(r)) {
Michael S. Tsirkind905c062012-06-24 19:25:00 +030010681 goto cancel_injection;
Avi Kivityd8368af2012-05-14 18:07:56 +030010682 }
10683
Jan Kiszka42dbaa52008-12-15 13:52:10 +010010684 preempt_disable();
10685
Sean Christophersone27bc042022-01-28 00:51:50 +000010686 static_call(kvm_x86_prepare_switch_to_guest)(vcpu);
Paolo Bonzinib95234c2016-12-19 13:57:33 +010010687
10688 /*
10689 * Disable IRQs before setting IN_GUEST_MODE. Posted interrupt
10690 * IPI are then delayed after guest entry, which ensures that they
10691 * result in virtual interrupt delivery.
10692 */
10693 local_irq_disable();
Maxim Levitsky66fa2262022-02-08 06:48:42 -050010694
10695 /* Store vcpu->apicv_active before vcpu->mode. */
10696 smp_store_release(&vcpu->mode, IN_GUEST_MODE);
Xiao Guangrong6b7e2d02011-01-12 15:40:31 +080010697
Sean Christopherson2031f282022-04-15 00:43:43 +000010698 kvm_vcpu_srcu_read_unlock(vcpu);
Michael S. Tsirkin01b71912013-11-04 22:36:25 +020010699
Lan Tianyu0f127d12016-03-13 11:10:29 +080010700 /*
Paolo Bonzinib95234c2016-12-19 13:57:33 +010010701 * 1) We should set ->mode before checking ->requests. Please see
Andrew Jonescde9af62017-04-26 22:32:24 +020010702 * the comment in kvm_vcpu_exiting_guest_mode().
Paolo Bonzinib95234c2016-12-19 13:57:33 +010010703 *
Luwei Kang81b01662019-01-31 16:52:02 +080010704 * 2) For APICv, we should set ->mode before checking PID.ON. This
Paolo Bonzinib95234c2016-12-19 13:57:33 +010010705 * pairs with the memory barrier implicit in pi_test_and_set_on
10706 * (see vmx_deliver_posted_interrupt).
10707 *
10708 * 3) This also orders the write to mode from any reads to the page
10709 * tables done while the VCPU is running. Please see the comment
10710 * in kvm_flush_remote_tlbs.
Xiao Guangrong6b7e2d02011-01-12 15:40:31 +080010711 */
Michael S. Tsirkin01b71912013-11-04 22:36:25 +020010712 smp_mb__after_srcu_read_unlock();
Hollis Blanchardb6c7a5d2007-11-01 14:16:10 -050010713
Paolo Bonzinib95234c2016-12-19 13:57:33 +010010714 /*
Sean Christopherson0f65a9d32021-12-08 01:52:26 +000010715 * Process pending posted interrupts to handle the case where the
10716 * notification IRQ arrived in the host, or was never sent (because the
10717 * target vCPU wasn't running). Do this regardless of the vCPU's APICv
10718 * status, KVM doesn't update assigned devices when APICv is inhibited,
10719 * i.e. they can post interrupts even if APICv is temporarily disabled.
Paolo Bonzinib95234c2016-12-19 13:57:33 +010010720 */
Paolo Bonzini37c4dbf2021-11-22 19:43:10 -050010721 if (kvm_lapic_enabled(vcpu))
10722 static_call_cond(kvm_x86_sync_pir_to_irr)(vcpu);
Hollis Blanchardb6c7a5d2007-11-01 14:16:10 -050010723
Wanpeng Li5a9f5442020-04-28 14:23:26 +080010724 if (kvm_vcpu_exit_request(vcpu)) {
Xiao Guangrong6b7e2d02011-01-12 15:40:31 +080010725 vcpu->mode = OUTSIDE_GUEST_MODE;
Hollis Blanchardb6c7a5d2007-11-01 14:16:10 -050010726 smp_wmb();
10727 local_irq_enable();
10728 preempt_enable();
Sean Christopherson2031f282022-04-15 00:43:43 +000010729 kvm_vcpu_srcu_read_lock(vcpu);
Hollis Blanchardb6c7a5d2007-11-01 14:16:10 -050010730 r = 1;
Michael S. Tsirkind905c062012-06-24 19:25:00 +030010731 goto cancel_injection;
Hollis Blanchardb6c7a5d2007-11-01 14:16:10 -050010732 }
10733
Paolo Bonzinic43203c2016-06-01 22:26:00 +020010734 if (req_immediate_exit) {
10735 kvm_make_request(KVM_REQ_EVENT, vcpu);
Jason Baronb36464772021-01-14 22:27:56 -050010736 static_call(kvm_x86_request_immediate_exit)(vcpu);
Paolo Bonzinic43203c2016-06-01 22:26:00 +020010737 }
Nadav Har'Eld6185f22011-09-22 13:52:56 +030010738
Sean Christopherson2620fe22020-01-17 11:30:51 -080010739 fpregs_assert_state_consistent();
10740 if (test_thread_flag(TIF_NEED_FPU_LOAD))
10741 switch_fpu_return();
Rik van Riel5f409e22019-04-03 18:41:52 +020010742
Jing Liuec5be882022-01-05 04:35:23 -080010743 if (vcpu->arch.guest_fpu.xfd_err)
10744 wrmsrl(MSR_IA32_XFD_ERR, vcpu->arch.guest_fpu.xfd_err);
10745
Hollis Blanchardb6c7a5d2007-11-01 14:16:10 -050010746 if (unlikely(vcpu->arch.switch_db_regs)) {
Hollis Blanchardb6c7a5d2007-11-01 14:16:10 -050010747 set_debugreg(0, 7);
10748 set_debugreg(vcpu->arch.eff_db[0], 0);
10749 set_debugreg(vcpu->arch.eff_db[1], 1);
10750 set_debugreg(vcpu->arch.eff_db[2], 2);
10751 set_debugreg(vcpu->arch.eff_db[3], 3);
Lai Jiangshanf85d4012021-06-29 01:26:32 +080010752 } else if (unlikely(hw_breakpoint_active())) {
10753 set_debugreg(0, 7);
Hollis Blanchardb6c7a5d2007-11-01 14:16:10 -050010754 }
10755
Mark Rutlandb2d2af72022-02-01 13:29:24 +000010756 guest_timing_enter_irqoff();
10757
Paolo Bonzinid89d04a2021-02-02 10:44:23 -050010758 for (;;) {
Sean Christophersonee49a892021-10-21 17:49:25 -070010759 /*
10760 * Assert that vCPU vs. VM APICv state is consistent. An APICv
10761 * update must kick and wait for all vCPUs before toggling the
10762 * per-VM state, and responsing vCPUs must wait for the update
10763 * to complete before servicing KVM_REQ_APICV_UPDATE.
10764 */
Suravee Suthikulpanitf8d8ac22022-05-19 05:27:06 -050010765 WARN_ON_ONCE((kvm_vcpu_apicv_activated(vcpu) != kvm_vcpu_apicv_active(vcpu)) &&
10766 (kvm_get_apic_mode(vcpu) != LAPIC_MODE_DISABLED));
Sean Christophersonee49a892021-10-21 17:49:25 -070010767
Sean Christophersone27bc042022-01-28 00:51:50 +000010768 exit_fastpath = static_call(kvm_x86_vcpu_run)(vcpu);
Paolo Bonzinid89d04a2021-02-02 10:44:23 -050010769 if (likely(exit_fastpath != EXIT_FASTPATH_REENTER_GUEST))
10770 break;
10771
Paolo Bonzini37c4dbf2021-11-22 19:43:10 -050010772 if (kvm_lapic_enabled(vcpu))
10773 static_call_cond(kvm_x86_sync_pir_to_irr)(vcpu);
Paolo Bonzinide7cd3f2021-10-20 06:27:36 -040010774
10775 if (unlikely(kvm_vcpu_exit_request(vcpu))) {
Paolo Bonzinid89d04a2021-02-02 10:44:23 -050010776 exit_fastpath = EXIT_FASTPATH_EXIT_HANDLED;
10777 break;
10778 }
Paolo Bonzinide7cd3f2021-10-20 06:27:36 -040010779 }
Hollis Blanchardb6c7a5d2007-11-01 14:16:10 -050010780
Frederic Weisbecker24f1e32c2009-09-09 19:22:48 +020010781 /*
Paolo Bonzinic77fb5f2014-02-21 10:17:24 +010010782 * Do this here before restoring debug registers on the host. And
10783 * since we do this before handling the vmexit, a DR access vmexit
10784 * can (a) read the correct value of the debug registers, (b) set
10785 * KVM_DEBUGREG_WONT_EXIT again.
10786 */
10787 if (unlikely(vcpu->arch.switch_db_regs & KVM_DEBUGREG_WONT_EXIT)) {
Paolo Bonzinic77fb5f2014-02-21 10:17:24 +010010788 WARN_ON(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP);
Jason Baronb36464772021-01-14 22:27:56 -050010789 static_call(kvm_x86_sync_dirty_debug_regs)(vcpu);
Paolo Bonzini70e4da72016-02-26 12:28:40 +010010790 kvm_update_dr0123(vcpu);
Paolo Bonzini70e4da72016-02-26 12:28:40 +010010791 kvm_update_dr7(vcpu);
Paolo Bonzinic77fb5f2014-02-21 10:17:24 +010010792 }
10793
10794 /*
Frederic Weisbecker24f1e32c2009-09-09 19:22:48 +020010795 * If the guest has used debug registers, at least dr7
10796 * will be disabled while returning to the host.
10797 * If we don't have active breakpoints in the host, we don't
10798 * care about the messed up debug address registers. But if
10799 * we have some of them active, restore the old state.
10800 */
Frederic Weisbecker59d8eb52009-11-10 11:03:12 +010010801 if (hw_breakpoint_active())
Frederic Weisbecker24f1e32c2009-09-09 19:22:48 +020010802 hw_breakpoint_restore();
Hollis Blanchardb6c7a5d2007-11-01 14:16:10 -050010803
Jim Mattsonc9671182020-06-03 16:56:23 -070010804 vcpu->arch.last_vmentry_cpu = vcpu->cpu;
Haozhong Zhang4ba76532015-10-20 15:39:07 +080010805 vcpu->arch.last_guest_tsc = kvm_read_l1_tsc(vcpu, rdtsc());
Zachary Amsden1d5f0662010-08-19 22:07:30 -100010806
Xiao Guangrong6b7e2d02011-01-12 15:40:31 +080010807 vcpu->mode = OUTSIDE_GUEST_MODE;
Avi Kivityd94e1dc2010-05-03 16:54:48 +030010808 smp_wmb();
Yang Zhanga547c6d2013-04-11 19:25:10 +080010809
Kevin Tianb5274b12022-01-05 04:35:32 -080010810 /*
10811 * Sync xfd before calling handle_exit_irqoff() which may
10812 * rely on the fact that guest_fpu::xfd is up-to-date (e.g.
10813 * in #NM irqoff handler).
10814 */
10815 if (vcpu->arch.xfd_no_write_intercept)
10816 fpu_sync_guest_vmexit_xfd_state();
10817
Jason Baronb36464772021-01-14 22:27:56 -050010818 static_call(kvm_x86_handle_exit_irqoff)(vcpu);
Hollis Blanchardb6c7a5d2007-11-01 14:16:10 -050010819
Jing Liuec5be882022-01-05 04:35:23 -080010820 if (vcpu->arch.guest_fpu.xfd_err)
10821 wrmsrl(MSR_IA32_XFD_ERR, 0);
10822
Sean Christophersond7a08882019-07-10 09:07:34 -070010823 /*
10824 * Consume any pending interrupts, including the possible source of
10825 * VM-Exit on SVM and any ticks that occur between VM-Exit and now.
10826 * An instruction is required after local_irq_enable() to fully unblock
10827 * interrupts on processors that implement an interrupt shadow, the
10828 * stat.exits increment will do nicely.
10829 */
Sean Christophersondb215752021-11-11 02:07:32 +000010830 kvm_before_interrupt(vcpu, KVM_HANDLING_IRQ);
Sean Christophersond7a08882019-07-10 09:07:34 -070010831 local_irq_enable();
Hollis Blanchardb6c7a5d2007-11-01 14:16:10 -050010832 ++vcpu->stat.exits;
Sean Christophersond7a08882019-07-10 09:07:34 -070010833 local_irq_disable();
Hollis Blanchardb6c7a5d2007-11-01 14:16:10 -050010834 kvm_after_interrupt(vcpu);
10835
Wanpeng Li16045712021-05-04 17:27:30 -070010836 /*
10837 * Wait until after servicing IRQs to account guest time so that any
10838 * ticks that occurred while running the guest are properly accounted
10839 * to the guest. Waiting until IRQs are enabled degrades the accuracy
10840 * of accounting via context tracking, but the loss of accuracy is
10841 * acceptable for all known use cases.
10842 */
Mark Rutlandb2d2af72022-02-01 13:29:24 +000010843 guest_timing_exit_irqoff();
Wanpeng Li16045712021-05-04 17:27:30 -070010844
Paolo Bonzinif2485b32016-06-15 15:23:11 +020010845 local_irq_enable();
Hollis Blanchardb6c7a5d2007-11-01 14:16:10 -050010846 preempt_enable();
10847
Sean Christopherson2031f282022-04-15 00:43:43 +000010848 kvm_vcpu_srcu_read_lock(vcpu);
Marcelo Tosatti3200f402008-03-29 20:17:59 -030010849
Hollis Blanchardb6c7a5d2007-11-01 14:16:10 -050010850 /*
10851 * Profile KVM exit RIPs:
10852 */
10853 if (unlikely(prof_on == KVM_PROFILING)) {
Marcelo Tosatti5fdbf972008-06-27 14:58:02 -030010854 unsigned long rip = kvm_rip_read(vcpu);
10855 profile_hit(KVM_PROFILING, (void *)rip);
Hollis Blanchardb6c7a5d2007-11-01 14:16:10 -050010856 }
10857
Zachary Amsdencc578282012-02-03 15:43:50 -020010858 if (unlikely(vcpu->arch.tsc_always_catchup))
10859 kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
Avi Kivity298101d2007-11-25 13:41:11 +020010860
Michael S. Tsirkin5cfb1d52012-06-24 19:24:54 +030010861 if (vcpu->arch.apic_attention)
10862 kvm_lapic_sync_from_vapic(vcpu);
Avi Kivityb93463a2007-10-25 16:52:32 +020010863
Jason Baronb36464772021-01-14 22:27:56 -050010864 r = static_call(kvm_x86_handle_exit)(vcpu, exit_fastpath);
Michael S. Tsirkind905c062012-06-24 19:25:00 +030010865 return r;
10866
10867cancel_injection:
Sean Christopherson8081ad02020-04-22 19:25:40 -070010868 if (req_immediate_exit)
10869 kvm_make_request(KVM_REQ_EVENT, vcpu);
Jason Baronb36464772021-01-14 22:27:56 -050010870 static_call(kvm_x86_cancel_injection)(vcpu);
Michael S. Tsirkinae7a2a32012-06-24 19:25:07 +030010871 if (unlikely(vcpu->arch.apic_attention))
10872 kvm_lapic_sync_from_vapic(vcpu);
Hollis Blanchardb6c7a5d2007-11-01 14:16:10 -050010873out:
Marcelo Tosattid7690172008-09-08 15:23:48 -030010874 return r;
10875}
10876
Paolo Bonzini8d25b7b2022-02-19 04:28:20 -050010877/* Called within kvm->srcu read side. */
Sean Christopherson2031f282022-04-15 00:43:43 +000010878static inline int vcpu_block(struct kvm_vcpu *vcpu)
Paolo Bonzini362c6982015-02-06 12:48:04 +010010879{
Sean Christopherson98c25ea2021-12-08 01:52:17 +000010880 bool hv_timer;
10881
Sean Christophersonc3e8abf2021-12-08 01:52:19 +000010882 if (!kvm_arch_vcpu_runnable(vcpu)) {
Sean Christopherson98c25ea2021-12-08 01:52:17 +000010883 /*
10884 * Switch to the software timer before halt-polling/blocking as
10885 * the guest's timer may be a break event for the vCPU, and the
10886 * hypervisor timer runs only when the CPU is in guest mode.
10887 * Switch before halt-polling so that KVM recognizes an expired
10888 * timer before blocking.
10889 */
10890 hv_timer = kvm_lapic_hv_timer_in_use(vcpu);
10891 if (hv_timer)
10892 kvm_lapic_switch_to_sw_timer(vcpu);
10893
Sean Christopherson2031f282022-04-15 00:43:43 +000010894 kvm_vcpu_srcu_read_unlock(vcpu);
Sean Christophersoncdafece2021-10-08 19:12:11 -070010895 if (vcpu->arch.mp_state == KVM_MP_STATE_HALTED)
10896 kvm_vcpu_halt(vcpu);
10897 else
10898 kvm_vcpu_block(vcpu);
Sean Christopherson2031f282022-04-15 00:43:43 +000010899 kvm_vcpu_srcu_read_lock(vcpu);
Feng Wubf9f6ac2015-09-18 22:29:55 +080010900
Sean Christopherson98c25ea2021-12-08 01:52:17 +000010901 if (hv_timer)
10902 kvm_lapic_switch_to_hv_timer(vcpu);
10903
Paolo Bonzini599275c2022-09-21 00:32:00 +000010904 /*
10905 * If the vCPU is not runnable, a signal or another host event
10906 * of some kind is pending; service it without changing the
10907 * vCPU's activity state.
10908 */
10909 if (!kvm_arch_vcpu_runnable(vcpu))
Paolo Bonzini9c8fd1b2015-02-06 12:58:42 +010010910 return 1;
10911 }
Gleb Natapov09cec752009-03-23 15:11:44 +020010912
Paolo Bonzini26844fe2022-09-21 00:31:59 +000010913 /*
10914 * Evaluate nested events before exiting the halted state. This allows
10915 * the halt state to be recorded properly in the VMCS12's activity
10916 * state field (AMD does not have a similar field and a VM-Exit always
10917 * causes a spurious wakeup from HLT).
10918 */
10919 if (is_guest_mode(vcpu)) {
10920 if (kvm_check_nested_events(vcpu) < 0)
10921 return 0;
10922 }
10923
Jim Mattson4fe09bc2021-06-04 10:26:04 -070010924 if (kvm_apic_accept_events(vcpu) < 0)
10925 return 0;
Paolo Bonzini362c6982015-02-06 12:48:04 +010010926 switch(vcpu->arch.mp_state) {
10927 case KVM_MP_STATE_HALTED:
Tom Lendacky647daca2021-01-04 14:20:01 -060010928 case KVM_MP_STATE_AP_RESET_HOLD:
Paolo Bonzini362c6982015-02-06 12:48:04 +010010929 vcpu->arch.pv.pv_unhalted = false;
10930 vcpu->arch.mp_state =
10931 KVM_MP_STATE_RUNNABLE;
Gustavo A. R. Silvadf561f662020-08-23 17:36:59 -050010932 fallthrough;
Paolo Bonzini362c6982015-02-06 12:48:04 +010010933 case KVM_MP_STATE_RUNNABLE:
10934 vcpu->arch.apf.halted = false;
10935 break;
10936 case KVM_MP_STATE_INIT_RECEIVED:
10937 break;
10938 default:
Paolo Bonzini22c6a0e2022-08-11 12:41:25 -040010939 WARN_ON_ONCE(1);
10940 break;
Paolo Bonzini362c6982015-02-06 12:48:04 +010010941 }
10942 return 1;
10943}
10944
Paolo Bonzini5d9bc642015-10-13 10:18:53 +020010945static inline bool kvm_vcpu_running(struct kvm_vcpu *vcpu)
10946{
10947 return (vcpu->arch.mp_state == KVM_MP_STATE_RUNNABLE &&
10948 !vcpu->arch.apf.halted);
10949}
10950
Paolo Bonzini8d25b7b2022-02-19 04:28:20 -050010951/* Called within kvm->srcu read side. */
Paolo Bonzini362c6982015-02-06 12:48:04 +010010952static int vcpu_run(struct kvm_vcpu *vcpu)
Marcelo Tosattid7690172008-09-08 15:23:48 -030010953{
10954 int r;
10955
Paolo Bonzinic595cee2018-07-02 13:07:14 +020010956 vcpu->arch.l1tf_flush_l1d = true;
Marcelo Tosattid7690172008-09-08 15:23:48 -030010957
Paolo Bonzini362c6982015-02-06 12:48:04 +010010958 for (;;) {
Paolo Bonzini6cd88242022-06-07 10:09:03 -040010959 /*
10960 * If another guest vCPU requests a PV TLB flush in the middle
10961 * of instruction emulation, the rest of the emulation could
10962 * use a stale page translation. Assume that any code after
10963 * this point can start executing an instruction.
10964 */
10965 vcpu->arch.at_instruction_boundary = false;
Paolo Bonzini58f800d2015-10-13 21:32:50 +020010966 if (kvm_vcpu_running(vcpu)) {
Avi Kivity851ba692009-08-24 11:10:17 +030010967 r = vcpu_enter_guest(vcpu);
Feng Wubf9f6ac2015-09-18 22:29:55 +080010968 } else {
Sean Christopherson2031f282022-04-15 00:43:43 +000010969 r = vcpu_block(vcpu);
Feng Wubf9f6ac2015-09-18 22:29:55 +080010970 }
10971
Gleb Natapov09cec752009-03-23 15:11:44 +020010972 if (r <= 0)
10973 break;
10974
Marcelo Tosatti084071d2021-05-25 10:41:17 -030010975 kvm_clear_request(KVM_REQ_UNBLOCK, vcpu);
David Woodhouse7caf9572022-03-03 15:41:15 +000010976 if (kvm_xen_has_pending_events(vcpu))
10977 kvm_xen_inject_pending_events(vcpu);
10978
Gleb Natapov09cec752009-03-23 15:11:44 +020010979 if (kvm_cpu_has_pending_timer(vcpu))
10980 kvm_inject_pending_timer_irqs(vcpu);
10981
Matt Gingell782d4222015-11-16 15:26:00 -080010982 if (dm_request_for_irq_injection(vcpu) &&
10983 kvm_vcpu_ready_for_interrupt_injection(vcpu)) {
Paolo Bonzini4ca7dd82015-07-30 10:32:16 +020010984 r = 0;
10985 vcpu->run->exit_reason = KVM_EXIT_IRQ_WINDOW_OPEN;
Gleb Natapov09cec752009-03-23 15:11:44 +020010986 ++vcpu->stat.request_irq_exits;
Paolo Bonzini362c6982015-02-06 12:48:04 +010010987 break;
Gleb Natapov09cec752009-03-23 15:11:44 +020010988 }
Gleb Natapovaf585b92010-10-14 11:22:46 +020010989
Thomas Gleixnerf3020b82020-07-30 09:19:01 +020010990 if (__xfer_to_guest_mode_work_pending()) {
Sean Christopherson2031f282022-04-15 00:43:43 +000010991 kvm_vcpu_srcu_read_unlock(vcpu);
Thomas Gleixner72c3c0f2020-07-23 00:00:09 +020010992 r = xfer_to_guest_mode_handle_work(vcpu);
Sean Christopherson2031f282022-04-15 00:43:43 +000010993 kvm_vcpu_srcu_read_lock(vcpu);
Thomas Gleixner72c3c0f2020-07-23 00:00:09 +020010994 if (r)
10995 return r;
Marcelo Tosattid7690172008-09-08 15:23:48 -030010996 }
10997 }
10998
Hollis Blanchardb6c7a5d2007-11-01 14:16:10 -050010999 return r;
11000}
11001
Gleb Natapov716d51a2012-09-03 15:24:26 +030011002static inline int complete_emulated_io(struct kvm_vcpu *vcpu)
11003{
Sean Christopherson2d089352022-04-15 00:43:41 +000011004 return kvm_emulate_instruction(vcpu, EMULTYPE_NO_DECODE);
Gleb Natapov716d51a2012-09-03 15:24:26 +030011005}
11006
11007static int complete_emulated_pio(struct kvm_vcpu *vcpu)
11008{
11009 BUG_ON(!vcpu->arch.pio.count);
11010
11011 return complete_emulated_io(vcpu);
11012}
11013
Avi Kivityf78146b2012-04-18 19:22:47 +030011014/*
11015 * Implements the following, as a state machine:
11016 *
11017 * read:
11018 * for each fragment
Xiao Guangrong87da7e62012-10-24 14:07:59 +080011019 * for each mmio piece in the fragment
11020 * write gpa, len
11021 * exit
11022 * copy data
Avi Kivityf78146b2012-04-18 19:22:47 +030011023 * execute insn
11024 *
11025 * write:
11026 * for each fragment
Xiao Guangrong87da7e62012-10-24 14:07:59 +080011027 * for each mmio piece in the fragment
11028 * write gpa, len
11029 * copy data
11030 * exit
Avi Kivityf78146b2012-04-18 19:22:47 +030011031 */
Gleb Natapov716d51a2012-09-03 15:24:26 +030011032static int complete_emulated_mmio(struct kvm_vcpu *vcpu)
Avi Kivity5287f192010-01-19 14:20:10 +020011033{
11034 struct kvm_run *run = vcpu->run;
Avi Kivityf78146b2012-04-18 19:22:47 +030011035 struct kvm_mmio_fragment *frag;
Xiao Guangrong87da7e62012-10-24 14:07:59 +080011036 unsigned len;
Avi Kivity5287f192010-01-19 14:20:10 +020011037
Gleb Natapov716d51a2012-09-03 15:24:26 +030011038 BUG_ON(!vcpu->mmio_needed);
Avi Kivity5287f192010-01-19 14:20:10 +020011039
Gleb Natapov716d51a2012-09-03 15:24:26 +030011040 /* Complete previous fragment */
Xiao Guangrong87da7e62012-10-24 14:07:59 +080011041 frag = &vcpu->mmio_fragments[vcpu->mmio_cur_fragment];
11042 len = min(8u, frag->len);
Gleb Natapov716d51a2012-09-03 15:24:26 +030011043 if (!vcpu->mmio_is_write)
Xiao Guangrong87da7e62012-10-24 14:07:59 +080011044 memcpy(frag->data, run->mmio.data, len);
11045
11046 if (frag->len <= 8) {
11047 /* Switch to the next fragment. */
11048 frag++;
11049 vcpu->mmio_cur_fragment++;
11050 } else {
11051 /* Go forward to the next mmio piece. */
11052 frag->data += len;
11053 frag->gpa += len;
11054 frag->len -= len;
11055 }
11056
Andrew Honiga08d3b32014-02-27 19:35:14 +010011057 if (vcpu->mmio_cur_fragment >= vcpu->mmio_nr_fragments) {
Gleb Natapov716d51a2012-09-03 15:24:26 +030011058 vcpu->mmio_needed = 0;
Paolo Bonzini0912c972013-08-27 15:41:43 +020011059
11060 /* FIXME: return into emulator if single-stepping. */
Avi Kivitycef4dea2010-01-20 12:01:20 +020011061 if (vcpu->mmio_is_write)
Gleb Natapov716d51a2012-09-03 15:24:26 +030011062 return 1;
11063 vcpu->mmio_read_completed = 1;
11064 return complete_emulated_io(vcpu);
Avi Kivity5287f192010-01-19 14:20:10 +020011065 }
Xiao Guangrong87da7e62012-10-24 14:07:59 +080011066
Gleb Natapov716d51a2012-09-03 15:24:26 +030011067 run->exit_reason = KVM_EXIT_MMIO;
11068 run->mmio.phys_addr = frag->gpa;
11069 if (vcpu->mmio_is_write)
Xiao Guangrong87da7e62012-10-24 14:07:59 +080011070 memcpy(run->mmio.data, frag->data, min(8u, frag->len));
11071 run->mmio.len = min(8u, frag->len);
Gleb Natapov716d51a2012-09-03 15:24:26 +030011072 run->mmio.is_write = vcpu->mmio_is_write;
11073 vcpu->arch.complete_userspace_io = complete_emulated_mmio;
11074 return 0;
Avi Kivity5287f192010-01-19 14:20:10 +020011075}
11076
Sebastian Andrzej Siewior822f3122018-09-12 15:33:45 +020011077/* Swap (qemu) user FPU context for the guest FPU context. */
11078static void kvm_load_guest_fpu(struct kvm_vcpu *vcpu)
11079{
Sean Christophersone27bc042022-01-28 00:51:50 +000011080 /* Exclude PKRU, it's restored separately immediately after VM-Exit. */
Thomas Gleixnerd69c1382021-10-22 20:55:53 +020011081 fpu_swap_kvm_fpstate(&vcpu->arch.guest_fpu, true);
Sebastian Andrzej Siewior822f3122018-09-12 15:33:45 +020011082 trace_kvm_fpu(1);
11083}
11084
11085/* When vcpu_run ends, restore user space FPU context. */
11086static void kvm_put_guest_fpu(struct kvm_vcpu *vcpu)
11087{
Thomas Gleixnerd69c1382021-10-22 20:55:53 +020011088 fpu_swap_kvm_fpstate(&vcpu->arch.guest_fpu, false);
Sebastian Andrzej Siewior822f3122018-09-12 15:33:45 +020011089 ++vcpu->stat.fpu_reload;
11090 trace_kvm_fpu(0);
11091}
11092
Tianjia Zhang1b94f6f2020-04-16 13:10:57 +080011093int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu)
Hollis Blanchardb6c7a5d2007-11-01 14:16:10 -050011094{
Sean Christopherson7709aba2022-08-30 23:16:08 +000011095 struct kvm_queued_exception *ex = &vcpu->arch.exception;
Tianjia Zhang1b94f6f2020-04-16 13:10:57 +080011096 struct kvm_run *kvm_run = vcpu->run;
Hollis Blanchardb6c7a5d2007-11-01 14:16:10 -050011097 int r;
Hollis Blanchardb6c7a5d2007-11-01 14:16:10 -050011098
Christoffer Dallaccb7572017-12-04 21:35:25 +010011099 vcpu_load(vcpu);
Jan H. Schönherr20b70352017-11-24 22:39:01 +010011100 kvm_sigset_activate(vcpu);
Chenyi Qiang15aad3b2020-11-06 17:03:13 +080011101 kvm_run->flags = 0;
Peter Xu5663d8f2017-12-12 17:15:02 +010011102 kvm_load_guest_fpu(vcpu);
11103
Sean Christopherson2031f282022-04-15 00:43:43 +000011104 kvm_vcpu_srcu_read_lock(vcpu);
Avi Kivityac9f6dc2008-07-06 15:48:31 +030011105 if (unlikely(vcpu->arch.mp_state == KVM_MP_STATE_UNINITIALIZED)) {
Jan H. Schönherr2f173d22017-09-06 18:34:06 +020011106 if (kvm_run->immediate_exit) {
11107 r = -EINTR;
11108 goto out;
11109 }
Sean Christopherson98c25ea2021-12-08 01:52:17 +000011110 /*
11111 * It should be impossible for the hypervisor timer to be in
11112 * use before KVM has ever run the vCPU.
11113 */
11114 WARN_ON_ONCE(kvm_lapic_hv_timer_in_use(vcpu));
Paolo Bonzini8d25b7b2022-02-19 04:28:20 -050011115
Sean Christopherson2031f282022-04-15 00:43:43 +000011116 kvm_vcpu_srcu_read_unlock(vcpu);
Sean Christophersonc91d4492021-10-08 19:12:10 -070011117 kvm_vcpu_block(vcpu);
Sean Christopherson2031f282022-04-15 00:43:43 +000011118 kvm_vcpu_srcu_read_lock(vcpu);
Paolo Bonzini8d25b7b2022-02-19 04:28:20 -050011119
Jim Mattson4fe09bc2021-06-04 10:26:04 -070011120 if (kvm_apic_accept_events(vcpu) < 0) {
11121 r = 0;
11122 goto out;
11123 }
Avi Kivityac9f6dc2008-07-06 15:48:31 +030011124 r = -EAGAIN;
Jan H. Schönherra0595002017-09-06 00:27:19 +020011125 if (signal_pending(current)) {
11126 r = -EINTR;
Tianjia Zhang1b94f6f2020-04-16 13:10:57 +080011127 kvm_run->exit_reason = KVM_EXIT_INTR;
Jan H. Schönherra0595002017-09-06 00:27:19 +020011128 ++vcpu->stat.signal_exits;
11129 }
Avi Kivityac9f6dc2008-07-06 15:48:31 +030011130 goto out;
11131 }
11132
Sean Christophersone489a4a2021-07-02 15:04:29 -070011133 if ((kvm_run->kvm_valid_regs & ~KVM_SYNC_X86_VALID_FIELDS) ||
11134 (kvm_run->kvm_dirty_regs & ~KVM_SYNC_X86_VALID_FIELDS)) {
Ken Hofsass01643c52018-01-31 16:03:36 -080011135 r = -EINVAL;
11136 goto out;
11137 }
11138
Tianjia Zhang1b94f6f2020-04-16 13:10:57 +080011139 if (kvm_run->kvm_dirty_regs) {
Ken Hofsass01643c52018-01-31 16:03:36 -080011140 r = sync_regs(vcpu);
11141 if (r != 0)
11142 goto out;
11143 }
11144
Hollis Blanchardb6c7a5d2007-11-01 14:16:10 -050011145 /* re-sync apic's tpr */
Paolo Bonzini35754c92015-07-29 12:05:37 +020011146 if (!lapic_in_kernel(vcpu)) {
Andre Przywaraeea1cff2010-12-21 11:12:00 +010011147 if (kvm_set_cr8(vcpu, kvm_run->cr8) != 0) {
11148 r = -EINVAL;
11149 goto out;
11150 }
11151 }
Hollis Blanchardb6c7a5d2007-11-01 14:16:10 -050011152
Sean Christopherson7709aba2022-08-30 23:16:08 +000011153 /*
11154 * If userspace set a pending exception and L2 is active, convert it to
11155 * a pending VM-Exit if L1 wants to intercept the exception.
11156 */
11157 if (vcpu->arch.exception_from_userspace && is_guest_mode(vcpu) &&
11158 kvm_x86_ops.nested_ops->is_exception_vmexit(vcpu, ex->vector,
11159 ex->error_code)) {
11160 kvm_queue_exception_vmexit(vcpu, ex->vector,
11161 ex->has_error_code, ex->error_code,
11162 ex->has_payload, ex->payload);
11163 ex->injected = false;
11164 ex->pending = false;
11165 }
11166 vcpu->arch.exception_from_userspace = false;
11167
Gleb Natapov716d51a2012-09-03 15:24:26 +030011168 if (unlikely(vcpu->arch.complete_userspace_io)) {
11169 int (*cui)(struct kvm_vcpu *) = vcpu->arch.complete_userspace_io;
11170 vcpu->arch.complete_userspace_io = NULL;
11171 r = cui(vcpu);
11172 if (r <= 0)
Peter Xu5663d8f2017-12-12 17:15:02 +010011173 goto out;
Sean Christopherson0bc27322022-07-11 23:27:50 +000011174 } else {
11175 WARN_ON_ONCE(vcpu->arch.pio.count);
11176 WARN_ON_ONCE(vcpu->mmio_needed);
11177 }
Avi Kivity5287f192010-01-19 14:20:10 +020011178
Sean Christophersonfc4fad72021-12-28 23:24:36 +000011179 if (kvm_run->immediate_exit) {
Paolo Bonzini460df4c2017-02-08 11:50:15 +010011180 r = -EINTR;
Sean Christophersonfc4fad72021-12-28 23:24:36 +000011181 goto out;
11182 }
11183
11184 r = static_call(kvm_x86_vcpu_pre_run)(vcpu);
11185 if (r <= 0)
11186 goto out;
11187
11188 r = vcpu_run(vcpu);
Hollis Blanchardb6c7a5d2007-11-01 14:16:10 -050011189
11190out:
Peter Xu5663d8f2017-12-12 17:15:02 +010011191 kvm_put_guest_fpu(vcpu);
Tianjia Zhang1b94f6f2020-04-16 13:10:57 +080011192 if (kvm_run->kvm_valid_regs)
Ken Hofsass01643c52018-01-31 16:03:36 -080011193 store_regs(vcpu);
Marcelo Tosattif1d86e42010-05-03 23:04:27 -030011194 post_kvm_run_save(vcpu);
Sean Christopherson2031f282022-04-15 00:43:43 +000011195 kvm_vcpu_srcu_read_unlock(vcpu);
Hollis Blanchardb6c7a5d2007-11-01 14:16:10 -050011196
Paolo Bonzini8d25b7b2022-02-19 04:28:20 -050011197 kvm_sigset_deactivate(vcpu);
Christoffer Dallaccb7572017-12-04 21:35:25 +010011198 vcpu_put(vcpu);
Hollis Blanchardb6c7a5d2007-11-01 14:16:10 -050011199 return r;
11200}
11201
Ken Hofsass01643c52018-01-31 16:03:36 -080011202static void __get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
Hollis Blanchardb6c7a5d2007-11-01 14:16:10 -050011203{
Gleb Natapov7ae441e2011-03-31 12:06:41 +020011204 if (vcpu->arch.emulate_regs_need_sync_to_vcpu) {
11205 /*
11206 * We are here if userspace calls get_regs() in the middle of
11207 * instruction emulation. Registers state needs to be copied
Guo Chao4a969982012-06-28 15:17:27 +080011208 * back from emulation context to vcpu. Userspace shouldn't do
Gleb Natapov7ae441e2011-03-31 12:06:41 +020011209 * that usually, but some bad designed PV devices (vmware
11210 * backdoor interface) need this to work
11211 */
Sean Christophersonc9b8b072020-02-18 15:29:48 -080011212 emulator_writeback_register_cache(vcpu->arch.emulate_ctxt);
Gleb Natapov7ae441e2011-03-31 12:06:41 +020011213 vcpu->arch.emulate_regs_need_sync_to_vcpu = false;
11214 }
Sean Christophersonde3cd112019-04-30 10:36:17 -070011215 regs->rax = kvm_rax_read(vcpu);
11216 regs->rbx = kvm_rbx_read(vcpu);
11217 regs->rcx = kvm_rcx_read(vcpu);
11218 regs->rdx = kvm_rdx_read(vcpu);
11219 regs->rsi = kvm_rsi_read(vcpu);
11220 regs->rdi = kvm_rdi_read(vcpu);
Paolo Bonzinie9c16c72019-04-30 22:07:26 +020011221 regs->rsp = kvm_rsp_read(vcpu);
Sean Christophersonde3cd112019-04-30 10:36:17 -070011222 regs->rbp = kvm_rbp_read(vcpu);
Hollis Blanchardb6c7a5d2007-11-01 14:16:10 -050011223#ifdef CONFIG_X86_64
Sean Christophersonde3cd112019-04-30 10:36:17 -070011224 regs->r8 = kvm_r8_read(vcpu);
11225 regs->r9 = kvm_r9_read(vcpu);
11226 regs->r10 = kvm_r10_read(vcpu);
11227 regs->r11 = kvm_r11_read(vcpu);
11228 regs->r12 = kvm_r12_read(vcpu);
11229 regs->r13 = kvm_r13_read(vcpu);
11230 regs->r14 = kvm_r14_read(vcpu);
11231 regs->r15 = kvm_r15_read(vcpu);
Hollis Blanchardb6c7a5d2007-11-01 14:16:10 -050011232#endif
11233
Marcelo Tosatti5fdbf972008-06-27 14:58:02 -030011234 regs->rip = kvm_rip_read(vcpu);
Jan Kiszka91586a32009-10-05 13:07:21 +020011235 regs->rflags = kvm_get_rflags(vcpu);
Ken Hofsass01643c52018-01-31 16:03:36 -080011236}
Hollis Blanchardb6c7a5d2007-11-01 14:16:10 -050011237
Ken Hofsass01643c52018-01-31 16:03:36 -080011238int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
11239{
11240 vcpu_load(vcpu);
11241 __get_regs(vcpu, regs);
Christoffer Dall1fc9b762017-12-04 21:35:26 +010011242 vcpu_put(vcpu);
Hollis Blanchardb6c7a5d2007-11-01 14:16:10 -050011243 return 0;
11244}
11245
Ken Hofsass01643c52018-01-31 16:03:36 -080011246static void __set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
Hollis Blanchardb6c7a5d2007-11-01 14:16:10 -050011247{
Gleb Natapov7ae441e2011-03-31 12:06:41 +020011248 vcpu->arch.emulate_regs_need_sync_from_vcpu = true;
11249 vcpu->arch.emulate_regs_need_sync_to_vcpu = false;
11250
Sean Christophersonde3cd112019-04-30 10:36:17 -070011251 kvm_rax_write(vcpu, regs->rax);
11252 kvm_rbx_write(vcpu, regs->rbx);
11253 kvm_rcx_write(vcpu, regs->rcx);
11254 kvm_rdx_write(vcpu, regs->rdx);
11255 kvm_rsi_write(vcpu, regs->rsi);
11256 kvm_rdi_write(vcpu, regs->rdi);
Paolo Bonzinie9c16c72019-04-30 22:07:26 +020011257 kvm_rsp_write(vcpu, regs->rsp);
Sean Christophersonde3cd112019-04-30 10:36:17 -070011258 kvm_rbp_write(vcpu, regs->rbp);
Hollis Blanchardb6c7a5d2007-11-01 14:16:10 -050011259#ifdef CONFIG_X86_64
Sean Christophersonde3cd112019-04-30 10:36:17 -070011260 kvm_r8_write(vcpu, regs->r8);
11261 kvm_r9_write(vcpu, regs->r9);
11262 kvm_r10_write(vcpu, regs->r10);
11263 kvm_r11_write(vcpu, regs->r11);
11264 kvm_r12_write(vcpu, regs->r12);
11265 kvm_r13_write(vcpu, regs->r13);
11266 kvm_r14_write(vcpu, regs->r14);
11267 kvm_r15_write(vcpu, regs->r15);
Hollis Blanchardb6c7a5d2007-11-01 14:16:10 -050011268#endif
11269
Marcelo Tosatti5fdbf972008-06-27 14:58:02 -030011270 kvm_rip_write(vcpu, regs->rip);
Wanpeng Lid73235d2017-12-07 00:30:08 -080011271 kvm_set_rflags(vcpu, regs->rflags | X86_EFLAGS_FIXED);
Hollis Blanchardb6c7a5d2007-11-01 14:16:10 -050011272
Jan Kiszkab4f14ab2008-04-30 17:59:04 +020011273 vcpu->arch.exception.pending = false;
Sean Christopherson7709aba2022-08-30 23:16:08 +000011274 vcpu->arch.exception_vmexit.pending = false;
Jan Kiszkab4f14ab2008-04-30 17:59:04 +020011275
Avi Kivity3842d132010-07-27 12:30:24 +030011276 kvm_make_request(KVM_REQ_EVENT, vcpu);
Ken Hofsass01643c52018-01-31 16:03:36 -080011277}
Avi Kivity3842d132010-07-27 12:30:24 +030011278
Ken Hofsass01643c52018-01-31 16:03:36 -080011279int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
11280{
11281 vcpu_load(vcpu);
11282 __set_regs(vcpu, regs);
Christoffer Dall875656f2017-12-04 21:35:27 +010011283 vcpu_put(vcpu);
Hollis Blanchardb6c7a5d2007-11-01 14:16:10 -050011284 return 0;
11285}
11286
Maxim Levitsky6dba9402021-06-07 12:02:02 +030011287static void __get_sregs_common(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)
Hollis Blanchardb6c7a5d2007-11-01 14:16:10 -050011288{
Gleb Natapov89a27f42010-02-16 10:51:48 +020011289 struct desc_ptr dt;
Hollis Blanchardb6c7a5d2007-11-01 14:16:10 -050011290
Tom Lendacky52657132020-12-10 11:09:59 -060011291 if (vcpu->arch.guest_state_protected)
11292 goto skip_protected_regs;
11293
Guillaume Thouvenin3e6e0aa2008-05-27 10:18:46 +020011294 kvm_get_segment(vcpu, &sregs->cs, VCPU_SREG_CS);
11295 kvm_get_segment(vcpu, &sregs->ds, VCPU_SREG_DS);
11296 kvm_get_segment(vcpu, &sregs->es, VCPU_SREG_ES);
11297 kvm_get_segment(vcpu, &sregs->fs, VCPU_SREG_FS);
11298 kvm_get_segment(vcpu, &sregs->gs, VCPU_SREG_GS);
11299 kvm_get_segment(vcpu, &sregs->ss, VCPU_SREG_SS);
Hollis Blanchardb6c7a5d2007-11-01 14:16:10 -050011300
Guillaume Thouvenin3e6e0aa2008-05-27 10:18:46 +020011301 kvm_get_segment(vcpu, &sregs->tr, VCPU_SREG_TR);
11302 kvm_get_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);
Hollis Blanchardb6c7a5d2007-11-01 14:16:10 -050011303
Jason Baronb36464772021-01-14 22:27:56 -050011304 static_call(kvm_x86_get_idt)(vcpu, &dt);
Gleb Natapov89a27f42010-02-16 10:51:48 +020011305 sregs->idt.limit = dt.size;
11306 sregs->idt.base = dt.address;
Jason Baronb36464772021-01-14 22:27:56 -050011307 static_call(kvm_x86_get_gdt)(vcpu, &dt);
Gleb Natapov89a27f42010-02-16 10:51:48 +020011308 sregs->gdt.limit = dt.size;
11309 sregs->gdt.base = dt.address;
Hollis Blanchardb6c7a5d2007-11-01 14:16:10 -050011310
Zhang Xiantaoad312c72007-12-13 23:50:52 +080011311 sregs->cr2 = vcpu->arch.cr2;
Avi Kivity9f8fe502010-12-05 17:30:00 +020011312 sregs->cr3 = kvm_read_cr3(vcpu);
Tom Lendacky52657132020-12-10 11:09:59 -060011313
11314skip_protected_regs:
11315 sregs->cr0 = kvm_read_cr0(vcpu);
Avi Kivityfc78f512009-12-07 12:16:48 +020011316 sregs->cr4 = kvm_read_cr4(vcpu);
Avi Kivity2d3ad1f2008-02-24 11:20:43 +020011317 sregs->cr8 = kvm_get_cr8(vcpu);
Avi Kivityf6801df2010-01-21 15:31:50 +020011318 sregs->efer = vcpu->arch.efer;
Hollis Blanchardb6c7a5d2007-11-01 14:16:10 -050011319 sregs->apic_base = kvm_get_apic_base(vcpu);
Maxim Levitsky6dba9402021-06-07 12:02:02 +030011320}
Hollis Blanchardb6c7a5d2007-11-01 14:16:10 -050011321
Maxim Levitsky6dba9402021-06-07 12:02:02 +030011322static void __get_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)
11323{
11324 __get_sregs_common(vcpu, sregs);
11325
11326 if (vcpu->arch.guest_state_protected)
11327 return;
Hollis Blanchardb6c7a5d2007-11-01 14:16:10 -050011328
Liran Alon04140b42018-03-23 03:01:31 +030011329 if (vcpu->arch.interrupt.injected && !vcpu->arch.interrupt.soft)
Gleb Natapov14d0bc12009-04-21 17:45:11 +030011330 set_bit(vcpu->arch.interrupt.nr,
11331 (unsigned long *)sregs->interrupt_bitmap);
Ken Hofsass01643c52018-01-31 16:03:36 -080011332}
Gleb Natapov16d7a192009-04-21 17:45:10 +030011333
Maxim Levitsky6dba9402021-06-07 12:02:02 +030011334static void __get_sregs2(struct kvm_vcpu *vcpu, struct kvm_sregs2 *sregs2)
11335{
11336 int i;
11337
11338 __get_sregs_common(vcpu, (struct kvm_sregs *)sregs2);
11339
11340 if (vcpu->arch.guest_state_protected)
11341 return;
11342
11343 if (is_pae_paging(vcpu)) {
11344 for (i = 0 ; i < 4 ; i++)
11345 sregs2->pdptrs[i] = kvm_pdptr_read(vcpu, i);
11346 sregs2->flags |= KVM_SREGS2_FLAGS_PDPTRS_VALID;
11347 }
11348}
11349
Ken Hofsass01643c52018-01-31 16:03:36 -080011350int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
11351 struct kvm_sregs *sregs)
11352{
11353 vcpu_load(vcpu);
11354 __get_sregs(vcpu, sregs);
Christoffer Dallbcdec412017-12-04 21:35:28 +010011355 vcpu_put(vcpu);
Hollis Blanchardb6c7a5d2007-11-01 14:16:10 -050011356 return 0;
11357}
11358
Marcelo Tosatti62d9f0d2008-04-11 13:24:45 -030011359int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
11360 struct kvm_mp_state *mp_state)
11361{
Jim Mattson4fe09bc2021-06-04 10:26:04 -070011362 int r;
11363
Christoffer Dallfd232562017-12-04 21:35:30 +010011364 vcpu_load(vcpu);
Sean Christophersonf958bd22019-12-09 12:19:31 -080011365 if (kvm_mpx_supported())
11366 kvm_load_guest_fpu(vcpu);
Christoffer Dallfd232562017-12-04 21:35:30 +010011367
Jim Mattson4fe09bc2021-06-04 10:26:04 -070011368 r = kvm_apic_accept_events(vcpu);
11369 if (r < 0)
11370 goto out;
11371 r = 0;
11372
Tom Lendacky647daca2021-01-04 14:20:01 -060011373 if ((vcpu->arch.mp_state == KVM_MP_STATE_HALTED ||
11374 vcpu->arch.mp_state == KVM_MP_STATE_AP_RESET_HOLD) &&
11375 vcpu->arch.pv.pv_unhalted)
Srivatsa Vaddagiri6aef2662013-08-26 14:18:34 +053011376 mp_state->mp_state = KVM_MP_STATE_RUNNABLE;
11377 else
11378 mp_state->mp_state = vcpu->arch.mp_state;
11379
Jim Mattson4fe09bc2021-06-04 10:26:04 -070011380out:
Sean Christophersonf958bd22019-12-09 12:19:31 -080011381 if (kvm_mpx_supported())
11382 kvm_put_guest_fpu(vcpu);
Christoffer Dallfd232562017-12-04 21:35:30 +010011383 vcpu_put(vcpu);
Jim Mattson4fe09bc2021-06-04 10:26:04 -070011384 return r;
Marcelo Tosatti62d9f0d2008-04-11 13:24:45 -030011385}
11386
11387int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
11388 struct kvm_mp_state *mp_state)
11389{
Christoffer Dalle83dff52017-12-04 21:35:31 +010011390 int ret = -EINVAL;
11391
11392 vcpu_load(vcpu);
11393
Paolo Bonzini22c6a0e2022-08-11 12:41:25 -040011394 switch (mp_state->mp_state) {
11395 case KVM_MP_STATE_UNINITIALIZED:
11396 case KVM_MP_STATE_HALTED:
11397 case KVM_MP_STATE_AP_RESET_HOLD:
11398 case KVM_MP_STATE_INIT_RECEIVED:
11399 case KVM_MP_STATE_SIPI_RECEIVED:
11400 if (!lapic_in_kernel(vcpu))
11401 goto out;
11402 break;
11403
11404 case KVM_MP_STATE_RUNNABLE:
11405 break;
11406
11407 default:
Christoffer Dalle83dff52017-12-04 21:35:31 +010011408 goto out;
Paolo Bonzini22c6a0e2022-08-11 12:41:25 -040011409 }
Jan Kiszka66450a22013-03-13 12:42:34 +010011410
Liran Alon27cbe7d2019-11-11 11:16:40 +020011411 /*
Sean Christopherson1b7a1b72022-09-21 00:31:52 +000011412 * Pending INITs are reported using KVM_SET_VCPU_EVENTS, disallow
11413 * forcing the guest into INIT/SIPI if those events are supposed to be
11414 * blocked. KVM prioritizes SMI over INIT, so reject INIT/SIPI state
11415 * if an SMI is pending as well.
Liran Alon27cbe7d2019-11-11 11:16:40 +020011416 */
Sean Christopherson1b7a1b72022-09-21 00:31:52 +000011417 if ((!kvm_apic_init_sipi_allowed(vcpu) || vcpu->arch.smi_pending) &&
David Hildenbrand28bf2882017-03-23 11:46:03 +010011418 (mp_state->mp_state == KVM_MP_STATE_SIPI_RECEIVED ||
11419 mp_state->mp_state == KVM_MP_STATE_INIT_RECEIVED))
Christoffer Dalle83dff52017-12-04 21:35:31 +010011420 goto out;
David Hildenbrand28bf2882017-03-23 11:46:03 +010011421
Jan Kiszka66450a22013-03-13 12:42:34 +010011422 if (mp_state->mp_state == KVM_MP_STATE_SIPI_RECEIVED) {
11423 vcpu->arch.mp_state = KVM_MP_STATE_INIT_RECEIVED;
11424 set_bit(KVM_APIC_SIPI, &vcpu->arch.apic->pending_events);
11425 } else
11426 vcpu->arch.mp_state = mp_state->mp_state;
Avi Kivity3842d132010-07-27 12:30:24 +030011427 kvm_make_request(KVM_REQ_EVENT, vcpu);
Christoffer Dalle83dff52017-12-04 21:35:31 +010011428
11429 ret = 0;
11430out:
11431 vcpu_put(vcpu);
11432 return ret;
Marcelo Tosatti62d9f0d2008-04-11 13:24:45 -030011433}
11434
Kevin Wolf7f3d35f2012-02-08 14:34:38 +010011435int kvm_task_switch(struct kvm_vcpu *vcpu, u16 tss_selector, int idt_index,
11436 int reason, bool has_error_code, u32 error_code)
Hollis Blanchardb6c7a5d2007-11-01 14:16:10 -050011437{
Sean Christophersonc9b8b072020-02-18 15:29:48 -080011438 struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;
Mohammed Gamal8ec47222010-08-16 00:47:01 +030011439 int ret;
Hollis Blanchardb6c7a5d2007-11-01 14:16:10 -050011440
Mohammed Gamal8ec47222010-08-16 00:47:01 +030011441 init_emulate_ctxt(vcpu);
Izik Eidus37817f22008-03-24 23:14:53 +020011442
Kevin Wolf7f3d35f2012-02-08 14:34:38 +010011443 ret = emulator_task_switch(ctxt, tss_selector, idt_index, reason,
Takuya Yoshikawa9d741912011-05-29 21:53:48 +090011444 has_error_code, error_code);
Sean Christopherson10517782019-08-27 14:40:35 -070011445 if (ret) {
11446 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
11447 vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION;
11448 vcpu->run->internal.ndata = 0;
Sean Christopherson60fc3d02019-08-27 14:40:38 -070011449 return 0;
Sean Christopherson10517782019-08-27 14:40:35 -070011450 }
Gleb Natapovc6975182010-02-18 12:15:01 +020011451
Takuya Yoshikawa9d741912011-05-29 21:53:48 +090011452 kvm_rip_write(vcpu, ctxt->eip);
11453 kvm_set_rflags(vcpu, ctxt->eflags);
Sean Christopherson60fc3d02019-08-27 14:40:38 -070011454 return 1;
Izik Eidus37817f22008-03-24 23:14:53 +020011455}
11456EXPORT_SYMBOL_GPL(kvm_task_switch);
11457
Sean Christophersonee69c922020-10-06 18:44:16 -070011458static bool kvm_is_valid_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)
Lan Tianyuf2981032017-12-14 03:01:52 -050011459{
Tianyu Lan37b95952018-01-16 17:34:07 +080011460 if ((sregs->efer & EFER_LME) && (sregs->cr0 & X86_CR0_PG)) {
Lan Tianyuf2981032017-12-14 03:01:52 -050011461 /*
11462 * When EFER.LME and CR0.PG are set, the processor is in
11463 * 64-bit mode (though maybe in a 32-bit code segment).
11464 * CR4.PAE and EFER.LMA must be set.
11465 */
Sean Christophersonee69c922020-10-06 18:44:16 -070011466 if (!(sregs->cr4 & X86_CR4_PAE) || !(sregs->efer & EFER_LMA))
11467 return false;
Sean Christophersonca29e142021-02-03 16:01:12 -080011468 if (kvm_vcpu_is_illegal_gpa(vcpu, sregs->cr3))
Paolo Bonzinic1c35cf2020-11-13 08:30:38 -050011469 return false;
Lan Tianyuf2981032017-12-14 03:01:52 -050011470 } else {
11471 /*
11472 * Not in 64-bit mode: EFER.LMA is clear and the code
11473 * segment cannot be 64-bit.
11474 */
11475 if (sregs->efer & EFER_LMA || sregs->cs.l)
Sean Christophersonee69c922020-10-06 18:44:16 -070011476 return false;
Lan Tianyuf2981032017-12-14 03:01:52 -050011477 }
11478
Sean Christophersonee69c922020-10-06 18:44:16 -070011479 return kvm_is_valid_cr4(vcpu, sregs->cr4);
Lan Tianyuf2981032017-12-14 03:01:52 -050011480}
11481
Maxim Levitsky6dba9402021-06-07 12:02:02 +030011482static int __set_sregs_common(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs,
11483 int *mmu_reset_needed, bool update_pdptrs)
Hollis Blanchardb6c7a5d2007-11-01 14:16:10 -050011484{
Jan Kiszka58cb6282014-01-24 16:48:44 +010011485 struct msr_data apic_base_msr;
Maxim Levitsky6dba9402021-06-07 12:02:02 +030011486 int idx;
Gleb Natapov89a27f42010-02-16 10:51:48 +020011487 struct desc_ptr dt;
Christoffer Dallb4ef9d42017-12-04 21:35:29 +010011488
Sean Christophersonee69c922020-10-06 18:44:16 -070011489 if (!kvm_is_valid_sregs(vcpu, sregs))
Maxim Levitsky6dba9402021-06-07 12:02:02 +030011490 return -EINVAL;
Lan Tianyuf2981032017-12-14 03:01:52 -050011491
Jim Mattsond3802282017-08-10 10:14:13 -070011492 apic_base_msr.data = sregs->apic_base;
11493 apic_base_msr.host_initiated = true;
11494 if (kvm_set_apic_base(vcpu, &apic_base_msr))
Maxim Levitsky6dba9402021-06-07 12:02:02 +030011495 return -EINVAL;
Petr Matousek6d1068b2012-11-06 19:24:07 +010011496
Tom Lendacky52657132020-12-10 11:09:59 -060011497 if (vcpu->arch.guest_state_protected)
Maxim Levitsky6dba9402021-06-07 12:02:02 +030011498 return 0;
Tom Lendacky52657132020-12-10 11:09:59 -060011499
Gleb Natapov89a27f42010-02-16 10:51:48 +020011500 dt.size = sregs->idt.limit;
11501 dt.address = sregs->idt.base;
Jason Baronb36464772021-01-14 22:27:56 -050011502 static_call(kvm_x86_set_idt)(vcpu, &dt);
Gleb Natapov89a27f42010-02-16 10:51:48 +020011503 dt.size = sregs->gdt.limit;
11504 dt.address = sregs->gdt.base;
Jason Baronb36464772021-01-14 22:27:56 -050011505 static_call(kvm_x86_set_gdt)(vcpu, &dt);
Hollis Blanchardb6c7a5d2007-11-01 14:16:10 -050011506
Zhang Xiantaoad312c72007-12-13 23:50:52 +080011507 vcpu->arch.cr2 = sregs->cr2;
Maxim Levitsky6dba9402021-06-07 12:02:02 +030011508 *mmu_reset_needed |= kvm_read_cr3(vcpu) != sregs->cr3;
Jan Kiszkadc7e7952009-07-01 20:52:03 +020011509 vcpu->arch.cr3 = sregs->cr3;
Lai Jiangshan3883bc9d2021-11-08 20:44:02 +080011510 kvm_register_mark_dirty(vcpu, VCPU_EXREG_CR3);
Michael Roth405329f2021-12-16 11:13:54 -060011511 static_call_cond(kvm_x86_post_set_cr3)(vcpu, sregs->cr3);
Hollis Blanchardb6c7a5d2007-11-01 14:16:10 -050011512
Avi Kivity2d3ad1f2008-02-24 11:20:43 +020011513 kvm_set_cr8(vcpu, sregs->cr8);
Hollis Blanchardb6c7a5d2007-11-01 14:16:10 -050011514
Maxim Levitsky6dba9402021-06-07 12:02:02 +030011515 *mmu_reset_needed |= vcpu->arch.efer != sregs->efer;
Jason Baronb36464772021-01-14 22:27:56 -050011516 static_call(kvm_x86_set_efer)(vcpu, sregs->efer);
Hollis Blanchardb6c7a5d2007-11-01 14:16:10 -050011517
Maxim Levitsky6dba9402021-06-07 12:02:02 +030011518 *mmu_reset_needed |= kvm_read_cr0(vcpu) != sregs->cr0;
Jason Baronb36464772021-01-14 22:27:56 -050011519 static_call(kvm_x86_set_cr0)(vcpu, sregs->cr0);
Paul Knowlesd7306162008-02-06 11:02:35 +000011520 vcpu->arch.cr0 = sregs->cr0;
Hollis Blanchardb6c7a5d2007-11-01 14:16:10 -050011521
Maxim Levitsky6dba9402021-06-07 12:02:02 +030011522 *mmu_reset_needed |= kvm_read_cr4(vcpu) != sregs->cr4;
Jason Baronb36464772021-01-14 22:27:56 -050011523 static_call(kvm_x86_set_cr4)(vcpu, sregs->cr4);
Xiao Guangrong63f42e02011-01-12 15:39:18 +080011524
Maxim Levitsky6dba9402021-06-07 12:02:02 +030011525 if (update_pdptrs) {
11526 idx = srcu_read_lock(&vcpu->kvm->srcu);
11527 if (is_pae_paging(vcpu)) {
Lai Jiangshan2df4a5e2021-11-24 20:20:52 +080011528 load_pdptrs(vcpu, kvm_read_cr3(vcpu));
Maxim Levitsky6dba9402021-06-07 12:02:02 +030011529 *mmu_reset_needed = 1;
11530 }
11531 srcu_read_unlock(&vcpu->kvm->srcu, idx);
Marcelo Tosatti7c93be442009-10-26 16:48:33 -020011532 }
Hollis Blanchardb6c7a5d2007-11-01 14:16:10 -050011533
Guillaume Thouvenin3e6e0aa2008-05-27 10:18:46 +020011534 kvm_set_segment(vcpu, &sregs->cs, VCPU_SREG_CS);
11535 kvm_set_segment(vcpu, &sregs->ds, VCPU_SREG_DS);
11536 kvm_set_segment(vcpu, &sregs->es, VCPU_SREG_ES);
11537 kvm_set_segment(vcpu, &sregs->fs, VCPU_SREG_FS);
11538 kvm_set_segment(vcpu, &sregs->gs, VCPU_SREG_GS);
11539 kvm_set_segment(vcpu, &sregs->ss, VCPU_SREG_SS);
Hollis Blanchardb6c7a5d2007-11-01 14:16:10 -050011540
Guillaume Thouvenin3e6e0aa2008-05-27 10:18:46 +020011541 kvm_set_segment(vcpu, &sregs->tr, VCPU_SREG_TR);
11542 kvm_set_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);
Hollis Blanchardb6c7a5d2007-11-01 14:16:10 -050011543
Mikhail Ershov5f0269f2009-08-03 14:58:25 +030011544 update_cr8_intercept(vcpu);
11545
Marcelo Tosatti9c3e4aa2008-09-10 16:40:55 -030011546 /* Older userspace won't unhalt the vcpu on reset. */
Gleb Natapovc5af89b2009-06-09 15:56:26 +030011547 if (kvm_vcpu_is_bsp(vcpu) && kvm_rip_read(vcpu) == 0xfff0 &&
Marcelo Tosatti9c3e4aa2008-09-10 16:40:55 -030011548 sregs->cs.selector == 0xf000 && sregs->cs.base == 0xffff0000 &&
Avi Kivity3eeb3282010-01-21 15:31:48 +020011549 !is_protmode(vcpu))
Marcelo Tosatti9c3e4aa2008-09-10 16:40:55 -030011550 vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
11551
Maxim Levitsky6dba9402021-06-07 12:02:02 +030011552 return 0;
11553}
11554
11555static int __set_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)
11556{
11557 int pending_vec, max_bits;
11558 int mmu_reset_needed = 0;
11559 int ret = __set_sregs_common(vcpu, sregs, &mmu_reset_needed, true);
11560
11561 if (ret)
11562 return ret;
11563
11564 if (mmu_reset_needed)
11565 kvm_mmu_reset_context(vcpu);
11566
Tom Lendacky52657132020-12-10 11:09:59 -060011567 max_bits = KVM_NR_INTERRUPTS;
11568 pending_vec = find_first_bit(
11569 (const unsigned long *)sregs->interrupt_bitmap, max_bits);
Maxim Levitsky6dba9402021-06-07 12:02:02 +030011570
Tom Lendacky52657132020-12-10 11:09:59 -060011571 if (pending_vec < max_bits) {
11572 kvm_queue_interrupt(vcpu, pending_vec, false);
11573 pr_debug("Set back pending irq %d\n", pending_vec);
Maxim Levitsky6dba9402021-06-07 12:02:02 +030011574 kvm_make_request(KVM_REQ_EVENT, vcpu);
Tom Lendacky52657132020-12-10 11:09:59 -060011575 }
Maxim Levitsky6dba9402021-06-07 12:02:02 +030011576 return 0;
11577}
Tom Lendacky52657132020-12-10 11:09:59 -060011578
Maxim Levitsky6dba9402021-06-07 12:02:02 +030011579static int __set_sregs2(struct kvm_vcpu *vcpu, struct kvm_sregs2 *sregs2)
11580{
11581 int mmu_reset_needed = 0;
11582 bool valid_pdptrs = sregs2->flags & KVM_SREGS2_FLAGS_PDPTRS_VALID;
11583 bool pae = (sregs2->cr0 & X86_CR0_PG) && (sregs2->cr4 & X86_CR4_PAE) &&
11584 !(sregs2->efer & EFER_LMA);
11585 int i, ret;
Avi Kivity3842d132010-07-27 12:30:24 +030011586
Maxim Levitsky6dba9402021-06-07 12:02:02 +030011587 if (sregs2->flags & ~KVM_SREGS2_FLAGS_PDPTRS_VALID)
11588 return -EINVAL;
11589
11590 if (valid_pdptrs && (!pae || vcpu->arch.guest_state_protected))
11591 return -EINVAL;
11592
11593 ret = __set_sregs_common(vcpu, (struct kvm_sregs *)sregs2,
11594 &mmu_reset_needed, !valid_pdptrs);
11595 if (ret)
11596 return ret;
11597
11598 if (valid_pdptrs) {
11599 for (i = 0; i < 4 ; i++)
11600 kvm_pdptr_write(vcpu, i, sregs2->pdptrs[i]);
11601
11602 kvm_register_mark_dirty(vcpu, VCPU_EXREG_PDPTR);
11603 mmu_reset_needed = 1;
Maxim Levitsky158a48e2021-06-07 12:02:03 +030011604 vcpu->arch.pdptrs_from_userspace = true;
Maxim Levitsky6dba9402021-06-07 12:02:02 +030011605 }
11606 if (mmu_reset_needed)
11607 kvm_mmu_reset_context(vcpu);
11608 return 0;
Ken Hofsass01643c52018-01-31 16:03:36 -080011609}
11610
11611int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
11612 struct kvm_sregs *sregs)
11613{
11614 int ret;
11615
11616 vcpu_load(vcpu);
11617 ret = __set_sregs(vcpu, sregs);
Christoffer Dallb4ef9d42017-12-04 21:35:29 +010011618 vcpu_put(vcpu);
11619 return ret;
Hollis Blanchardb6c7a5d2007-11-01 14:16:10 -050011620}
11621
Maxim Levitskycae72dc2021-11-08 11:02:45 +020011622static void kvm_arch_vcpu_guestdbg_update_apicv_inhibit(struct kvm *kvm)
11623{
Sean Christopherson320af552022-03-11 04:35:16 +000011624 bool set = false;
Maxim Levitskycae72dc2021-11-08 11:02:45 +020011625 struct kvm_vcpu *vcpu;
Marc Zyngier46808a42021-11-16 16:04:02 +000011626 unsigned long i;
Maxim Levitskycae72dc2021-11-08 11:02:45 +020011627
Sean Christopherson0047fb332022-04-20 01:37:32 +000011628 if (!enable_apicv)
11629 return;
11630
Maxim Levitskycae72dc2021-11-08 11:02:45 +020011631 down_write(&kvm->arch.apicv_update_lock);
11632
11633 kvm_for_each_vcpu(i, vcpu, kvm) {
11634 if (vcpu->guest_debug & KVM_GUESTDBG_BLOCKIRQ) {
Sean Christopherson320af552022-03-11 04:35:16 +000011635 set = true;
Maxim Levitskycae72dc2021-11-08 11:02:45 +020011636 break;
11637 }
11638 }
Sean Christopherson320af552022-03-11 04:35:16 +000011639 __kvm_set_or_clear_apicv_inhibit(kvm, APICV_INHIBIT_REASON_BLOCKIRQ, set);
Maxim Levitskycae72dc2021-11-08 11:02:45 +020011640 up_write(&kvm->arch.apicv_update_lock);
11641}
11642
Jan Kiszkad0bfb942008-12-15 13:52:10 +010011643int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
11644 struct kvm_guest_debug *dbg)
Hollis Blanchardb6c7a5d2007-11-01 14:16:10 -050011645{
Jan Kiszka355be0b2009-10-03 00:31:21 +020011646 unsigned long rflags;
Jan Kiszkaae675ef2008-12-15 13:52:10 +010011647 int i, r;
Hollis Blanchardb6c7a5d2007-11-01 14:16:10 -050011648
Tom Lendacky8d4846b2020-12-10 11:09:43 -060011649 if (vcpu->arch.guest_state_protected)
11650 return -EINVAL;
11651
Christoffer Dall66b56562017-12-04 21:35:33 +010011652 vcpu_load(vcpu);
11653
Jan Kiszka4f926bf22009-10-30 12:46:59 +010011654 if (dbg->control & (KVM_GUESTDBG_INJECT_DB | KVM_GUESTDBG_INJECT_BP)) {
11655 r = -EBUSY;
Sean Christopherson7709aba2022-08-30 23:16:08 +000011656 if (kvm_is_exception_pending(vcpu))
Avi Kivity2122ff52010-05-13 11:25:04 +030011657 goto out;
Jan Kiszka4f926bf22009-10-30 12:46:59 +010011658 if (dbg->control & KVM_GUESTDBG_INJECT_DB)
11659 kvm_queue_exception(vcpu, DB_VECTOR);
11660 else
11661 kvm_queue_exception(vcpu, BP_VECTOR);
11662 }
11663
Jan Kiszka91586a32009-10-05 13:07:21 +020011664 /*
11665 * Read rflags as long as potentially injected trace flags are still
11666 * filtered out.
11667 */
11668 rflags = kvm_get_rflags(vcpu);
Jan Kiszka355be0b2009-10-03 00:31:21 +020011669
11670 vcpu->guest_debug = dbg->control;
11671 if (!(vcpu->guest_debug & KVM_GUESTDBG_ENABLE))
11672 vcpu->guest_debug = 0;
11673
11674 if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) {
Jan Kiszkaae675ef2008-12-15 13:52:10 +010011675 for (i = 0; i < KVM_NR_DB_REGS; ++i)
11676 vcpu->arch.eff_db[i] = dbg->arch.debugreg[i];
Jan Kiszkac8639012012-09-21 05:42:55 +020011677 vcpu->arch.guest_debug_dr7 = dbg->arch.debugreg[7];
Jan Kiszkaae675ef2008-12-15 13:52:10 +010011678 } else {
11679 for (i = 0; i < KVM_NR_DB_REGS; i++)
11680 vcpu->arch.eff_db[i] = vcpu->arch.db[i];
Jan Kiszkaae675ef2008-12-15 13:52:10 +010011681 }
Jan Kiszkac8639012012-09-21 05:42:55 +020011682 kvm_update_dr7(vcpu);
Jan Kiszkaae675ef2008-12-15 13:52:10 +010011683
Jan Kiszkaf92653e2010-02-23 17:47:55 +010011684 if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP)
Yuan Yaoe87e46d2021-05-26 14:38:28 +080011685 vcpu->arch.singlestep_rip = kvm_get_linear_rip(vcpu);
Hollis Blanchardb6c7a5d2007-11-01 14:16:10 -050011686
Jan Kiszka91586a32009-10-05 13:07:21 +020011687 /*
11688 * Trigger an rflags update that will inject or remove the trace
11689 * flags.
11690 */
11691 kvm_set_rflags(vcpu, rflags);
Jan Kiszkad0bfb942008-12-15 13:52:10 +010011692
Jason Baronb36464772021-01-14 22:27:56 -050011693 static_call(kvm_x86_update_exception_bitmap)(vcpu);
Jan Kiszka355be0b2009-10-03 00:31:21 +020011694
Maxim Levitskycae72dc2021-11-08 11:02:45 +020011695 kvm_arch_vcpu_guestdbg_update_apicv_inhibit(vcpu->kvm);
11696
Jan Kiszka4f926bf22009-10-30 12:46:59 +010011697 r = 0;
Hollis Blanchardb6c7a5d2007-11-01 14:16:10 -050011698
Avi Kivity2122ff52010-05-13 11:25:04 +030011699out:
Christoffer Dall66b56562017-12-04 21:35:33 +010011700 vcpu_put(vcpu);
Hollis Blanchardb6c7a5d2007-11-01 14:16:10 -050011701 return r;
11702}
11703
11704/*
Zhang Xiantao8b00679132007-11-16 13:05:55 +080011705 * Translate a guest virtual address to a guest physical address.
11706 */
11707int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
11708 struct kvm_translation *tr)
11709{
11710 unsigned long vaddr = tr->linear_address;
11711 gpa_t gpa;
Marcelo Tosattif656ce02009-12-23 14:35:25 -020011712 int idx;
Zhang Xiantao8b00679132007-11-16 13:05:55 +080011713
Christoffer Dall1da5b612017-12-04 21:35:32 +010011714 vcpu_load(vcpu);
11715
Marcelo Tosattif656ce02009-12-23 14:35:25 -020011716 idx = srcu_read_lock(&vcpu->kvm->srcu);
Gleb Natapov1871c602010-02-10 14:21:32 +020011717 gpa = kvm_mmu_gva_to_gpa_system(vcpu, vaddr, NULL);
Marcelo Tosattif656ce02009-12-23 14:35:25 -020011718 srcu_read_unlock(&vcpu->kvm->srcu, idx);
Zhang Xiantao8b00679132007-11-16 13:05:55 +080011719 tr->physical_address = gpa;
Hou Wenlong6e1d2a32022-07-01 17:24:13 +080011720 tr->valid = gpa != INVALID_GPA;
Zhang Xiantao8b00679132007-11-16 13:05:55 +080011721 tr->writeable = 1;
11722 tr->usermode = 0;
Zhang Xiantao8b00679132007-11-16 13:05:55 +080011723
Christoffer Dall1da5b612017-12-04 21:35:32 +010011724 vcpu_put(vcpu);
Zhang Xiantao8b00679132007-11-16 13:05:55 +080011725 return 0;
11726}
11727
Hollis Blanchardd0752062007-10-31 17:24:25 -050011728int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
11729{
Christoffer Dall13931232017-12-04 21:35:34 +010011730 struct fxregs_state *fxsave;
Hollis Blanchardd0752062007-10-31 17:24:25 -050011731
Thomas Gleixnerd69c1382021-10-22 20:55:53 +020011732 if (fpstate_is_confidential(&vcpu->arch.guest_fpu))
Tom Lendackyed02b212020-12-10 11:10:01 -060011733 return 0;
11734
Christoffer Dall13931232017-12-04 21:35:34 +010011735 vcpu_load(vcpu);
11736
Thomas Gleixnerd69c1382021-10-22 20:55:53 +020011737 fxsave = &vcpu->arch.guest_fpu.fpstate->regs.fxsave;
Hollis Blanchardd0752062007-10-31 17:24:25 -050011738 memcpy(fpu->fpr, fxsave->st_space, 128);
11739 fpu->fcw = fxsave->cwd;
11740 fpu->fsw = fxsave->swd;
11741 fpu->ftwx = fxsave->twd;
11742 fpu->last_opcode = fxsave->fop;
11743 fpu->last_ip = fxsave->rip;
11744 fpu->last_dp = fxsave->rdp;
Jordan Borgner0e96f312018-10-28 12:58:28 +000011745 memcpy(fpu->xmm, fxsave->xmm_space, sizeof(fxsave->xmm_space));
Hollis Blanchardd0752062007-10-31 17:24:25 -050011746
Christoffer Dall13931232017-12-04 21:35:34 +010011747 vcpu_put(vcpu);
Hollis Blanchardd0752062007-10-31 17:24:25 -050011748 return 0;
11749}
11750
11751int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
11752{
Christoffer Dall6a96bc72017-12-04 21:35:35 +010011753 struct fxregs_state *fxsave;
11754
Thomas Gleixnerd69c1382021-10-22 20:55:53 +020011755 if (fpstate_is_confidential(&vcpu->arch.guest_fpu))
Tom Lendackyed02b212020-12-10 11:10:01 -060011756 return 0;
11757
Christoffer Dall6a96bc72017-12-04 21:35:35 +010011758 vcpu_load(vcpu);
11759
Thomas Gleixnerd69c1382021-10-22 20:55:53 +020011760 fxsave = &vcpu->arch.guest_fpu.fpstate->regs.fxsave;
Hollis Blanchardd0752062007-10-31 17:24:25 -050011761
Hollis Blanchardd0752062007-10-31 17:24:25 -050011762 memcpy(fxsave->st_space, fpu->fpr, 128);
11763 fxsave->cwd = fpu->fcw;
11764 fxsave->swd = fpu->fsw;
11765 fxsave->twd = fpu->ftwx;
11766 fxsave->fop = fpu->last_opcode;
11767 fxsave->rip = fpu->last_ip;
11768 fxsave->rdp = fpu->last_dp;
Jordan Borgner0e96f312018-10-28 12:58:28 +000011769 memcpy(fxsave->xmm_space, fpu->xmm, sizeof(fxsave->xmm_space));
Hollis Blanchardd0752062007-10-31 17:24:25 -050011770
Christoffer Dall6a96bc72017-12-04 21:35:35 +010011771 vcpu_put(vcpu);
Hollis Blanchardd0752062007-10-31 17:24:25 -050011772 return 0;
11773}
11774
Ken Hofsass01643c52018-01-31 16:03:36 -080011775static void store_regs(struct kvm_vcpu *vcpu)
11776{
11777 BUILD_BUG_ON(sizeof(struct kvm_sync_regs) > SYNC_REGS_SIZE_BYTES);
11778
11779 if (vcpu->run->kvm_valid_regs & KVM_SYNC_X86_REGS)
11780 __get_regs(vcpu, &vcpu->run->s.regs.regs);
11781
11782 if (vcpu->run->kvm_valid_regs & KVM_SYNC_X86_SREGS)
11783 __get_sregs(vcpu, &vcpu->run->s.regs.sregs);
11784
11785 if (vcpu->run->kvm_valid_regs & KVM_SYNC_X86_EVENTS)
11786 kvm_vcpu_ioctl_x86_get_vcpu_events(
11787 vcpu, &vcpu->run->s.regs.events);
11788}
11789
11790static int sync_regs(struct kvm_vcpu *vcpu)
11791{
Ken Hofsass01643c52018-01-31 16:03:36 -080011792 if (vcpu->run->kvm_dirty_regs & KVM_SYNC_X86_REGS) {
11793 __set_regs(vcpu, &vcpu->run->s.regs.regs);
11794 vcpu->run->kvm_dirty_regs &= ~KVM_SYNC_X86_REGS;
11795 }
11796 if (vcpu->run->kvm_dirty_regs & KVM_SYNC_X86_SREGS) {
11797 if (__set_sregs(vcpu, &vcpu->run->s.regs.sregs))
11798 return -EINVAL;
11799 vcpu->run->kvm_dirty_regs &= ~KVM_SYNC_X86_SREGS;
11800 }
11801 if (vcpu->run->kvm_dirty_regs & KVM_SYNC_X86_EVENTS) {
11802 if (kvm_vcpu_ioctl_x86_set_vcpu_events(
11803 vcpu, &vcpu->run->s.regs.events))
11804 return -EINVAL;
11805 vcpu->run->kvm_dirty_regs &= ~KVM_SYNC_X86_EVENTS;
11806 }
11807
11808 return 0;
11809}
11810
Sean Christopherson897cc382019-12-18 13:55:09 -080011811int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id)
Zhang Xiantaoe9b11c12007-11-14 20:38:21 +080011812{
Zeng Guang1d5e7402022-04-19 23:44:09 +080011813 if (kvm_check_tsc_unstable() && kvm->created_vcpus)
Sean Christopherson897cc382019-12-18 13:55:09 -080011814 pr_warn_once("kvm: SMP vm created on host with unstable TSC; "
11815 "guest TSC will not be reliable\n");
Liang Lic447e762015-05-21 04:41:25 +080011816
Zeng Guang35875312022-04-19 23:44:44 +080011817 if (!kvm->arch.max_vcpu_ids)
11818 kvm->arch.max_vcpu_ids = KVM_MAX_VCPU_IDS;
11819
11820 if (id >= kvm->arch.max_vcpu_ids)
11821 return -EINVAL;
11822
Chao Gaod588bb92022-04-19 23:45:10 +080011823 return static_call(kvm_x86_vcpu_precreate)(kvm);
Avi Kivity26e52152007-11-20 15:30:24 +020011824}
Zhang Xiantaoe9b11c12007-11-14 20:38:21 +080011825
Sean Christophersone529ef62019-12-18 13:55:15 -080011826int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu)
Avi Kivity26e52152007-11-20 15:30:24 +020011827{
Sean Christopherson95a0d012019-12-18 13:55:23 -080011828 struct page *page;
11829 int r;
Zhang Xiantaoe9b11c12007-11-14 20:38:21 +080011830
Sean Christopherson63f5a192021-06-22 10:56:52 -070011831 vcpu->arch.last_vmentry_cpu = -1;
Sean Christopherson71170032021-09-20 17:02:54 -070011832 vcpu->arch.regs_avail = ~0;
11833 vcpu->arch.regs_dirty = ~0;
Sean Christopherson63f5a192021-06-22 10:56:52 -070011834
Michal Luczaj52491a382022-10-13 21:12:19 +000011835 kvm_gpc_init(&vcpu->arch.pv_time);
11836
Sean Christopherson95a0d012019-12-18 13:55:23 -080011837 if (!irqchip_in_kernel(vcpu->kvm) || kvm_vcpu_is_reset_bsp(vcpu))
11838 vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
11839 else
11840 vcpu->arch.mp_state = KVM_MP_STATE_UNINITIALIZED;
11841
Sean Christopherson95a0d012019-12-18 13:55:23 -080011842 r = kvm_mmu_create(vcpu);
11843 if (r < 0)
11844 return r;
11845
11846 if (irqchip_in_kernel(vcpu->kvm)) {
Sean Christopherson95a0d012019-12-18 13:55:23 -080011847 r = kvm_create_lapic(vcpu, lapic_timer_advance_ns);
11848 if (r < 0)
11849 goto fail_mmu_destroy;
Sean Christopherson423ecfe2022-04-20 01:37:31 +000011850
11851 /*
11852 * Defer evaluating inhibits until the vCPU is first run, as
11853 * this vCPU will not get notified of any changes until this
11854 * vCPU is visible to other vCPUs (marked online and added to
11855 * the set of vCPUs). Opportunistically mark APICv active as
11856 * VMX in particularly is highly unlikely to have inhibits.
11857 * Ignore the current per-VM APICv state so that vCPU creation
11858 * is guaranteed to run with a deterministic value, the request
11859 * will ensure the vCPU gets the correct state before VM-Entry.
11860 */
11861 if (enable_apicv) {
Sean Christophersonce0a58f42022-06-14 23:05:47 +000011862 vcpu->arch.apic->apicv_active = true;
Sean Christopherson423ecfe2022-04-20 01:37:31 +000011863 kvm_make_request(KVM_REQ_APICV_UPDATE, vcpu);
11864 }
Sean Christopherson95a0d012019-12-18 13:55:23 -080011865 } else
Cun Li6e4e3b42021-01-11 23:24:35 +080011866 static_branch_inc(&kvm_has_noapic_vcpu);
Sean Christopherson95a0d012019-12-18 13:55:23 -080011867
11868 r = -ENOMEM;
11869
Shakeel Butt93bb59c2020-12-18 14:01:38 -080011870 page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO);
Sean Christopherson95a0d012019-12-18 13:55:23 -080011871 if (!page)
11872 goto fail_free_lapic;
11873 vcpu->arch.pio_data = page_address(page);
11874
Jue Wang087acc42022-06-10 10:11:31 -070011875 vcpu->arch.mce_banks = kcalloc(KVM_MAX_MCE_BANKS * 4, sizeof(u64),
Sean Christopherson95a0d012019-12-18 13:55:23 -080011876 GFP_KERNEL_ACCOUNT);
Jue Wang281b5272022-06-10 10:11:32 -070011877 vcpu->arch.mci_ctl2_banks = kcalloc(KVM_MAX_MCE_BANKS, sizeof(u64),
11878 GFP_KERNEL_ACCOUNT);
11879 if (!vcpu->arch.mce_banks || !vcpu->arch.mci_ctl2_banks)
Miaohe Lin3c0ba052022-09-01 20:23:00 +080011880 goto fail_free_mce_banks;
Sean Christopherson95a0d012019-12-18 13:55:23 -080011881 vcpu->arch.mcg_cap = KVM_MAX_MCE_BANKS;
11882
11883 if (!zalloc_cpumask_var(&vcpu->arch.wbinvd_dirty_mask,
11884 GFP_KERNEL_ACCOUNT))
11885 goto fail_free_mce_banks;
11886
Sean Christophersonc9b8b072020-02-18 15:29:48 -080011887 if (!alloc_emulate_ctxt(vcpu))
11888 goto free_wbinvd_dirty_mask;
11889
Thomas Gleixnerd69c1382021-10-22 20:55:53 +020011890 if (!fpu_alloc_guest_fpstate(&vcpu->arch.guest_fpu)) {
11891 pr_err("kvm: failed to allocate vcpu's fpu\n");
Sean Christophersonc9b8b072020-02-18 15:29:48 -080011892 goto free_emulate_ctxt;
Sean Christopherson95a0d012019-12-18 13:55:23 -080011893 }
11894
Sean Christopherson95a0d012019-12-18 13:55:23 -080011895 vcpu->arch.maxphyaddr = cpuid_query_maxphyaddr(vcpu);
Sean Christophersona8ac8642021-02-03 16:01:15 -080011896 vcpu->arch.reserved_gpa_bits = kvm_vcpu_reserved_gpa_bits_raw(vcpu);
Sean Christopherson95a0d012019-12-18 13:55:23 -080011897
11898 vcpu->arch.pat = MSR_IA32_CR_PAT_DEFAULT;
11899
11900 kvm_async_pf_hash_reset(vcpu);
11901 kvm_pmu_init(vcpu);
11902
11903 vcpu->arch.pending_external_vector = -1;
11904 vcpu->arch.preempted_in_kernel = false;
11905
Vineeth Pillai3c86c0d2021-06-03 15:14:36 +000011906#if IS_ENABLED(CONFIG_HYPERV)
11907 vcpu->arch.hv_root_tdp = INVALID_PAGE;
11908#endif
11909
Jason Baronb36464772021-01-14 22:27:56 -050011910 r = static_call(kvm_x86_vcpu_create)(vcpu);
Sean Christopherson95a0d012019-12-18 13:55:23 -080011911 if (r)
11912 goto free_guest_fpu;
Sean Christopherson5f73db12019-12-18 13:55:18 -080011913
Sean Christopherson0cf91352019-03-07 15:43:02 -080011914 vcpu->arch.arch_capabilities = kvm_get_arch_capabilities();
Jim Mattsone53d88af2018-10-30 12:20:21 -070011915 vcpu->arch.msr_platform_info = MSR_PLATFORM_INFO_CPUID_FAULT;
David Woodhouse942c2492022-03-03 15:41:21 +000011916 kvm_xen_init_vcpu(vcpu);
Xiao Guangrong19efffa2015-06-15 16:55:31 +080011917 kvm_vcpu_mtrr_init(vcpu);
Christoffer Dallec7660c2017-12-04 21:35:23 +010011918 vcpu_load(vcpu);
David Woodhouseffbb61d2022-02-25 14:53:02 +000011919 kvm_set_tsc_khz(vcpu, vcpu->kvm->arch.default_tsc_khz);
Nadav Amitd28bc9d2015-04-13 14:34:08 +030011920 kvm_vcpu_reset(vcpu, false);
Sean Christophersonc9060662021-06-09 16:42:33 -070011921 kvm_init_mmu(vcpu);
Zhang Xiantaoe9b11c12007-11-14 20:38:21 +080011922 vcpu_put(vcpu);
Christoffer Dallec7660c2017-12-04 21:35:23 +010011923 return 0;
Sean Christopherson95a0d012019-12-18 13:55:23 -080011924
11925free_guest_fpu:
Thomas Gleixnerd69c1382021-10-22 20:55:53 +020011926 fpu_free_guest_fpstate(&vcpu->arch.guest_fpu);
Sean Christophersonc9b8b072020-02-18 15:29:48 -080011927free_emulate_ctxt:
11928 kmem_cache_free(x86_emulator_cache, vcpu->arch.emulate_ctxt);
Sean Christopherson95a0d012019-12-18 13:55:23 -080011929free_wbinvd_dirty_mask:
11930 free_cpumask_var(vcpu->arch.wbinvd_dirty_mask);
11931fail_free_mce_banks:
11932 kfree(vcpu->arch.mce_banks);
Jue Wang281b5272022-06-10 10:11:32 -070011933 kfree(vcpu->arch.mci_ctl2_banks);
Sean Christopherson95a0d012019-12-18 13:55:23 -080011934 free_page((unsigned long)vcpu->arch.pio_data);
11935fail_free_lapic:
11936 kvm_free_lapic(vcpu);
11937fail_mmu_destroy:
11938 kvm_mmu_destroy(vcpu);
11939 return r;
Zhang Xiantaoe9b11c12007-11-14 20:38:21 +080011940}
11941
Dominik Dingel31928aa2014-12-04 15:47:07 +010011942void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
Marcelo Tosatti42897d82012-11-27 23:29:02 -020011943{
Andrew Jones332967a2014-02-28 12:52:55 +010011944 struct kvm *kvm = vcpu->kvm;
Marcelo Tosatti42897d82012-11-27 23:29:02 -020011945
Christoffer Dallec7660c2017-12-04 21:35:23 +010011946 if (mutex_lock_killable(&vcpu->mutex))
Dominik Dingel31928aa2014-12-04 15:47:07 +010011947 return;
Christoffer Dallec7660c2017-12-04 21:35:23 +010011948 vcpu_load(vcpu);
Paolo Bonzini0c899c22020-09-24 14:45:27 +020011949 kvm_synchronize_tsc(vcpu, 0);
Marcelo Tosatti42897d82012-11-27 23:29:02 -020011950 vcpu_put(vcpu);
Marcelo Tosatti2d5ba192019-06-03 19:52:44 -030011951
11952 /* poll control enabled by default */
11953 vcpu->arch.msr_kvm_poll_control = 1;
11954
Christoffer Dallec7660c2017-12-04 21:35:23 +010011955 mutex_unlock(&vcpu->mutex);
Marcelo Tosatti42897d82012-11-27 23:29:02 -020011956
Wanpeng Lib34de572020-02-28 11:18:41 +080011957 if (kvmclock_periodic_sync && vcpu->vcpu_idx == 0)
11958 schedule_delayed_work(&kvm->arch.kvmclock_sync_work,
11959 KVMCLOCK_SYNC_PERIOD);
Marcelo Tosatti42897d82012-11-27 23:29:02 -020011960}
11961
Hollis Blanchardd40ccc62007-11-19 14:04:43 -060011962void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
Zhang Xiantaoe9b11c12007-11-14 20:38:21 +080011963{
Sean Christopherson95a0d012019-12-18 13:55:23 -080011964 int idx;
Gleb Natapov344d9582010-10-14 11:22:50 +020011965
Sean Christopherson50b143e2019-12-18 13:55:07 -080011966 kvmclock_reset(vcpu);
Zhang Xiantaoe9b11c12007-11-14 20:38:21 +080011967
Jason Baronb36464772021-01-14 22:27:56 -050011968 static_call(kvm_x86_vcpu_free)(vcpu);
Sean Christopherson50b143e2019-12-18 13:55:07 -080011969
Sean Christophersonc9b8b072020-02-18 15:29:48 -080011970 kmem_cache_free(x86_emulator_cache, vcpu->arch.emulate_ctxt);
Sean Christopherson50b143e2019-12-18 13:55:07 -080011971 free_cpumask_var(vcpu->arch.wbinvd_dirty_mask);
Thomas Gleixnerd69c1382021-10-22 20:55:53 +020011972 fpu_free_guest_fpstate(&vcpu->arch.guest_fpu);
Sean Christopherson95a0d012019-12-18 13:55:23 -080011973
David Woodhousea795cd42022-03-03 15:41:13 +000011974 kvm_xen_destroy_vcpu(vcpu);
Sean Christopherson95a0d012019-12-18 13:55:23 -080011975 kvm_hv_vcpu_uninit(vcpu);
11976 kvm_pmu_destroy(vcpu);
11977 kfree(vcpu->arch.mce_banks);
Jue Wang281b5272022-06-10 10:11:32 -070011978 kfree(vcpu->arch.mci_ctl2_banks);
Sean Christopherson95a0d012019-12-18 13:55:23 -080011979 kvm_free_lapic(vcpu);
11980 idx = srcu_read_lock(&vcpu->kvm->srcu);
11981 kvm_mmu_destroy(vcpu);
11982 srcu_read_unlock(&vcpu->kvm->srcu, idx);
11983 free_page((unsigned long)vcpu->arch.pio_data);
Vitaly Kuznetsov255cbec2020-10-01 15:05:40 +020011984 kvfree(vcpu->arch.cpuid_entries);
Sean Christopherson95a0d012019-12-18 13:55:23 -080011985 if (!lapic_in_kernel(vcpu))
Cun Li6e4e3b42021-01-11 23:24:35 +080011986 static_branch_dec(&kvm_has_noapic_vcpu);
Zhang Xiantaoe9b11c12007-11-14 20:38:21 +080011987}
11988
Nadav Amitd28bc9d2015-04-13 14:34:08 +030011989void kvm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event)
Zhang Xiantaoe9b11c12007-11-14 20:38:21 +080011990{
Sean Christopherson25b97842021-09-29 15:24:26 -070011991 struct kvm_cpuid_entry2 *cpuid_0x1;
Sean Christopherson0aa18372021-06-22 10:56:48 -070011992 unsigned long old_cr0 = kvm_read_cr0(vcpu);
Sean Christopherson4c72ab52021-07-13 09:33:24 -070011993 unsigned long new_cr0;
Sean Christopherson0aa18372021-06-22 10:56:48 -070011994
Sean Christopherson62dd57d2021-09-20 17:03:03 -070011995 /*
11996 * Several of the "set" flows, e.g. ->set_cr0(), read other registers
11997 * to handle side effects. RESET emulation hits those flows and relies
11998 * on emulated/virtualized registers, including those that are loaded
11999 * into hardware, to be zeroed at vCPU creation. Use CRs as a sentinel
12000 * to detect improper or missing initialization.
12001 */
12002 WARN_ON_ONCE(!init_event &&
12003 (old_cr0 || kvm_read_cr3(vcpu) || kvm_read_cr4(vcpu)));
Zhang Xiantaoe9b11c12007-11-14 20:38:21 +080012004
Radim Krčmářb7e31be2018-03-01 15:24:25 +010012005 kvm_lapic_reset(vcpu, init_event);
12006
Paolo Bonzinie69fab52015-06-04 10:44:44 +020012007 vcpu->arch.hflags = 0;
12008
Paolo Bonzinic43203c2016-06-01 22:26:00 +020012009 vcpu->arch.smi_pending = 0;
Liran Alon52797bf2017-11-15 13:43:14 +020012010 vcpu->arch.smi_count = 0;
Avi Kivity7460fb4a2011-09-20 13:43:14 +030012011 atomic_set(&vcpu->arch.nmi_queued, 0);
12012 vcpu->arch.nmi_pending = 0;
Jan Kiszka448fa4a2008-09-26 09:30:48 +020012013 vcpu->arch.nmi_injected = false;
Nadav Amit5f7552d2014-06-30 12:03:02 +030012014 kvm_clear_interrupt_queue(vcpu);
12015 kvm_clear_exception_queue(vcpu);
Jan Kiszka448fa4a2008-09-26 09:30:48 +020012016
Jan Kiszka42dbaa52008-12-15 13:52:10 +010012017 memset(vcpu->arch.db, 0, sizeof(vcpu->arch.db));
Nadav Amitae561ed2015-04-02 03:10:37 +030012018 kvm_update_dr0123(vcpu);
Chenyi Qiang9a3ecd52021-02-02 17:04:31 +080012019 vcpu->arch.dr6 = DR6_ACTIVE_LOW;
Jan Kiszka42dbaa52008-12-15 13:52:10 +010012020 vcpu->arch.dr7 = DR7_FIXED_1;
Jan Kiszkac8639012012-09-21 05:42:55 +020012021 kvm_update_dr7(vcpu);
Jan Kiszka42dbaa52008-12-15 13:52:10 +010012022
Nadav Amit11190222015-04-02 03:10:38 +030012023 vcpu->arch.cr2 = 0;
12024
Avi Kivity3842d132010-07-27 12:30:24 +030012025 kvm_make_request(KVM_REQ_EVENT, vcpu);
Vitaly Kuznetsov2635b5c2020-05-25 16:41:20 +020012026 vcpu->arch.apf.msr_en_val = 0;
12027 vcpu->arch.apf.msr_int_val = 0;
Glauber Costac9aaa892011-07-11 15:28:14 -040012028 vcpu->arch.st.msr_val = 0;
Avi Kivity3842d132010-07-27 12:30:24 +030012029
Glauber Costa12f9a482011-02-01 14:16:40 -050012030 kvmclock_reset(vcpu);
12031
Gleb Natapovaf585b92010-10-14 11:22:46 +020012032 kvm_clear_async_pf_completion_queue(vcpu);
12033 kvm_async_pf_hash_reset(vcpu);
12034 vcpu->arch.apf.halted = false;
Zhang Xiantaoe9b11c12007-11-14 20:38:21 +080012035
Thomas Gleixnerd69c1382021-10-22 20:55:53 +020012036 if (vcpu->arch.guest_fpu.fpstate && kvm_mpx_supported()) {
12037 struct fpstate *fpstate = vcpu->arch.guest_fpu.fpstate;
Wanpeng Lia554d202017-10-11 05:10:19 -070012038
12039 /*
Sean Christophersona61353a2022-09-21 00:31:53 +000012040 * All paths that lead to INIT are required to load the guest's
12041 * FPU state (because most paths are buried in KVM_RUN).
Wanpeng Lia554d202017-10-11 05:10:19 -070012042 */
Rik van Rielf775b132017-11-14 16:54:23 -050012043 if (init_event)
12044 kvm_put_guest_fpu(vcpu);
Thomas Gleixner087df482021-10-13 16:55:31 +020012045
12046 fpstate_clear_xstate_component(fpstate, XFEATURE_BNDREGS);
12047 fpstate_clear_xstate_component(fpstate, XFEATURE_BNDCSR);
12048
Rik van Rielf775b132017-11-14 16:54:23 -050012049 if (init_event)
12050 kvm_load_guest_fpu(vcpu);
Wanpeng Lia554d202017-10-11 05:10:19 -070012051 }
12052
Paolo Bonzini64d60672015-05-07 11:36:11 +020012053 if (!init_event) {
Nadav Amitd28bc9d2015-04-13 14:34:08 +030012054 kvm_pmu_reset(vcpu);
Paolo Bonzini64d60672015-05-07 11:36:11 +020012055 vcpu->arch.smbase = 0x30000;
Kyle Hueydb2336a2017-03-20 01:16:28 -070012056
Kyle Hueydb2336a2017-03-20 01:16:28 -070012057 vcpu->arch.msr_misc_features_enables = 0;
Sean Christopherson9fc22292022-06-11 00:57:49 +000012058 vcpu->arch.ia32_misc_enable_msr = MSR_IA32_MISC_ENABLE_PEBS_UNAVAIL |
12059 MSR_IA32_MISC_ENABLE_BTS_UNAVAIL;
Wanpeng Lia554d202017-10-11 05:10:19 -070012060
Like Xu05a9e062022-01-26 17:22:26 +000012061 __kvm_set_xcr(vcpu, 0, XFEATURE_MASK_FP);
12062 __kvm_set_msr(vcpu, MSR_IA32_XSS, 0, true);
Paolo Bonzini64d60672015-05-07 11:36:11 +020012063 }
Gleb Natapovf5132b02011-11-10 14:57:22 +020012064
Sean Christophersonff8828c2021-09-20 17:02:56 -070012065 /* All GPRs except RDX (handled below) are zeroed on RESET/INIT. */
Julian Stecklina66f7b722012-12-05 15:26:19 +010012066 memset(vcpu->arch.regs, 0, sizeof(vcpu->arch.regs));
Sean Christophersonff8828c2021-09-20 17:02:56 -070012067 kvm_register_mark_dirty(vcpu, VCPU_REGS_RSP);
Julian Stecklina66f7b722012-12-05 15:26:19 +010012068
Sean Christopherson49d86652021-07-13 09:32:57 -070012069 /*
12070 * Fall back to KVM's default Family/Model/Stepping of 0x600 (P6/Athlon)
12071 * if no CPUID match is found. Note, it's impossible to get a match at
12072 * RESET since KVM emulates RESET before exposing the vCPU to userspace,
Sean Christopherson25b97842021-09-29 15:24:26 -070012073 * i.e. it's impossible for kvm_find_cpuid_entry() to find a valid entry
12074 * on RESET. But, go through the motions in case that's ever remedied.
Sean Christopherson49d86652021-07-13 09:32:57 -070012075 */
Sean Christopherson277ad7d2022-07-12 02:06:45 +020012076 cpuid_0x1 = kvm_find_cpuid_entry(vcpu, 1);
Sean Christopherson25b97842021-09-29 15:24:26 -070012077 kvm_rdx_write(vcpu, cpuid_0x1 ? cpuid_0x1->eax : 0x600);
Sean Christopherson49d86652021-07-13 09:32:57 -070012078
Jason Baronb36464772021-01-14 22:27:56 -050012079 static_call(kvm_x86_vcpu_reset)(vcpu, init_event);
Sean Christopherson0aa18372021-06-22 10:56:48 -070012080
Sean Christophersonf39e8052021-07-13 09:33:14 -070012081 kvm_set_rflags(vcpu, X86_EFLAGS_FIXED);
12082 kvm_rip_write(vcpu, 0xfff0);
12083
Sean Christopherson03a6e842021-09-20 17:02:55 -070012084 vcpu->arch.cr3 = 0;
12085 kvm_register_mark_dirty(vcpu, VCPU_EXREG_CR3);
12086
Sean Christopherson4c72ab52021-07-13 09:33:24 -070012087 /*
12088 * CR0.CD/NW are set on RESET, preserved on INIT. Note, some versions
12089 * of Intel's SDM list CD/NW as being set on INIT, but they contradict
12090 * (or qualify) that with a footnote stating that CD/NW are preserved.
12091 */
12092 new_cr0 = X86_CR0_ET;
12093 if (init_event)
12094 new_cr0 |= (old_cr0 & (X86_CR0_NW | X86_CR0_CD));
12095 else
12096 new_cr0 |= X86_CR0_NW | X86_CR0_CD;
12097
12098 static_call(kvm_x86_set_cr0)(vcpu, new_cr0);
Sean Christophersonf39e8052021-07-13 09:33:14 -070012099 static_call(kvm_x86_set_cr4)(vcpu, 0);
12100 static_call(kvm_x86_set_efer)(vcpu, 0);
12101 static_call(kvm_x86_update_exception_bitmap)(vcpu);
12102
Sean Christopherson0aa18372021-06-22 10:56:48 -070012103 /*
Paolo Bonzinib5f61c02022-02-14 11:50:36 -050012104 * On the standard CR0/CR4/EFER modification paths, there are several
12105 * complex conditions determining whether the MMU has to be reset and/or
12106 * which PCIDs have to be flushed. However, CR0.WP and the paging-related
12107 * bits in CR4 and EFER are irrelevant if CR0.PG was '0'; and a reset+flush
12108 * is needed anyway if CR0.PG was '1' (which can only happen for INIT, as
12109 * CR0 will be '0' prior to RESET). So we only need to check CR0.PG here.
Sean Christopherson0aa18372021-06-22 10:56:48 -070012110 */
Paolo Bonzinib5f61c02022-02-14 11:50:36 -050012111 if (old_cr0 & X86_CR0_PG) {
12112 kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu);
Sean Christopherson0aa18372021-06-22 10:56:48 -070012113 kvm_mmu_reset_context(vcpu);
Paolo Bonzinib5f61c02022-02-14 11:50:36 -050012114 }
Sean Christophersondf37ed32021-07-13 09:32:39 -070012115
12116 /*
12117 * Intel's SDM states that all TLB entries are flushed on INIT. AMD's
12118 * APM states the TLBs are untouched by INIT, but it also states that
12119 * the TLBs are flushed on "External initialization of the processor."
12120 * Flush the guest TLB regardless of vendor, there is no meaningful
12121 * benefit in relying on the guest to flush the TLB immediately after
12122 * INIT. A spurious TLB flush is benign and likely negligible from a
12123 * performance perspective.
12124 */
12125 if (init_event)
12126 kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu);
Zhang Xiantaoe9b11c12007-11-14 20:38:21 +080012127}
Sean Christopherson265e4352021-07-13 09:33:22 -070012128EXPORT_SYMBOL_GPL(kvm_vcpu_reset);
Zhang Xiantaoe9b11c12007-11-14 20:38:21 +080012129
Paolo Bonzini2b4a2732014-11-24 14:35:24 +010012130void kvm_vcpu_deliver_sipi_vector(struct kvm_vcpu *vcpu, u8 vector)
Jan Kiszka66450a22013-03-13 12:42:34 +010012131{
12132 struct kvm_segment cs;
12133
12134 kvm_get_segment(vcpu, &cs, VCPU_SREG_CS);
12135 cs.selector = vector << 8;
12136 cs.base = vector << 12;
12137 kvm_set_segment(vcpu, &cs, VCPU_SREG_CS);
12138 kvm_rip_write(vcpu, 0);
Zhang Xiantaoe9b11c12007-11-14 20:38:21 +080012139}
Tom Lendacky647daca2021-01-04 14:20:01 -060012140EXPORT_SYMBOL_GPL(kvm_vcpu_deliver_sipi_vector);
Zhang Xiantaoe9b11c12007-11-14 20:38:21 +080012141
Radim Krčmář13a34e02014-08-28 15:13:03 +020012142int kvm_arch_hardware_enable(void)
Zhang Xiantaoe9b11c12007-11-14 20:38:21 +080012143{
Zachary Amsdenca84d1a2010-08-19 22:07:28 -100012144 struct kvm *kvm;
12145 struct kvm_vcpu *vcpu;
Marc Zyngier46808a42021-11-16 16:04:02 +000012146 unsigned long i;
Zachary Amsden0dd6a6e2012-02-03 15:43:56 -020012147 int ret;
12148 u64 local_tsc;
12149 u64 max_tsc = 0;
12150 bool stable, backwards_tsc = false;
Avi Kivity18863bd2009-09-07 11:12:18 +030012151
Sean Christopherson7e34fbd2020-09-23 11:03:55 -070012152 kvm_user_return_msr_cpu_online();
Jason Baronb36464772021-01-14 22:27:56 -050012153 ret = static_call(kvm_x86_hardware_enable)();
Zachary Amsden0dd6a6e2012-02-03 15:43:56 -020012154 if (ret != 0)
12155 return ret;
12156
Andy Lutomirski4ea16362015-06-25 18:44:07 +020012157 local_tsc = rdtsc();
Vitaly Kuznetsovb0c39dc2018-01-24 14:23:36 +010012158 stable = !kvm_check_tsc_unstable();
Zachary Amsden0dd6a6e2012-02-03 15:43:56 -020012159 list_for_each_entry(kvm, &vm_list, vm_list) {
12160 kvm_for_each_vcpu(i, vcpu, kvm) {
12161 if (!stable && vcpu->cpu == smp_processor_id())
Guo Hui Liu105b21b2014-09-12 13:43:19 +080012162 kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
Zachary Amsden0dd6a6e2012-02-03 15:43:56 -020012163 if (stable && vcpu->arch.last_host_tsc > local_tsc) {
12164 backwards_tsc = true;
12165 if (vcpu->arch.last_host_tsc > max_tsc)
12166 max_tsc = vcpu->arch.last_host_tsc;
12167 }
12168 }
12169 }
12170
12171 /*
12172 * Sometimes, even reliable TSCs go backwards. This happens on
12173 * platforms that reset TSC during suspend or hibernate actions, but
12174 * maintain synchronization. We must compensate. Fortunately, we can
12175 * detect that condition here, which happens early in CPU bringup,
12176 * before any KVM threads can be running. Unfortunately, we can't
12177 * bring the TSCs fully up to date with real time, as we aren't yet far
12178 * enough into CPU bringup that we know how much real time has actually
Jason A. Donenfeld9285ec42019-06-21 22:32:48 +020012179 * elapsed; our helper function, ktime_get_boottime_ns() will be using boot
Zachary Amsden0dd6a6e2012-02-03 15:43:56 -020012180 * variables that haven't been updated yet.
12181 *
12182 * So we simply find the maximum observed TSC above, then record the
12183 * adjustment to TSC in each VCPU. When the VCPU later gets loaded,
12184 * the adjustment will be applied. Note that we accumulate
12185 * adjustments, in case multiple suspend cycles happen before some VCPU
12186 * gets a chance to run again. In the event that no KVM threads get a
12187 * chance to run, we will miss the entire elapsed period, as we'll have
12188 * reset last_host_tsc, so VCPUs will not have the TSC adjusted and may
12189 * loose cycle time. This isn't too big a deal, since the loss will be
12190 * uniform across all VCPUs (not to mention the scenario is extremely
12191 * unlikely). It is possible that a second hibernate recovery happens
12192 * much faster than a first, causing the observed TSC here to be
12193 * smaller; this would require additional padding adjustment, which is
12194 * why we set last_host_tsc to the local tsc observed here.
12195 *
12196 * N.B. - this code below runs only on platforms with reliable TSC,
12197 * as that is the only way backwards_tsc is set above. Also note
12198 * that this runs for ALL vcpus, which is not a bug; all VCPUs should
12199 * have the same delta_cyc adjustment applied if backwards_tsc
12200 * is detected. Note further, this adjustment is only done once,
12201 * as we reset last_host_tsc on all VCPUs to stop this from being
12202 * called multiple times (one for each physical CPU bringup).
12203 *
Guo Chao4a969982012-06-28 15:17:27 +080012204 * Platforms with unreliable TSCs don't have to deal with this, they
Zachary Amsden0dd6a6e2012-02-03 15:43:56 -020012205 * will be compensated by the logic in vcpu_load, which sets the TSC to
12206 * catchup mode. This will catchup all VCPUs to real time, but cannot
12207 * guarantee that they stay in perfect synchronization.
12208 */
12209 if (backwards_tsc) {
12210 u64 delta_cyc = max_tsc - local_tsc;
12211 list_for_each_entry(kvm, &vm_list, vm_list) {
Ladi Proseka826faf12017-06-26 09:56:43 +020012212 kvm->arch.backwards_tsc_observed = true;
Zachary Amsden0dd6a6e2012-02-03 15:43:56 -020012213 kvm_for_each_vcpu(i, vcpu, kvm) {
12214 vcpu->arch.tsc_offset_adjustment += delta_cyc;
12215 vcpu->arch.last_host_tsc = local_tsc;
Guo Hui Liu105b21b2014-09-12 13:43:19 +080012216 kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu);
Zachary Amsden0dd6a6e2012-02-03 15:43:56 -020012217 }
12218
12219 /*
12220 * We have to disable TSC offset matching.. if you were
12221 * booting a VM while issuing an S4 host suspend....
12222 * you may have some problem. Solving this issue is
12223 * left as an exercise to the reader.
12224 */
12225 kvm->arch.last_tsc_nsec = 0;
12226 kvm->arch.last_tsc_write = 0;
12227 }
12228
12229 }
12230 return 0;
Zhang Xiantaoe9b11c12007-11-14 20:38:21 +080012231}
12232
Radim Krčmář13a34e02014-08-28 15:13:03 +020012233void kvm_arch_hardware_disable(void)
Zhang Xiantaoe9b11c12007-11-14 20:38:21 +080012234{
Jason Baronb36464772021-01-14 22:27:56 -050012235 static_call(kvm_x86_hardware_disable)();
Radim Krčmář13a34e02014-08-28 15:13:03 +020012236 drop_user_return_notifiers();
Zhang Xiantaoe9b11c12007-11-14 20:38:21 +080012237}
12238
Like Xufdc298d2022-03-29 23:50:51 +000012239static inline void kvm_ops_update(struct kvm_x86_init_ops *ops)
12240{
12241 memcpy(&kvm_x86_ops, ops->runtime_ops, sizeof(kvm_x86_ops));
12242
12243#define __KVM_X86_OP(func) \
12244 static_call_update(kvm_x86_##func, kvm_x86_ops.func);
12245#define KVM_X86_OP(func) \
12246 WARN_ON(!kvm_x86_ops.func); __KVM_X86_OP(func)
12247#define KVM_X86_OP_OPTIONAL __KVM_X86_OP
12248#define KVM_X86_OP_OPTIONAL_RET0(func) \
12249 static_call_update(kvm_x86_##func, (void *)kvm_x86_ops.func ? : \
12250 (void *)__static_call_return0);
12251#include <asm/kvm-x86-ops.h>
12252#undef __KVM_X86_OP
Like Xu8f969c02022-03-29 23:50:52 +000012253
Like Xu34886e72022-03-29 23:50:53 +000012254 kvm_pmu_ops_update(ops->pmu_ops);
Like Xufdc298d2022-03-29 23:50:51 +000012255}
12256
Sean Christophersonb9904082020-03-21 13:25:55 -070012257int kvm_arch_hardware_setup(void *opaque)
Zhang Xiantaoe9b11c12007-11-14 20:38:21 +080012258{
Sean Christophersond008dfd2020-03-21 13:25:56 -070012259 struct kvm_x86_init_ops *ops = opaque;
Nadav Amit9e9c3fe2015-04-12 21:47:15 +030012260 int r;
12261
Sean Christopherson91661982020-03-02 15:57:06 -080012262 rdmsrl_safe(MSR_EFER, &host_efer);
12263
Paolo Bonzini408e9a32020-03-05 16:11:56 +010012264 if (boot_cpu_has(X86_FEATURE_XSAVES))
12265 rdmsrl(MSR_IA32_XSS, host_xss);
12266
Like Xu968635a2022-04-11 18:19:44 +080012267 kvm_init_pmu_capability();
12268
Sean Christophersond008dfd2020-03-21 13:25:56 -070012269 r = ops->hardware_setup();
Nadav Amit9e9c3fe2015-04-12 21:47:15 +030012270 if (r != 0)
12271 return r;
12272
Like Xufdc298d2022-03-29 23:50:51 +000012273 kvm_ops_update(ops);
Sean Christopherson69c6f692020-03-21 13:25:59 -070012274
Sean Christopherson33271a92021-11-11 02:07:34 +000012275 kvm_register_perf_callbacks(ops->handle_intel_pt_intr);
Sean Christopherson5c7df802021-11-11 02:07:23 +000012276
Paolo Bonzini408e9a32020-03-05 16:11:56 +010012277 if (!kvm_cpu_cap_has(X86_FEATURE_XSAVES))
Sean Christopherson938c8742022-05-24 21:56:23 +080012278 kvm_caps.supported_xss = 0;
Paolo Bonzini408e9a32020-03-05 16:11:56 +010012279
Paolo Bonzini139f7422020-05-05 09:40:46 -040012280#define __kvm_cpu_cap_has(UNUSED_, f) kvm_cpu_cap_has(f)
12281 cr4_reserved_bits = __cr4_reserved_bits(__kvm_cpu_cap_has, UNUSED_);
12282#undef __kvm_cpu_cap_has
Sean Christophersonb11306b2019-12-10 14:44:13 -080012283
Sean Christopherson938c8742022-05-24 21:56:23 +080012284 if (kvm_caps.has_tsc_control) {
Haozhong Zhang35181e82015-10-20 15:39:03 +080012285 /*
12286 * Make sure the user can only configure tsc_khz values that
12287 * fit into a signed integer.
Marcelo Tosatti273ba452018-06-11 14:12:10 -030012288 * A min value is not calculated because it will always
Haozhong Zhang35181e82015-10-20 15:39:03 +080012289 * be 1 on all machines.
12290 */
12291 u64 max = min(0x7fffffffULL,
Sean Christopherson938c8742022-05-24 21:56:23 +080012292 __scale_tsc(kvm_caps.max_tsc_scaling_ratio, tsc_khz));
12293 kvm_caps.max_guest_tsc_khz = max;
Haozhong Zhang35181e82015-10-20 15:39:03 +080012294 }
Sean Christopherson938c8742022-05-24 21:56:23 +080012295 kvm_caps.default_tsc_scaling_ratio = 1ULL << kvm_caps.tsc_scaling_ratio_frac_bits;
Nadav Amit9e9c3fe2015-04-12 21:47:15 +030012296 kvm_init_msr_list();
12297 return 0;
Zhang Xiantaoe9b11c12007-11-14 20:38:21 +080012298}
12299
12300void kvm_arch_hardware_unsetup(void)
12301{
Sean Christophersone1bfc242021-11-11 02:07:33 +000012302 kvm_unregister_perf_callbacks();
Sean Christopherson5c7df802021-11-11 02:07:23 +000012303
Jason Baronb36464772021-01-14 22:27:56 -050012304 static_call(kvm_x86_hardware_unsetup)();
Zhang Xiantaoe9b11c12007-11-14 20:38:21 +080012305}
12306
Sean Christophersonb9904082020-03-21 13:25:55 -070012307int kvm_arch_check_processor_compat(void *opaque)
Zhang Xiantaoe9b11c12007-11-14 20:38:21 +080012308{
Sean Christophersonf1cdecf2019-12-10 14:44:14 -080012309 struct cpuinfo_x86 *c = &cpu_data(smp_processor_id());
Sean Christophersond008dfd2020-03-21 13:25:56 -070012310 struct kvm_x86_init_ops *ops = opaque;
Sean Christophersonf1cdecf2019-12-10 14:44:14 -080012311
12312 WARN_ON(!irqs_disabled());
12313
Paolo Bonzini139f7422020-05-05 09:40:46 -040012314 if (__cr4_reserved_bits(cpu_has, c) !=
12315 __cr4_reserved_bits(cpu_has, &boot_cpu_data))
Sean Christophersonf1cdecf2019-12-10 14:44:14 -080012316 return -EIO;
12317
Sean Christophersond008dfd2020-03-21 13:25:56 -070012318 return ops->check_processor_compatibility();
Zhang Xiantaoe9b11c12007-11-14 20:38:21 +080012319}
12320
Paolo Bonzinid71ba782015-07-29 11:56:48 +020012321bool kvm_vcpu_is_reset_bsp(struct kvm_vcpu *vcpu)
12322{
12323 return vcpu->kvm->arch.bsp_vcpu_id == vcpu->vcpu_id;
12324}
12325EXPORT_SYMBOL_GPL(kvm_vcpu_is_reset_bsp);
12326
12327bool kvm_vcpu_is_bsp(struct kvm_vcpu *vcpu)
12328{
12329 return (vcpu->arch.apic_base & MSR_IA32_APICBASE_BSP) != 0;
12330}
12331
Cun Li6e4e3b42021-01-11 23:24:35 +080012332__read_mostly DEFINE_STATIC_KEY_FALSE(kvm_has_noapic_vcpu);
12333EXPORT_SYMBOL_GPL(kvm_has_noapic_vcpu);
Gleb Natapov54e98182012-08-05 15:58:32 +030012334
Radim Krčmáře790d9e2014-08-21 18:08:05 +020012335void kvm_arch_sched_in(struct kvm_vcpu *vcpu, int cpu)
12336{
Like Xub35e5542019-10-27 18:52:43 +080012337 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
12338
Paolo Bonzinic595cee2018-07-02 13:07:14 +020012339 vcpu->arch.l1tf_flush_l1d = true;
Like Xub35e5542019-10-27 18:52:43 +080012340 if (pmu->version && unlikely(pmu->event_count)) {
12341 pmu->need_cleanup = true;
12342 kvm_make_request(KVM_REQ_PMU, vcpu);
12343 }
Jason Baronb36464772021-01-14 22:27:56 -050012344 static_call(kvm_x86_sched_in)(vcpu, cpu);
Radim Krčmáře790d9e2014-08-21 18:08:05 +020012345}
12346
Sean Christopherson562b6b02020-01-26 16:41:13 -080012347void kvm_arch_free_vm(struct kvm *kvm)
12348{
Vitaly Kuznetsov05f04ae2021-01-26 14:48:09 +010012349 kfree(to_kvm_hv(kvm)->hv_pa_pg);
Juergen Gross78b497f2021-09-03 15:08:05 +020012350 __kvm_arch_free_vm(kvm);
Sean Christopherson562b6b02020-01-26 16:41:13 -080012351}
12352
12353
Carsten Ottee08b9632012-01-04 10:25:20 +010012354int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
Zhang Xiantaod19a9cd2007-11-18 18:43:45 +080012355{
Haimin Zhangeb7511b2021-09-03 10:37:06 +080012356 int ret;
Paolo Bonzini869b4422021-09-16 18:15:36 +000012357 unsigned long flags;
Haimin Zhangeb7511b2021-09-03 10:37:06 +080012358
Carsten Ottee08b9632012-01-04 10:25:20 +010012359 if (type)
12360 return -EINVAL;
12361
Haimin Zhangeb7511b2021-09-03 10:37:06 +080012362 ret = kvm_page_track_init(kvm);
12363 if (ret)
Paolo Bonzinia1a39122022-03-25 12:42:52 -040012364 goto out;
12365
12366 ret = kvm_mmu_init_vm(kvm);
12367 if (ret)
12368 goto out_page_track;
Haimin Zhangeb7511b2021-09-03 10:37:06 +080012369
Junaid Shahidb24ede22022-07-29 15:43:29 -070012370 ret = static_call(kvm_x86_vm_init)(kvm);
12371 if (ret)
12372 goto out_uninit_mmu;
12373
Paolo Bonzini6ef768f2014-11-20 13:45:31 +010012374 INIT_HLIST_HEAD(&kvm->arch.mask_notifier_list);
Ben-Ami Yassour4d5c5d02008-07-28 19:26:26 +030012375 INIT_LIST_HEAD(&kvm->arch.assigned_dev_head);
Alex Williamsone0f0bbc2013-10-30 11:02:30 -060012376 atomic_set(&kvm->arch.noncoherent_dma_count, 0);
Zhang Xiantaod19a9cd2007-11-18 18:43:45 +080012377
Sheng Yang5550af42008-10-15 20:15:06 +080012378 /* Reserve bit 0 of irq_sources_bitmap for userspace irq source */
12379 set_bit(KVM_USERSPACE_IRQ_SOURCE_ID, &kvm->arch.irq_sources_bitmap);
Alex Williamson7a844282012-09-21 11:58:03 -060012380 /* Reserve bit 1 of irq_sources_bitmap for irqfd-resampler */
12381 set_bit(KVM_IRQFD_RESAMPLE_IRQ_SOURCE_ID,
12382 &kvm->arch.irq_sources_bitmap);
Sheng Yang5550af42008-10-15 20:15:06 +080012383
Jan Kiszka038f8c12011-02-04 10:49:11 +010012384 raw_spin_lock_init(&kvm->arch.tsc_write_lock);
Gleb Natapov1e08ec42012-09-13 17:19:24 +030012385 mutex_init(&kvm->arch.apic_map_lock);
Paolo Bonzini869b4422021-09-16 18:15:36 +000012386 seqcount_raw_spinlock_init(&kvm->arch.pvclock_sc, &kvm->arch.tsc_write_lock);
Paolo Bonzini8171cd62020-01-22 14:36:09 +010012387 kvm->arch.kvmclock_offset = -get_kvmclock_base_ns();
Paolo Bonzini869b4422021-09-16 18:15:36 +000012388
12389 raw_spin_lock_irqsave(&kvm->arch.tsc_write_lock, flags);
Marcelo Tosattid8281992012-11-27 23:29:01 -020012390 pvclock_update_vm_gtod_copy(kvm);
Paolo Bonzini869b4422021-09-16 18:15:36 +000012391 raw_spin_unlock_irqrestore(&kvm->arch.tsc_write_lock, flags);
Marcelo Tosatti53f658b32008-12-11 20:45:05 +010012392
Sean Christopherson741e5112022-02-25 14:53:03 +000012393 kvm->arch.default_tsc_khz = max_tsc_khz ? : tsc_khz;
Drew Schmitt6fbbde92018-08-20 10:32:15 -070012394 kvm->arch.guest_can_read_msr_platform_info = true;
David Dunnba7bb6632022-02-23 22:57:41 +000012395 kvm->arch.enable_pmu = enable_pmu;
Drew Schmitt6fbbde92018-08-20 10:32:15 -070012396
Vineeth Pillai3c86c0d2021-06-03 15:14:36 +000012397#if IS_ENABLED(CONFIG_HYPERV)
12398 spin_lock_init(&kvm->arch.hv_root_tdp_lock);
12399 kvm->arch.hv_root_tdp = INVALID_PAGE;
12400#endif
12401
Andrew Jones7e44e442014-02-28 12:52:54 +010012402 INIT_DELAYED_WORK(&kvm->arch.kvmclock_update_work, kvmclock_update_fn);
Andrew Jones332967a2014-02-28 12:52:55 +010012403 INIT_DELAYED_WORK(&kvm->arch.kvmclock_sync_work, kvmclock_sync_fn);
Andrew Jones7e44e442014-02-28 12:52:54 +010012404
Vitaly Kuznetsov4651fc52021-06-09 17:09:09 +020012405 kvm_apicv_init(kvm);
Roman Kagancbc02362018-02-01 16:48:31 +030012406 kvm_hv_init_vm(kvm);
Paolo Bonzini319afe62021-08-04 12:48:41 -040012407 kvm_xen_init_vm(kvm);
Xiao Guangrong0eb05bf2016-02-24 17:51:13 +080012408
Junaid Shahidb24ede22022-07-29 15:43:29 -070012409 return 0;
Paolo Bonzinia1a39122022-03-25 12:42:52 -040012410
Junaid Shahidb24ede22022-07-29 15:43:29 -070012411out_uninit_mmu:
12412 kvm_mmu_uninit_vm(kvm);
Paolo Bonzinia1a39122022-03-25 12:42:52 -040012413out_page_track:
12414 kvm_page_track_cleanup(kvm);
12415out:
12416 return ret;
Zhang Xiantaod19a9cd2007-11-18 18:43:45 +080012417}
12418
Junaid Shahid1aa9b952019-11-04 20:26:00 +010012419int kvm_arch_post_init_vm(struct kvm *kvm)
12420{
12421 return kvm_mmu_post_init_vm(kvm);
12422}
12423
Zhang Xiantaod19a9cd2007-11-18 18:43:45 +080012424static void kvm_unload_vcpu_mmu(struct kvm_vcpu *vcpu)
12425{
Christoffer Dallec7660c2017-12-04 21:35:23 +010012426 vcpu_load(vcpu);
Zhang Xiantaod19a9cd2007-11-18 18:43:45 +080012427 kvm_mmu_unload(vcpu);
12428 vcpu_put(vcpu);
12429}
12430
Maxim Levitsky6fcee032022-03-22 19:24:42 +020012431static void kvm_unload_vcpu_mmus(struct kvm *kvm)
Zhang Xiantaod19a9cd2007-11-18 18:43:45 +080012432{
Marc Zyngier46808a42021-11-16 16:04:02 +000012433 unsigned long i;
Gleb Natapov988a2ca2009-06-09 15:56:29 +030012434 struct kvm_vcpu *vcpu;
Zhang Xiantaod19a9cd2007-11-18 18:43:45 +080012435
Gleb Natapovaf585b92010-10-14 11:22:46 +020012436 kvm_for_each_vcpu(i, vcpu, kvm) {
12437 kvm_clear_async_pf_completion_queue(vcpu);
Gleb Natapov988a2ca2009-06-09 15:56:29 +030012438 kvm_unload_vcpu_mmu(vcpu);
Gleb Natapovaf585b92010-10-14 11:22:46 +020012439 }
Zhang Xiantaod19a9cd2007-11-18 18:43:45 +080012440}
12441
Sheng Yangad8ba2c2009-01-06 10:03:02 +080012442void kvm_arch_sync_events(struct kvm *kvm)
12443{
Andrew Jones332967a2014-02-28 12:52:55 +010012444 cancel_delayed_work_sync(&kvm->arch.kvmclock_sync_work);
Andrew Jones7e44e442014-02-28 12:52:54 +010012445 cancel_delayed_work_sync(&kvm->arch.kvmclock_update_work);
Xiao Guangrongaea924f2010-07-10 17:37:56 +080012446 kvm_free_pit(kvm);
Sheng Yangad8ba2c2009-01-06 10:03:02 +080012447}
12448
Peter Xuff5a9832020-09-30 21:20:33 -040012449/**
12450 * __x86_set_memory_region: Setup KVM internal memory slot
12451 *
12452 * @kvm: the kvm pointer to the VM.
12453 * @id: the slot ID to setup.
12454 * @gpa: the GPA to install the slot (unused when @size == 0).
12455 * @size: the size of the slot. Set to zero to uninstall a slot.
12456 *
12457 * This function helps to setup a KVM internal memory slot. Specify
12458 * @size > 0 to install a new slot, while @size == 0 to uninstall a
12459 * slot. The return code can be one of the following:
12460 *
12461 * HVA: on success (uninstall will return a bogus HVA)
12462 * -errno: on error
12463 *
12464 * The caller should always use IS_ERR() to check the return value
12465 * before use. Note, the KVM internal memory slots are guaranteed to
12466 * remain valid and unchanged until the VM is destroyed, i.e., the
12467 * GPA->HVA translation will not change. However, the HVA is a user
12468 * address, i.e. its accessibility is not guaranteed, and must be
12469 * accessed via __copy_{to,from}_user().
12470 */
12471void __user * __x86_set_memory_region(struct kvm *kvm, int id, gpa_t gpa,
12472 u32 size)
Paolo Bonzini9da0e4d2015-05-18 13:33:16 +020012473{
12474 int i, r;
Kees Cook3f649ab2020-06-03 13:09:38 -070012475 unsigned long hva, old_npages;
Paolo Bonzinif0d648b2015-10-12 13:56:27 +020012476 struct kvm_memslots *slots = kvm_memslots(kvm);
Sean Christopherson0577d1a2020-02-18 13:07:31 -080012477 struct kvm_memory_slot *slot;
Paolo Bonzini9da0e4d2015-05-18 13:33:16 +020012478
12479 /* Called with kvm->slots_lock held. */
Paolo Bonzini1d8007b2015-10-12 13:38:32 +020012480 if (WARN_ON(id >= KVM_MEM_SLOTS_NUM))
Peter Xuff5a9832020-09-30 21:20:33 -040012481 return ERR_PTR_USR(-EINVAL);
Paolo Bonzini9da0e4d2015-05-18 13:33:16 +020012482
Paolo Bonzinif0d648b2015-10-12 13:56:27 +020012483 slot = id_to_memslot(slots, id);
12484 if (size) {
Sean Christopherson0577d1a2020-02-18 13:07:31 -080012485 if (slot && slot->npages)
Peter Xuff5a9832020-09-30 21:20:33 -040012486 return ERR_PTR_USR(-EEXIST);
Paolo Bonzinif0d648b2015-10-12 13:56:27 +020012487
12488 /*
12489 * MAP_SHARED to prevent internal slot pages from being moved
12490 * by fork()/COW.
12491 */
12492 hva = vm_mmap(NULL, 0, size, PROT_READ | PROT_WRITE,
12493 MAP_SHARED | MAP_ANONYMOUS, 0);
12494 if (IS_ERR((void *)hva))
Peter Xuff5a9832020-09-30 21:20:33 -040012495 return (void __user *)hva;
Paolo Bonzinif0d648b2015-10-12 13:56:27 +020012496 } else {
Sean Christopherson0577d1a2020-02-18 13:07:31 -080012497 if (!slot || !slot->npages)
Muhammad Usama Anjum46914532021-03-05 23:08:16 +050012498 return NULL;
Paolo Bonzinif0d648b2015-10-12 13:56:27 +020012499
Sean Christopherson0577d1a2020-02-18 13:07:31 -080012500 old_npages = slot->npages;
Zheng Zhan Liangb66f9ba2021-02-01 13:53:10 +080012501 hva = slot->userspace_addr;
Paolo Bonzinif0d648b2015-10-12 13:56:27 +020012502 }
12503
Paolo Bonzini9da0e4d2015-05-18 13:33:16 +020012504 for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) {
Paolo Bonzini1d8007b2015-10-12 13:38:32 +020012505 struct kvm_userspace_memory_region m;
Paolo Bonzini9da0e4d2015-05-18 13:33:16 +020012506
Paolo Bonzini1d8007b2015-10-12 13:38:32 +020012507 m.slot = id | (i << 16);
12508 m.flags = 0;
12509 m.guest_phys_addr = gpa;
Paolo Bonzinif0d648b2015-10-12 13:56:27 +020012510 m.userspace_addr = hva;
Paolo Bonzini1d8007b2015-10-12 13:38:32 +020012511 m.memory_size = size;
Paolo Bonzini9da0e4d2015-05-18 13:33:16 +020012512 r = __kvm_set_memory_region(kvm, &m);
12513 if (r < 0)
Peter Xuff5a9832020-09-30 21:20:33 -040012514 return ERR_PTR_USR(r);
Paolo Bonzini9da0e4d2015-05-18 13:33:16 +020012515 }
12516
Eric Biggers103c7632018-01-31 17:30:21 -080012517 if (!size)
Sean Christopherson0577d1a2020-02-18 13:07:31 -080012518 vm_munmap(hva, old_npages * PAGE_SIZE);
Paolo Bonzinif0d648b2015-10-12 13:56:27 +020012519
Peter Xuff5a9832020-09-30 21:20:33 -040012520 return (void __user *)hva;
Paolo Bonzini9da0e4d2015-05-18 13:33:16 +020012521}
12522EXPORT_SYMBOL_GPL(__x86_set_memory_region);
12523
Junaid Shahid1aa9b952019-11-04 20:26:00 +010012524void kvm_arch_pre_destroy_vm(struct kvm *kvm)
12525{
12526 kvm_mmu_pre_destroy_vm(kvm);
12527}
12528
Zhang Xiantaod19a9cd2007-11-18 18:43:45 +080012529void kvm_arch_destroy_vm(struct kvm *kvm)
12530{
Andrew Honig27469d22013-04-18 09:38:14 -070012531 if (current->mm == kvm->mm) {
12532 /*
12533 * Free memory regions allocated on behalf of userspace,
Bo Liuf7081832022-05-27 02:14:00 -040012534 * unless the memory map has changed due to process exit
Andrew Honig27469d22013-04-18 09:38:14 -070012535 * or fd copying.
12536 */
Peter Xu6a3c6232020-01-09 09:57:16 -050012537 mutex_lock(&kvm->slots_lock);
12538 __x86_set_memory_region(kvm, APIC_ACCESS_PAGE_PRIVATE_MEMSLOT,
12539 0, 0);
12540 __x86_set_memory_region(kvm, IDENTITY_PAGETABLE_PRIVATE_MEMSLOT,
12541 0, 0);
12542 __x86_set_memory_region(kvm, TSS_PRIVATE_MEMSLOT, 0, 0);
12543 mutex_unlock(&kvm->slots_lock);
Andrew Honig27469d22013-04-18 09:38:14 -070012544 }
Maxim Levitsky6fcee032022-03-22 19:24:42 +020012545 kvm_unload_vcpu_mmus(kvm);
Jason Baronb36464772021-01-14 22:27:56 -050012546 static_call_cond(kvm_x86_vm_destroy)(kvm);
Sean Christophersonb318e8d2021-03-16 11:44:33 -070012547 kvm_free_msr_filter(srcu_dereference_check(kvm->arch.msr_filter, &kvm->srcu, 1));
Peter Xuc7611592017-03-15 16:01:19 +080012548 kvm_pic_destroy(kvm);
12549 kvm_ioapic_destroy(kvm);
Maxim Levitsky6fcee032022-03-22 19:24:42 +020012550 kvm_destroy_vcpus(kvm);
Radim Krčmářaf1bae52016-07-12 22:09:30 +020012551 kvfree(rcu_dereference_check(kvm->arch.apic_map, 1));
Eric Hankland66bb8a02019-07-10 18:25:15 -070012552 kfree(srcu_dereference_check(kvm->arch.pmu_event_filter, &kvm->srcu, 1));
Xiao Guangrong13d268c2016-02-24 17:51:16 +080012553 kvm_mmu_uninit_vm(kvm);
Paolo Bonzini2beb6dad2017-03-27 17:53:50 +020012554 kvm_page_track_cleanup(kvm);
David Woodhouse7d6bbeb2021-02-02 15:48:05 +000012555 kvm_xen_destroy_vm(kvm);
Roman Kagancbc02362018-02-01 16:48:31 +030012556 kvm_hv_destroy_vm(kvm);
Zhang Xiantaod19a9cd2007-11-18 18:43:45 +080012557}
Zhang Xiantao0de10342007-11-20 16:25:04 +080012558
Ben Gardonc9b929b2021-05-18 10:34:08 -070012559static void memslot_rmap_free(struct kvm_memory_slot *slot)
Takuya Yoshikawadb3fe4e2012-02-08 13:02:18 +090012560{
12561 int i;
12562
Takuya Yoshikawad89cc612012-08-01 18:03:28 +090012563 for (i = 0; i < KVM_NR_PAGE_SIZES; ++i) {
Sean Christophersone96c81e2020-02-18 13:07:27 -080012564 kvfree(slot->arch.rmap[i]);
12565 slot->arch.rmap[i] = NULL;
Ben Gardonc9b929b2021-05-18 10:34:08 -070012566 }
12567}
Sean Christophersone96c81e2020-02-18 13:07:27 -080012568
Ben Gardonc9b929b2021-05-18 10:34:08 -070012569void kvm_arch_free_memslot(struct kvm *kvm, struct kvm_memory_slot *slot)
12570{
12571 int i;
Takuya Yoshikawad89cc612012-08-01 18:03:28 +090012572
Ben Gardonc9b929b2021-05-18 10:34:08 -070012573 memslot_rmap_free(slot);
12574
12575 for (i = 1; i < KVM_NR_PAGE_SIZES; ++i) {
Sean Christophersone96c81e2020-02-18 13:07:27 -080012576 kvfree(slot->arch.lpage_info[i - 1]);
12577 slot->arch.lpage_info[i - 1] = NULL;
Takuya Yoshikawadb3fe4e2012-02-08 13:02:18 +090012578 }
Xiao Guangrong21ebbed2016-02-24 17:51:09 +080012579
Sean Christophersone96c81e2020-02-18 13:07:27 -080012580 kvm_page_track_free_memslot(slot);
Takuya Yoshikawadb3fe4e2012-02-08 13:02:18 +090012581}
12582
David Stevens1e76a3c2021-10-15 12:30:21 -040012583int memslot_rmap_alloc(struct kvm_memory_slot *slot, unsigned long npages)
Ben Gardon56dd1012021-05-18 10:34:09 -070012584{
12585 const int sz = sizeof(*slot->arch.rmap[0]);
12586 int i;
12587
12588 for (i = 0; i < KVM_NR_PAGE_SIZES; ++i) {
12589 int level = i + 1;
Peter Xu4139b192021-07-30 18:04:51 -040012590 int lpages = __kvm_mmu_slot_lpages(slot, npages, level);
Ben Gardon56dd1012021-05-18 10:34:09 -070012591
Paolo Bonzinifa138432021-10-15 13:05:00 -040012592 if (slot->arch.rmap[i])
12593 continue;
Ben Gardond501f742021-05-18 10:34:14 -070012594
Paolo Bonzini37b2a652022-03-08 04:49:37 -050012595 slot->arch.rmap[i] = __vcalloc(lpages, sz, GFP_KERNEL_ACCOUNT);
Ben Gardon56dd1012021-05-18 10:34:09 -070012596 if (!slot->arch.rmap[i]) {
12597 memslot_rmap_free(slot);
12598 return -ENOMEM;
12599 }
12600 }
12601
12602 return 0;
12603}
12604
Ben Gardona2557402021-05-18 10:34:12 -070012605static int kvm_alloc_memslot_metadata(struct kvm *kvm,
Sean Christopherson9d7d18e2021-12-06 20:54:16 +010012606 struct kvm_memory_slot *slot)
Takuya Yoshikawadb3fe4e2012-02-08 13:02:18 +090012607{
Sean Christopherson9d7d18e2021-12-06 20:54:16 +010012608 unsigned long npages = slot->npages;
Ben Gardon56dd1012021-05-18 10:34:09 -070012609 int i, r;
Takuya Yoshikawadb3fe4e2012-02-08 13:02:18 +090012610
Sean Christophersonedd4fa32020-02-18 13:07:15 -080012611 /*
12612 * Clear out the previous array pointers for the KVM_MR_MOVE case. The
12613 * old arrays will be freed by __kvm_set_memory_region() if installing
12614 * the new memslot is successful.
12615 */
12616 memset(&slot->arch, 0, sizeof(slot->arch));
12617
Ben Gardone2209712021-05-18 10:34:13 -070012618 if (kvm_memslots_have_rmaps(kvm)) {
Ben Gardona2557402021-05-18 10:34:12 -070012619 r = memslot_rmap_alloc(slot, npages);
12620 if (r)
12621 return r;
12622 }
Ben Gardon56dd1012021-05-18 10:34:09 -070012623
12624 for (i = 1; i < KVM_NR_PAGE_SIZES; ++i) {
Xiao Guangrong92f94f12016-02-24 17:51:06 +080012625 struct kvm_lpage_info *linfo;
Takuya Yoshikawadb3fe4e2012-02-08 13:02:18 +090012626 unsigned long ugfn;
12627 int lpages;
Takuya Yoshikawad89cc612012-08-01 18:03:28 +090012628 int level = i + 1;
Takuya Yoshikawadb3fe4e2012-02-08 13:02:18 +090012629
Peter Xu4139b192021-07-30 18:04:51 -040012630 lpages = __kvm_mmu_slot_lpages(slot, npages, level);
Takuya Yoshikawadb3fe4e2012-02-08 13:02:18 +090012631
Paolo Bonzini37b2a652022-03-08 04:49:37 -050012632 linfo = __vcalloc(lpages, sizeof(*linfo), GFP_KERNEL_ACCOUNT);
Xiao Guangrong92f94f12016-02-24 17:51:06 +080012633 if (!linfo)
Takuya Yoshikawadb3fe4e2012-02-08 13:02:18 +090012634 goto out_free;
12635
Xiao Guangrong92f94f12016-02-24 17:51:06 +080012636 slot->arch.lpage_info[i - 1] = linfo;
12637
Takuya Yoshikawadb3fe4e2012-02-08 13:02:18 +090012638 if (slot->base_gfn & (KVM_PAGES_PER_HPAGE(level) - 1))
Xiao Guangrong92f94f12016-02-24 17:51:06 +080012639 linfo[0].disallow_lpage = 1;
Takuya Yoshikawadb3fe4e2012-02-08 13:02:18 +090012640 if ((slot->base_gfn + npages) & (KVM_PAGES_PER_HPAGE(level) - 1))
Xiao Guangrong92f94f12016-02-24 17:51:06 +080012641 linfo[lpages - 1].disallow_lpage = 1;
Takuya Yoshikawadb3fe4e2012-02-08 13:02:18 +090012642 ugfn = slot->userspace_addr >> PAGE_SHIFT;
12643 /*
12644 * If the gfn and userspace address are not aligned wrt each
Sean Christopherson600087b2020-03-02 15:57:05 -080012645 * other, disable large page support for this slot.
Takuya Yoshikawadb3fe4e2012-02-08 13:02:18 +090012646 */
Sean Christopherson600087b2020-03-02 15:57:05 -080012647 if ((slot->base_gfn ^ ugfn) & (KVM_PAGES_PER_HPAGE(level) - 1)) {
Takuya Yoshikawadb3fe4e2012-02-08 13:02:18 +090012648 unsigned long j;
12649
12650 for (j = 0; j < lpages; ++j)
Xiao Guangrong92f94f12016-02-24 17:51:06 +080012651 linfo[j].disallow_lpage = 1;
Takuya Yoshikawadb3fe4e2012-02-08 13:02:18 +090012652 }
12653 }
12654
David Stevensdeae4a12021-09-22 13:58:59 +090012655 if (kvm_page_track_create_memslot(kvm, slot, npages))
Xiao Guangrong21ebbed2016-02-24 17:51:09 +080012656 goto out_free;
12657
Takuya Yoshikawadb3fe4e2012-02-08 13:02:18 +090012658 return 0;
12659
12660out_free:
Ben Gardonc9b929b2021-05-18 10:34:08 -070012661 memslot_rmap_free(slot);
Takuya Yoshikawad89cc612012-08-01 18:03:28 +090012662
Ben Gardonc9b929b2021-05-18 10:34:08 -070012663 for (i = 1; i < KVM_NR_PAGE_SIZES; ++i) {
Thomas Huth548ef282015-02-24 21:29:25 +010012664 kvfree(slot->arch.lpage_info[i - 1]);
Takuya Yoshikawad89cc612012-08-01 18:03:28 +090012665 slot->arch.lpage_info[i - 1] = NULL;
Takuya Yoshikawadb3fe4e2012-02-08 13:02:18 +090012666 }
12667 return -ENOMEM;
12668}
12669
Sean Christopherson15248252019-02-05 12:54:17 -080012670void kvm_arch_memslots_updated(struct kvm *kvm, u64 gen)
Takuya Yoshikawae59dbe02013-07-04 13:40:29 +090012671{
Boris Ostrovsky91724812019-12-05 01:30:51 +000012672 struct kvm_vcpu *vcpu;
Marc Zyngier46808a42021-11-16 16:04:02 +000012673 unsigned long i;
Boris Ostrovsky91724812019-12-05 01:30:51 +000012674
Takuya Yoshikawae6dff7d2013-07-04 13:41:26 +090012675 /*
12676 * memslots->generation has been incremented.
12677 * mmio generation may have reached its maximum value.
12678 */
Sean Christopherson15248252019-02-05 12:54:17 -080012679 kvm_mmu_invalidate_mmio_sptes(kvm, gen);
Boris Ostrovsky91724812019-12-05 01:30:51 +000012680
12681 /* Force re-initialization of steal_time cache */
12682 kvm_for_each_vcpu(i, vcpu, kvm)
12683 kvm_vcpu_kick(vcpu);
Takuya Yoshikawae59dbe02013-07-04 13:40:29 +090012684}
12685
Marcelo Tosattif7784b82009-12-23 14:35:18 -020012686int kvm_arch_prepare_memory_region(struct kvm *kvm,
Sean Christopherson537a17b2021-12-06 20:54:11 +010012687 const struct kvm_memory_slot *old,
12688 struct kvm_memory_slot *new,
12689 enum kvm_mr_change change)
Zhang Xiantao0de10342007-11-20 16:25:04 +080012690{
Sean Christopherson86931ff2022-04-28 23:34:16 +000012691 if (change == KVM_MR_CREATE || change == KVM_MR_MOVE) {
12692 if ((new->base_gfn + new->npages - 1) > kvm_mmu_max_gfn())
12693 return -EINVAL;
12694
Sean Christopherson9d7d18e2021-12-06 20:54:16 +010012695 return kvm_alloc_memslot_metadata(kvm, new);
Sean Christopherson86931ff2022-04-28 23:34:16 +000012696 }
Sean Christopherson537a17b2021-12-06 20:54:11 +010012697
12698 if (change == KVM_MR_FLAGS_ONLY)
12699 memcpy(&new->arch, &old->arch, sizeof(old->arch));
12700 else if (WARN_ON_ONCE(change != KVM_MR_DELETE))
12701 return -EIO;
12702
Marcelo Tosattif7784b82009-12-23 14:35:18 -020012703 return 0;
12704}
12705
Makarand Sonarea85863c2021-02-12 16:50:12 -080012706
12707static void kvm_mmu_update_cpu_dirty_logging(struct kvm *kvm, bool enable)
12708{
12709 struct kvm_arch *ka = &kvm->arch;
12710
12711 if (!kvm_x86_ops.cpu_dirty_log_size)
12712 return;
12713
12714 if ((enable && ++ka->cpu_dirty_logging_count == 1) ||
12715 (!enable && --ka->cpu_dirty_logging_count == 0))
12716 kvm_make_all_cpus_request(kvm, KVM_REQ_UPDATE_CPU_DIRTY_LOGGING);
12717
12718 WARN_ON_ONCE(ka->cpu_dirty_logging_count < 0);
12719}
12720
Kai Huang88178fd2015-01-28 10:54:27 +080012721static void kvm_mmu_slot_apply_flags(struct kvm *kvm,
Anthony Yznaga37416792020-06-02 13:07:30 -070012722 struct kvm_memory_slot *old,
Hamza Mahfooz269e9552021-07-12 22:33:38 -040012723 const struct kvm_memory_slot *new,
Anthony Yznaga37416792020-06-02 13:07:30 -070012724 enum kvm_mr_change change)
Kai Huang88178fd2015-01-28 10:54:27 +080012725{
Sean Christopherson77aedf22021-12-06 20:54:20 +010012726 u32 old_flags = old ? old->flags : 0;
12727 u32 new_flags = new ? new->flags : 0;
12728 bool log_dirty_pages = new_flags & KVM_MEM_LOG_DIRTY_PAGES;
Makarand Sonarea85863c2021-02-12 16:50:12 -080012729
Anthony Yznaga37416792020-06-02 13:07:30 -070012730 /*
Makarand Sonarea85863c2021-02-12 16:50:12 -080012731 * Update CPU dirty logging if dirty logging is being toggled. This
12732 * applies to all operations.
12733 */
Sean Christopherson77aedf22021-12-06 20:54:20 +010012734 if ((old_flags ^ new_flags) & KVM_MEM_LOG_DIRTY_PAGES)
Makarand Sonarea85863c2021-02-12 16:50:12 -080012735 kvm_mmu_update_cpu_dirty_logging(kvm, log_dirty_pages);
12736
12737 /*
12738 * Nothing more to do for RO slots (which can't be dirtied and can't be
Sean Christophersonb6e16ae2021-02-12 16:50:13 -080012739 * made writable) or CREATE/MOVE/DELETE of a slot.
12740 *
12741 * For a memslot with dirty logging disabled:
12742 * CREATE: No dirty mappings will already exist.
12743 * MOVE/DELETE: The old mappings will already have been cleaned up by
12744 * kvm_arch_flush_shadow_memslot()
12745 *
12746 * For a memslot with dirty logging enabled:
12747 * CREATE: No shadow pages exist, thus nothing to write-protect
12748 * and no dirty bits to clear.
12749 * MOVE/DELETE: The old mappings will already have been cleaned up by
12750 * kvm_arch_flush_shadow_memslot().
Anthony Yznaga37416792020-06-02 13:07:30 -070012751 */
Sean Christopherson77aedf22021-12-06 20:54:20 +010012752 if ((change != KVM_MR_FLAGS_ONLY) || (new_flags & KVM_MEM_READONLY))
Kai Huang88178fd2015-01-28 10:54:27 +080012753 return;
Kai Huang88178fd2015-01-28 10:54:27 +080012754
12755 /*
Sean Christopherson52f460792021-02-12 16:50:11 -080012756 * READONLY and non-flags changes were filtered out above, and the only
12757 * other flag is LOG_DIRTY_PAGES, i.e. something is wrong if dirty
12758 * logging isn't being toggled on or off.
Anthony Yznaga37416792020-06-02 13:07:30 -070012759 */
Sean Christopherson77aedf22021-12-06 20:54:20 +010012760 if (WARN_ON_ONCE(!((old_flags ^ new_flags) & KVM_MEM_LOG_DIRTY_PAGES)))
Sean Christopherson52f460792021-02-12 16:50:11 -080012761 return;
12762
Sean Christophersonb6e16ae2021-02-12 16:50:13 -080012763 if (!log_dirty_pages) {
12764 /*
12765 * Dirty logging tracks sptes in 4k granularity, meaning that
12766 * large sptes have to be split. If live migration succeeds,
12767 * the guest in the source machine will be destroyed and large
12768 * sptes will be created in the destination. However, if the
12769 * guest continues to run in the source machine (for example if
12770 * live migration fails), small sptes will remain around and
12771 * cause bad performance.
12772 *
12773 * Scan sptes if dirty logging has been stopped, dropping those
12774 * which can be collapsed into a single large-page spte. Later
12775 * page faults will create the large-page sptes.
12776 */
Anthony Yznaga37416792020-06-02 13:07:30 -070012777 kvm_mmu_zap_collapsible_sptes(kvm, new);
Sean Christophersonb6e16ae2021-02-12 16:50:13 -080012778 } else {
Keqian Zhu89212912021-04-29 11:41:15 +080012779 /*
12780 * Initially-all-set does not require write protecting any page,
12781 * because they're all assumed to be dirty.
12782 */
12783 if (kvm_dirty_log_manual_protect_and_init_set(kvm))
12784 return;
Anthony Yznaga37416792020-06-02 13:07:30 -070012785
David Matlacka3fe5db2022-01-19 23:07:36 +000012786 if (READ_ONCE(eager_page_split))
12787 kvm_mmu_slot_try_split_huge_pages(kvm, new, PG_LEVEL_4K);
12788
Sean Christophersona018eba2021-02-12 16:50:10 -080012789 if (kvm_x86_ops.cpu_dirty_log_size) {
Keqian Zhu89212912021-04-29 11:41:15 +080012790 kvm_mmu_slot_leaf_clear_dirty(kvm, new);
12791 kvm_mmu_slot_remove_write_access(kvm, new, PG_LEVEL_2M);
12792 } else {
12793 kvm_mmu_slot_remove_write_access(kvm, new, PG_LEVEL_4K);
Jay Zhou3c9bd402020-02-27 09:32:27 +080012794 }
Junaid Shahidb64d7402022-08-10 15:49:39 -070012795
12796 /*
12797 * Unconditionally flush the TLBs after enabling dirty logging.
12798 * A flush is almost always going to be necessary (see below),
12799 * and unconditionally flushing allows the helpers to omit
12800 * the subtly complex checks when removing write access.
12801 *
12802 * Do the flush outside of mmu_lock to reduce the amount of
12803 * time mmu_lock is held. Flushing after dropping mmu_lock is
12804 * safe as KVM only needs to guarantee the slot is fully
12805 * write-protected before returning to userspace, i.e. before
12806 * userspace can consume the dirty status.
12807 *
12808 * Flushing outside of mmu_lock requires KVM to be careful when
12809 * making decisions based on writable status of an SPTE, e.g. a
12810 * !writable SPTE doesn't guarantee a CPU can't perform writes.
12811 *
12812 * Specifically, KVM also write-protects guest page tables to
12813 * monitor changes when using shadow paging, and must guarantee
12814 * no CPUs can write to those page before mmu_lock is dropped.
12815 * Because CPUs may have stale TLB entries at this point, a
12816 * !writable SPTE doesn't guarantee CPUs can't perform writes.
12817 *
12818 * KVM also allows making SPTES writable outside of mmu_lock,
12819 * e.g. to allow dirty logging without taking mmu_lock.
12820 *
12821 * To handle these scenarios, KVM uses a separate software-only
12822 * bit (MMU-writable) to track if a SPTE is !writable due to
12823 * a guest page table being write-protected (KVM clears the
12824 * MMU-writable flag when write-protecting for shadow paging).
12825 *
12826 * The use of MMU-writable is also the primary motivation for
12827 * the unconditional flush. Because KVM must guarantee that a
12828 * CPU doesn't contain stale, writable TLB entries for a
12829 * !MMU-writable SPTE, KVM must flush if it encounters any
12830 * MMU-writable SPTE regardless of whether the actual hardware
12831 * writable bit was set. I.e. KVM is almost guaranteed to need
12832 * to flush, while unconditionally flushing allows the "remove
12833 * write access" helpers to ignore MMU-writable entirely.
12834 *
12835 * See is_writable_pte() for more details (the case involving
12836 * access-tracked SPTEs is particularly relevant).
12837 */
12838 kvm_arch_flush_remote_tlbs_memslot(kvm, new);
Kai Huang88178fd2015-01-28 10:54:27 +080012839 }
12840}
12841
Marcelo Tosattif7784b82009-12-23 14:35:18 -020012842void kvm_arch_commit_memory_region(struct kvm *kvm,
Sean Christopherson9d4c1972020-02-18 13:07:24 -080012843 struct kvm_memory_slot *old,
Paolo Bonzinif36f3f22015-05-18 13:20:23 +020012844 const struct kvm_memory_slot *new,
Takuya Yoshikawa84826442013-02-27 19:45:25 +090012845 enum kvm_mr_change change)
Marcelo Tosattif7784b82009-12-23 14:35:18 -020012846{
Maciej S. Szmigieroe0c2b632021-12-06 20:54:23 +010012847 if (!kvm->arch.n_requested_mmu_pages &&
Maciej S. Szmigierof5756022021-12-06 20:54:24 +010012848 (change == KVM_MR_CREATE || change == KVM_MR_DELETE)) {
12849 unsigned long nr_mmu_pages;
12850
12851 nr_mmu_pages = kvm->nr_memslot_pages / KVM_MEMSLOT_PAGES_TO_MMU_PAGES_RATIO;
12852 nr_mmu_pages = max(nr_mmu_pages, KVM_MIN_ALLOC_MMU_PAGES);
12853 kvm_mmu_change_mmu_pages(kvm, nr_mmu_pages);
12854 }
Kai Huang1c91cad42015-01-28 10:54:26 +080012855
Hamza Mahfooz269e9552021-07-12 22:33:38 -040012856 kvm_mmu_slot_apply_flags(kvm, old, new, change);
Sean Christopherson21198842020-02-18 13:07:25 -080012857
12858 /* Free the arrays associated with the old memslot. */
12859 if (change == KVM_MR_MOVE)
Sean Christophersone96c81e2020-02-18 13:07:27 -080012860 kvm_arch_free_memslot(kvm, old);
Zhang Xiantao0de10342007-11-20 16:25:04 +080012861}
Zhang Xiantao1d737c82007-12-14 09:35:10 +080012862
Marcelo Tosatti2df72e92012-08-24 15:54:57 -030012863void kvm_arch_flush_shadow_all(struct kvm *kvm)
Marcelo Tosatti34d4cb82008-07-10 20:49:31 -030012864{
Sean Christopherson7390de12019-02-05 13:01:31 -080012865 kvm_mmu_zap_all(kvm);
Marcelo Tosatti34d4cb82008-07-10 20:49:31 -030012866}
12867
Marcelo Tosatti2df72e92012-08-24 15:54:57 -030012868void kvm_arch_flush_shadow_memslot(struct kvm *kvm,
12869 struct kvm_memory_slot *slot)
12870{
Xiaoguang Chenae7cd872016-10-09 15:41:44 +080012871 kvm_page_track_flush_slot(kvm, slot);
Marcelo Tosatti2df72e92012-08-24 15:54:57 -030012872}
12873
Liran Alone6c67d82018-09-04 10:56:52 +030012874static inline bool kvm_guest_apic_has_interrupt(struct kvm_vcpu *vcpu)
12875{
12876 return (is_guest_mode(vcpu) &&
Paolo Bonzini5be22262022-02-15 13:07:10 -050012877 static_call(kvm_x86_guest_apic_has_interrupt)(vcpu));
Liran Alone6c67d82018-09-04 10:56:52 +030012878}
12879
Paolo Bonzini5d9bc642015-10-13 10:18:53 +020012880static inline bool kvm_vcpu_has_events(struct kvm_vcpu *vcpu)
12881{
12882 if (!list_empty_careful(&vcpu->async_pf.done))
12883 return true;
12884
Paolo Bonzinibf7f9352022-09-21 00:31:54 +000012885 if (kvm_apic_has_pending_init_or_sipi(vcpu) &&
12886 kvm_apic_init_sipi_allowed(vcpu))
Paolo Bonzini5d9bc642015-10-13 10:18:53 +020012887 return true;
12888
12889 if (vcpu->arch.pv.pv_unhalted)
12890 return true;
12891
Sean Christopherson7709aba2022-08-30 23:16:08 +000012892 if (kvm_is_exception_pending(vcpu))
Wanpeng Lia5f01f82017-09-13 04:04:01 -070012893 return true;
12894
ZhuangYanying47a66ee2017-05-26 13:16:48 +080012895 if (kvm_test_request(KVM_REQ_NMI, vcpu) ||
12896 (vcpu->arch.nmi_pending &&
Jason Baronb36464772021-01-14 22:27:56 -050012897 static_call(kvm_x86_nmi_allowed)(vcpu, false)))
Paolo Bonzini5d9bc642015-10-13 10:18:53 +020012898 return true;
12899
ZhuangYanying47a66ee2017-05-26 13:16:48 +080012900 if (kvm_test_request(KVM_REQ_SMI, vcpu) ||
Paolo Bonzinia9fa7cb2020-04-23 11:02:36 -040012901 (vcpu->arch.smi_pending &&
Jason Baronb36464772021-01-14 22:27:56 -050012902 static_call(kvm_x86_smi_allowed)(vcpu, false)))
Paolo Bonzini73917732015-10-13 10:19:35 +020012903 return true;
12904
Paolo Bonzini5d9bc642015-10-13 10:18:53 +020012905 if (kvm_arch_interrupt_allowed(vcpu) &&
Liran Alone6c67d82018-09-04 10:56:52 +030012906 (kvm_cpu_has_interrupt(vcpu) ||
12907 kvm_guest_apic_has_interrupt(vcpu)))
Paolo Bonzini5d9bc642015-10-13 10:18:53 +020012908 return true;
12909
Andrey Smetanin1f4b34f2015-11-30 19:22:21 +030012910 if (kvm_hv_has_stimer_pending(vcpu))
12911 return true;
12912
Sean Christophersond2060bd2020-04-22 19:25:39 -070012913 if (is_guest_mode(vcpu) &&
Paolo Bonzini5b4ac1a2022-09-21 00:31:50 +000012914 kvm_x86_ops.nested_ops->has_events &&
12915 kvm_x86_ops.nested_ops->has_events(vcpu))
Sean Christophersond2060bd2020-04-22 19:25:39 -070012916 return true;
12917
David Woodhouse7caf9572022-03-03 15:41:15 +000012918 if (kvm_xen_has_pending_events(vcpu))
12919 return true;
12920
Paolo Bonzini5d9bc642015-10-13 10:18:53 +020012921 return false;
12922}
12923
Zhang Xiantao1d737c82007-12-14 09:35:10 +080012924int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
12925{
Paolo Bonzini5d9bc642015-10-13 10:18:53 +020012926 return kvm_vcpu_running(vcpu) || kvm_vcpu_has_events(vcpu);
Zhang Xiantao1d737c82007-12-14 09:35:10 +080012927}
Zhang Xiantao57361992007-12-17 14:21:40 +080012928
Haiwei Li10dbdf92021-04-21 11:25:13 +080012929bool kvm_arch_dy_has_pending_interrupt(struct kvm_vcpu *vcpu)
12930{
Sean Christophersonae801e12022-06-14 23:05:46 +000012931 if (kvm_vcpu_apicv_active(vcpu) &&
12932 static_call(kvm_x86_dy_apicv_has_pending_interrupt)(vcpu))
Haiwei Li10dbdf92021-04-21 11:25:13 +080012933 return true;
12934
12935 return false;
12936}
12937
Wanpeng Li17e433b2019-08-05 10:03:19 +080012938bool kvm_arch_dy_runnable(struct kvm_vcpu *vcpu)
12939{
12940 if (READ_ONCE(vcpu->arch.pv.pv_unhalted))
12941 return true;
12942
12943 if (kvm_test_request(KVM_REQ_NMI, vcpu) ||
12944 kvm_test_request(KVM_REQ_SMI, vcpu) ||
12945 kvm_test_request(KVM_REQ_EVENT, vcpu))
12946 return true;
12947
Haiwei Li10dbdf92021-04-21 11:25:13 +080012948 return kvm_arch_dy_has_pending_interrupt(vcpu);
Wanpeng Li17e433b2019-08-05 10:03:19 +080012949}
12950
Longpeng(Mike)199b5762017-08-08 12:05:32 +080012951bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu)
12952{
Wanpeng Lib86bb112021-04-22 16:34:19 +080012953 if (vcpu->arch.guest_state_protected)
12954 return true;
12955
Longpeng(Mike)de63ad42017-08-08 12:05:33 +080012956 return vcpu->arch.preempted_in_kernel;
Longpeng(Mike)199b5762017-08-08 12:05:32 +080012957}
12958
Sean Christophersone1bfc242021-11-11 02:07:33 +000012959unsigned long kvm_arch_vcpu_get_ip(struct kvm_vcpu *vcpu)
12960{
12961 return kvm_rip_read(vcpu);
12962}
12963
Christoffer Dallb6d33832012-03-08 16:44:24 -050012964int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
Zhang Xiantao57361992007-12-17 14:21:40 +080012965{
Christoffer Dallb6d33832012-03-08 16:44:24 -050012966 return kvm_vcpu_exiting_guest_mode(vcpu) == IN_GUEST_MODE;
Zhang Xiantao57361992007-12-17 14:21:40 +080012967}
Gleb Natapov78646122009-03-23 12:12:11 +020012968
12969int kvm_arch_interrupt_allowed(struct kvm_vcpu *vcpu)
12970{
Jason Baronb36464772021-01-14 22:27:56 -050012971 return static_call(kvm_x86_interrupt_allowed)(vcpu, false);
Gleb Natapov78646122009-03-23 12:12:11 +020012972}
Marcelo Tosatti229456f2009-06-17 09:22:14 -030012973
Nadav Amit82b32772014-11-02 11:54:45 +020012974unsigned long kvm_get_linear_rip(struct kvm_vcpu *vcpu)
12975{
Tom Lendacky7ed9abf2020-12-10 11:09:54 -060012976 /* Can't read the RIP when guest state is protected, just return 0 */
12977 if (vcpu->arch.guest_state_protected)
12978 return 0;
12979
Nadav Amit82b32772014-11-02 11:54:45 +020012980 if (is_64_bit_mode(vcpu))
12981 return kvm_rip_read(vcpu);
12982 return (u32)(get_segment_base(vcpu, VCPU_SREG_CS) +
12983 kvm_rip_read(vcpu));
12984}
12985EXPORT_SYMBOL_GPL(kvm_get_linear_rip);
12986
Jan Kiszkaf92653e2010-02-23 17:47:55 +010012987bool kvm_is_linear_rip(struct kvm_vcpu *vcpu, unsigned long linear_rip)
12988{
Nadav Amit82b32772014-11-02 11:54:45 +020012989 return kvm_get_linear_rip(vcpu) == linear_rip;
Jan Kiszkaf92653e2010-02-23 17:47:55 +010012990}
12991EXPORT_SYMBOL_GPL(kvm_is_linear_rip);
12992
Jan Kiszka94fe45d2009-10-18 13:24:44 +020012993unsigned long kvm_get_rflags(struct kvm_vcpu *vcpu)
12994{
12995 unsigned long rflags;
12996
Jason Baronb36464772021-01-14 22:27:56 -050012997 rflags = static_call(kvm_x86_get_rflags)(vcpu);
Jan Kiszka94fe45d2009-10-18 13:24:44 +020012998 if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP)
Jan Kiszkac310bac2010-02-23 17:47:58 +010012999 rflags &= ~X86_EFLAGS_TF;
Jan Kiszka94fe45d2009-10-18 13:24:44 +020013000 return rflags;
13001}
13002EXPORT_SYMBOL_GPL(kvm_get_rflags);
13003
Paolo Bonzini6addfc42014-03-27 11:29:28 +010013004static void __kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
Jan Kiszka94fe45d2009-10-18 13:24:44 +020013005{
13006 if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP &&
Jan Kiszkaf92653e2010-02-23 17:47:55 +010013007 kvm_is_linear_rip(vcpu, vcpu->arch.singlestep_rip))
Jan Kiszkac310bac2010-02-23 17:47:58 +010013008 rflags |= X86_EFLAGS_TF;
Jason Baronb36464772021-01-14 22:27:56 -050013009 static_call(kvm_x86_set_rflags)(vcpu, rflags);
Paolo Bonzini6addfc42014-03-27 11:29:28 +010013010}
13011
13012void kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
13013{
13014 __kvm_set_rflags(vcpu, rflags);
Avi Kivity3842d132010-07-27 12:30:24 +030013015 kvm_make_request(KVM_REQ_EVENT, vcpu);
Jan Kiszka94fe45d2009-10-18 13:24:44 +020013016}
13017EXPORT_SYMBOL_GPL(kvm_set_rflags);
13018
Gleb Natapovaf585b92010-10-14 11:22:46 +020013019static inline u32 kvm_async_pf_hash_fn(gfn_t gfn)
13020{
Peter Xudd03bca2020-04-16 11:58:59 -040013021 BUILD_BUG_ON(!is_power_of_2(ASYNC_PF_PER_VCPU));
13022
Gleb Natapovaf585b92010-10-14 11:22:46 +020013023 return hash_32(gfn & 0xffffffff, order_base_2(ASYNC_PF_PER_VCPU));
13024}
13025
13026static inline u32 kvm_async_pf_next_probe(u32 key)
13027{
Peter Xudd03bca2020-04-16 11:58:59 -040013028 return (key + 1) & (ASYNC_PF_PER_VCPU - 1);
Gleb Natapovaf585b92010-10-14 11:22:46 +020013029}
13030
13031static void kvm_add_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn)
13032{
13033 u32 key = kvm_async_pf_hash_fn(gfn);
13034
13035 while (vcpu->arch.apf.gfns[key] != ~0)
13036 key = kvm_async_pf_next_probe(key);
13037
13038 vcpu->arch.apf.gfns[key] = gfn;
13039}
13040
13041static u32 kvm_async_pf_gfn_slot(struct kvm_vcpu *vcpu, gfn_t gfn)
13042{
13043 int i;
13044 u32 key = kvm_async_pf_hash_fn(gfn);
13045
Peter Xudd03bca2020-04-16 11:58:59 -040013046 for (i = 0; i < ASYNC_PF_PER_VCPU &&
Xiao Guangrongc7d28c22010-11-01 17:00:30 +080013047 (vcpu->arch.apf.gfns[key] != gfn &&
13048 vcpu->arch.apf.gfns[key] != ~0); i++)
Gleb Natapovaf585b92010-10-14 11:22:46 +020013049 key = kvm_async_pf_next_probe(key);
13050
13051 return key;
13052}
13053
13054bool kvm_find_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn)
13055{
13056 return vcpu->arch.apf.gfns[kvm_async_pf_gfn_slot(vcpu, gfn)] == gfn;
13057}
13058
13059static void kvm_del_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn)
13060{
13061 u32 i, j, k;
13062
13063 i = j = kvm_async_pf_gfn_slot(vcpu, gfn);
Peter Xu0fd46042020-04-16 11:59:10 -040013064
13065 if (WARN_ON_ONCE(vcpu->arch.apf.gfns[i] != gfn))
13066 return;
13067
Gleb Natapovaf585b92010-10-14 11:22:46 +020013068 while (true) {
13069 vcpu->arch.apf.gfns[i] = ~0;
13070 do {
13071 j = kvm_async_pf_next_probe(j);
13072 if (vcpu->arch.apf.gfns[j] == ~0)
13073 return;
13074 k = kvm_async_pf_hash_fn(vcpu->arch.apf.gfns[j]);
13075 /*
13076 * k lies cyclically in ]i,j]
13077 * | i.k.j |
13078 * |....j i.k.| or |.k..j i...|
13079 */
13080 } while ((i <= j) ? (i < k && k <= j) : (i < k || k <= j));
13081 vcpu->arch.apf.gfns[i] = vcpu->arch.apf.gfns[j];
13082 i = j;
13083 }
13084}
13085
Vitaly Kuznetsov68fd66f2020-05-25 16:41:17 +020013086static inline int apf_put_user_notpresent(struct kvm_vcpu *vcpu)
Gleb Natapov7c907052010-10-14 11:22:53 +020013087{
Vitaly Kuznetsov68fd66f2020-05-25 16:41:17 +020013088 u32 reason = KVM_PV_REASON_PAGE_NOT_PRESENT;
13089
13090 return kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.apf.data, &reason,
13091 sizeof(reason));
13092}
13093
13094static inline int apf_put_user_ready(struct kvm_vcpu *vcpu, u32 token)
13095{
Vitaly Kuznetsov2635b5c2020-05-25 16:41:20 +020013096 unsigned int offset = offsetof(struct kvm_vcpu_pv_apf_data, token);
Paolo Bonzini4e335d92017-05-02 16:20:18 +020013097
Vitaly Kuznetsov2635b5c2020-05-25 16:41:20 +020013098 return kvm_write_guest_offset_cached(vcpu->kvm, &vcpu->arch.apf.data,
13099 &token, offset, sizeof(token));
13100}
13101
13102static inline bool apf_pageready_slot_free(struct kvm_vcpu *vcpu)
13103{
13104 unsigned int offset = offsetof(struct kvm_vcpu_pv_apf_data, token);
13105 u32 val;
13106
13107 if (kvm_read_guest_offset_cached(vcpu->kvm, &vcpu->arch.apf.data,
13108 &val, offset, sizeof(val)))
13109 return false;
13110
13111 return !val;
Gleb Natapov7c907052010-10-14 11:22:53 +020013112}
13113
Paolo Bonzini1dfdb452019-06-05 16:46:44 +020013114static bool kvm_can_deliver_async_pf(struct kvm_vcpu *vcpu)
13115{
Paolo Bonzini57cb3bb2022-02-09 05:17:38 -050013116
13117 if (!kvm_pv_async_pf_enabled(vcpu))
Paolo Bonzini1dfdb452019-06-05 16:46:44 +020013118 return false;
13119
Paolo Bonzini57cb3bb2022-02-09 05:17:38 -050013120 if (vcpu->arch.apf.send_user_only &&
13121 static_call(kvm_x86_get_cpl)(vcpu) == 0)
Paolo Bonzini1dfdb452019-06-05 16:46:44 +020013122 return false;
13123
Paolo Bonzini57cb3bb2022-02-09 05:17:38 -050013124 if (is_guest_mode(vcpu)) {
13125 /*
13126 * L1 needs to opt into the special #PF vmexits that are
13127 * used to deliver async page faults.
13128 */
13129 return vcpu->arch.apf.delivery_as_pf_vmexit;
13130 } else {
13131 /*
13132 * Play it safe in case the guest temporarily disables paging.
13133 * The real mode IDT in particular is unlikely to have a #PF
13134 * exception setup.
13135 */
13136 return is_paging(vcpu);
13137 }
Paolo Bonzini1dfdb452019-06-05 16:46:44 +020013138}
13139
13140bool kvm_can_do_async_pf(struct kvm_vcpu *vcpu)
13141{
13142 if (unlikely(!lapic_in_kernel(vcpu) ||
13143 kvm_event_needs_reinjection(vcpu) ||
Sean Christopherson7709aba2022-08-30 23:16:08 +000013144 kvm_is_exception_pending(vcpu)))
Paolo Bonzini1dfdb452019-06-05 16:46:44 +020013145 return false;
13146
13147 if (kvm_hlt_in_guest(vcpu->kvm) && !kvm_can_deliver_async_pf(vcpu))
13148 return false;
13149
13150 /*
13151 * If interrupts are off we cannot even use an artificial
13152 * halt state.
13153 */
Paolo Bonzinic300ab92020-04-23 14:08:58 -040013154 return kvm_arch_interrupt_allowed(vcpu);
Paolo Bonzini1dfdb452019-06-05 16:46:44 +020013155}
13156
Vitaly Kuznetsov2a18b7e2020-06-10 19:55:32 +020013157bool kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
Gleb Natapovaf585b92010-10-14 11:22:46 +020013158 struct kvm_async_pf *work)
13159{
Avi Kivity6389ee92010-11-29 16:12:30 +020013160 struct x86_exception fault;
13161
Sean Christopherson736c2912019-12-06 15:57:14 -080013162 trace_kvm_async_pf_not_present(work->arch.token, work->cr2_or_gpa);
Gleb Natapovaf585b92010-10-14 11:22:46 +020013163 kvm_add_async_pf_gfn(vcpu, work->arch.gfn);
Gleb Natapov7c907052010-10-14 11:22:53 +020013164
Paolo Bonzini1dfdb452019-06-05 16:46:44 +020013165 if (kvm_can_deliver_async_pf(vcpu) &&
Vitaly Kuznetsov68fd66f2020-05-25 16:41:17 +020013166 !apf_put_user_notpresent(vcpu)) {
Avi Kivity6389ee92010-11-29 16:12:30 +020013167 fault.vector = PF_VECTOR;
13168 fault.error_code_valid = true;
13169 fault.error_code = 0;
13170 fault.nested_page_fault = false;
13171 fault.address = work->arch.token;
Wanpeng Liadfe20f2017-07-13 18:30:41 -070013172 fault.async_page_fault = true;
Avi Kivity6389ee92010-11-29 16:12:30 +020013173 kvm_inject_page_fault(vcpu, &fault);
Vitaly Kuznetsov2a18b7e2020-06-10 19:55:32 +020013174 return true;
Paolo Bonzini1dfdb452019-06-05 16:46:44 +020013175 } else {
13176 /*
13177 * It is not possible to deliver a paravirtualized asynchronous
13178 * page fault, but putting the guest in an artificial halt state
13179 * can be beneficial nevertheless: if an interrupt arrives, we
13180 * can deliver it timely and perhaps the guest will schedule
13181 * another process. When the instruction that triggered a page
13182 * fault is retried, hopefully the page will be ready in the host.
13183 */
13184 kvm_make_request(KVM_REQ_APF_HALT, vcpu);
Vitaly Kuznetsov2a18b7e2020-06-10 19:55:32 +020013185 return false;
Gleb Natapov7c907052010-10-14 11:22:53 +020013186 }
Gleb Natapovaf585b92010-10-14 11:22:46 +020013187}
13188
13189void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
13190 struct kvm_async_pf *work)
13191{
Vitaly Kuznetsov2635b5c2020-05-25 16:41:20 +020013192 struct kvm_lapic_irq irq = {
13193 .delivery_mode = APIC_DM_FIXED,
13194 .vector = vcpu->arch.apf.vec
13195 };
Avi Kivity6389ee92010-11-29 16:12:30 +020013196
chai wenf2e10662013-10-14 22:22:33 +080013197 if (work->wakeup_all)
Gleb Natapov7c907052010-10-14 11:22:53 +020013198 work->arch.token = ~0; /* broadcast wakeup */
13199 else
13200 kvm_del_async_pf_gfn(vcpu, work->arch.gfn);
Sean Christopherson736c2912019-12-06 15:57:14 -080013201 trace_kvm_async_pf_ready(work->arch.token, work->cr2_or_gpa);
Gleb Natapov7c907052010-10-14 11:22:53 +020013202
Vitaly Kuznetsov2a18b7e2020-06-10 19:55:32 +020013203 if ((work->wakeup_all || work->notpresent_injected) &&
13204 kvm_pv_async_pf_enabled(vcpu) &&
Vitaly Kuznetsov557a9612020-05-25 16:41:21 +020013205 !apf_put_user_ready(vcpu, work->arch.token)) {
13206 vcpu->arch.apf.pageready_pending = true;
Vitaly Kuznetsov2635b5c2020-05-25 16:41:20 +020013207 kvm_apic_set_irq(vcpu, &irq, NULL);
Vitaly Kuznetsov557a9612020-05-25 16:41:21 +020013208 }
Vitaly Kuznetsov2635b5c2020-05-25 16:41:20 +020013209
Xiao Guangronge6d53e32010-11-01 17:01:28 +080013210 vcpu->arch.apf.halted = false;
Gleb Natapova4fa1632012-05-03 11:36:39 +030013211 vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
Gleb Natapov7c907052010-10-14 11:22:53 +020013212}
13213
Vitaly Kuznetsov557a9612020-05-25 16:41:21 +020013214void kvm_arch_async_page_present_queued(struct kvm_vcpu *vcpu)
13215{
13216 kvm_make_request(KVM_REQ_APF_READY, vcpu);
13217 if (!vcpu->arch.apf.pageready_pending)
13218 kvm_vcpu_kick(vcpu);
13219}
13220
Vitaly Kuznetsov7c0ade62020-05-25 16:41:18 +020013221bool kvm_arch_can_dequeue_async_page_present(struct kvm_vcpu *vcpu)
Gleb Natapov7c907052010-10-14 11:22:53 +020013222{
Vitaly Kuznetsov2635b5c2020-05-25 16:41:20 +020013223 if (!kvm_pv_async_pf_enabled(vcpu))
Gleb Natapov7c907052010-10-14 11:22:53 +020013224 return true;
13225 else
Vitaly Kuznetsov2f15d022021-04-22 11:29:48 +020013226 return kvm_lapic_enabled(vcpu) && apf_pageready_slot_free(vcpu);
Gleb Natapovaf585b92010-10-14 11:22:46 +020013227}
13228
Paolo Bonzini5544eb92015-07-07 15:41:58 +020013229void kvm_arch_start_assignment(struct kvm *kvm)
13230{
Marcelo Tosatti57ab8792021-05-25 10:41:16 -030013231 if (atomic_inc_return(&kvm->arch.assigned_device_count) == 1)
Sean Christophersone27bc042022-01-28 00:51:50 +000013232 static_call_cond(kvm_x86_pi_start_assignment)(kvm);
Paolo Bonzini5544eb92015-07-07 15:41:58 +020013233}
13234EXPORT_SYMBOL_GPL(kvm_arch_start_assignment);
13235
13236void kvm_arch_end_assignment(struct kvm *kvm)
13237{
13238 atomic_dec(&kvm->arch.assigned_device_count);
13239}
13240EXPORT_SYMBOL_GPL(kvm_arch_end_assignment);
13241
Peter Zijlstra742ab6d2022-06-14 23:15:32 +020013242bool noinstr kvm_arch_has_assigned_device(struct kvm *kvm)
Paolo Bonzini5544eb92015-07-07 15:41:58 +020013243{
Peter Zijlstra742ab6d2022-06-14 23:15:32 +020013244 return arch_atomic_read(&kvm->arch.assigned_device_count);
Paolo Bonzini5544eb92015-07-07 15:41:58 +020013245}
13246EXPORT_SYMBOL_GPL(kvm_arch_has_assigned_device);
13247
Alex Williamsone0f0bbc2013-10-30 11:02:30 -060013248void kvm_arch_register_noncoherent_dma(struct kvm *kvm)
13249{
13250 atomic_inc(&kvm->arch.noncoherent_dma_count);
13251}
13252EXPORT_SYMBOL_GPL(kvm_arch_register_noncoherent_dma);
13253
13254void kvm_arch_unregister_noncoherent_dma(struct kvm *kvm)
13255{
13256 atomic_dec(&kvm->arch.noncoherent_dma_count);
13257}
13258EXPORT_SYMBOL_GPL(kvm_arch_unregister_noncoherent_dma);
13259
13260bool kvm_arch_has_noncoherent_dma(struct kvm *kvm)
13261{
13262 return atomic_read(&kvm->arch.noncoherent_dma_count);
13263}
13264EXPORT_SYMBOL_GPL(kvm_arch_has_noncoherent_dma);
13265
Alex Williamson14717e22016-05-05 11:58:35 -060013266bool kvm_arch_has_irq_bypass(void)
13267{
Sean Christopherson92735b12019-08-02 15:06:17 -070013268 return true;
Alex Williamson14717e22016-05-05 11:58:35 -060013269}
13270
Feng Wu87276882015-09-18 22:29:40 +080013271int kvm_arch_irq_bypass_add_producer(struct irq_bypass_consumer *cons,
13272 struct irq_bypass_producer *prod)
13273{
13274 struct kvm_kernel_irqfd *irqfd =
13275 container_of(cons, struct kvm_kernel_irqfd, consumer);
Zhu Lingshan2edd9cb2020-07-31 14:55:29 +080013276 int ret;
Feng Wu87276882015-09-18 22:29:40 +080013277
Alex Williamson14717e22016-05-05 11:58:35 -060013278 irqfd->producer = prod;
Zhu Lingshan2edd9cb2020-07-31 14:55:29 +080013279 kvm_arch_start_assignment(irqfd->kvm);
Sean Christophersone27bc042022-01-28 00:51:50 +000013280 ret = static_call(kvm_x86_pi_update_irte)(irqfd->kvm,
Zhu Lingshan2edd9cb2020-07-31 14:55:29 +080013281 prod->irq, irqfd->gsi, 1);
Feng Wu87276882015-09-18 22:29:40 +080013282
Zhu Lingshan2edd9cb2020-07-31 14:55:29 +080013283 if (ret)
13284 kvm_arch_end_assignment(irqfd->kvm);
13285
13286 return ret;
Feng Wu87276882015-09-18 22:29:40 +080013287}
13288
13289void kvm_arch_irq_bypass_del_producer(struct irq_bypass_consumer *cons,
13290 struct irq_bypass_producer *prod)
13291{
13292 int ret;
13293 struct kvm_kernel_irqfd *irqfd =
13294 container_of(cons, struct kvm_kernel_irqfd, consumer);
13295
Feng Wu87276882015-09-18 22:29:40 +080013296 WARN_ON(irqfd->producer != prod);
13297 irqfd->producer = NULL;
13298
13299 /*
13300 * When producer of consumer is unregistered, we change back to
13301 * remapped mode, so we can re-use the current implementation
Andrea Gelminibb3541f2016-05-21 14:14:44 +020013302 * when the irq is masked/disabled or the consumer side (KVM
Feng Wu87276882015-09-18 22:29:40 +080013303 * int this case doesn't want to receive the interrupts.
13304 */
Sean Christophersone27bc042022-01-28 00:51:50 +000013305 ret = static_call(kvm_x86_pi_update_irte)(irqfd->kvm, prod->irq, irqfd->gsi, 0);
Feng Wu87276882015-09-18 22:29:40 +080013306 if (ret)
13307 printk(KERN_INFO "irq bypass consumer (token %p) unregistration"
13308 " fails: %d\n", irqfd->consumer.token, ret);
Zhu Lingshan2edd9cb2020-07-31 14:55:29 +080013309
13310 kvm_arch_end_assignment(irqfd->kvm);
Feng Wu87276882015-09-18 22:29:40 +080013311}
13312
13313int kvm_arch_update_irqfd_routing(struct kvm *kvm, unsigned int host_irq,
13314 uint32_t guest_irq, bool set)
13315{
Sean Christophersone27bc042022-01-28 00:51:50 +000013316 return static_call(kvm_x86_pi_update_irte)(kvm, host_irq, guest_irq, set);
Feng Wu87276882015-09-18 22:29:40 +080013317}
13318
Longpeng(Mike)515a0c72021-08-27 16:00:03 +080013319bool kvm_arch_irqfd_route_changed(struct kvm_kernel_irq_routing_entry *old,
13320 struct kvm_kernel_irq_routing_entry *new)
13321{
13322 if (new->type != KVM_IRQ_ROUTING_MSI)
13323 return true;
13324
13325 return !!memcmp(&old->msi, &new->msi, sizeof(new->msi));
13326}
13327
Feng Wu520040142016-01-25 16:53:33 +080013328bool kvm_vector_hashing_enabled(void)
13329{
13330 return vector_hashing;
13331}
Feng Wu520040142016-01-25 16:53:33 +080013332
Marcelo Tosatti2d5ba192019-06-03 19:52:44 -030013333bool kvm_arch_no_poll(struct kvm_vcpu *vcpu)
13334{
13335 return (vcpu->arch.msr_kvm_poll_control & 1) == 0;
13336}
13337EXPORT_SYMBOL_GPL(kvm_arch_no_poll);
13338
Maxim Levitsky841c2be2020-07-08 14:57:31 +030013339
13340int kvm_spec_ctrl_test_value(u64 value)
Paolo Bonzini6441fa62020-01-20 16:33:06 +010013341{
Maxim Levitsky841c2be2020-07-08 14:57:31 +030013342 /*
13343 * test that setting IA32_SPEC_CTRL to given value
13344 * is allowed by the host processor
13345 */
Paolo Bonzini6441fa62020-01-20 16:33:06 +010013346
Maxim Levitsky841c2be2020-07-08 14:57:31 +030013347 u64 saved_value;
13348 unsigned long flags;
13349 int ret = 0;
Paolo Bonzini6441fa62020-01-20 16:33:06 +010013350
Maxim Levitsky841c2be2020-07-08 14:57:31 +030013351 local_irq_save(flags);
Paolo Bonzini6441fa62020-01-20 16:33:06 +010013352
Maxim Levitsky841c2be2020-07-08 14:57:31 +030013353 if (rdmsrl_safe(MSR_IA32_SPEC_CTRL, &saved_value))
13354 ret = 1;
13355 else if (wrmsrl_safe(MSR_IA32_SPEC_CTRL, value))
13356 ret = 1;
13357 else
13358 wrmsrl(MSR_IA32_SPEC_CTRL, saved_value);
13359
13360 local_irq_restore(flags);
13361
13362 return ret;
Paolo Bonzini6441fa62020-01-20 16:33:06 +010013363}
Maxim Levitsky841c2be2020-07-08 14:57:31 +030013364EXPORT_SYMBOL_GPL(kvm_spec_ctrl_test_value);
Marcelo Tosatti2d5ba192019-06-03 19:52:44 -030013365
Mohammed Gamal89786142020-07-10 17:48:03 +020013366void kvm_fixup_and_inject_pf_error(struct kvm_vcpu *vcpu, gva_t gva, u16 error_code)
13367{
Lai Jiangshan1f5a21e2021-11-24 20:20:44 +080013368 struct kvm_mmu *mmu = vcpu->arch.walk_mmu;
Mohammed Gamal89786142020-07-10 17:48:03 +020013369 struct x86_exception fault;
Lai Jiangshan5b22bbe2022-03-11 15:03:41 +080013370 u64 access = error_code &
Paolo Bonzini19cf4b72020-08-17 13:53:04 -040013371 (PFERR_WRITE_MASK | PFERR_FETCH_MASK | PFERR_USER_MASK);
Mohammed Gamal89786142020-07-10 17:48:03 +020013372
13373 if (!(error_code & PFERR_PRESENT_MASK) ||
Hou Wenlong6e1d2a32022-07-01 17:24:13 +080013374 mmu->gva_to_gpa(vcpu, mmu, gva, access, &fault) != INVALID_GPA) {
Mohammed Gamal89786142020-07-10 17:48:03 +020013375 /*
13376 * If vcpu->arch.walk_mmu->gva_to_gpa succeeded, the page
13377 * tables probably do not match the TLB. Just proceed
13378 * with the error code that the processor gave.
13379 */
13380 fault.vector = PF_VECTOR;
13381 fault.error_code_valid = true;
13382 fault.error_code = error_code;
13383 fault.nested_page_fault = false;
13384 fault.address = gva;
Yu Zhang2bc685e2022-07-18 15:47:56 +080013385 fault.async_page_fault = false;
Mohammed Gamal89786142020-07-10 17:48:03 +020013386 }
13387 vcpu->arch.walk_mmu->inject_page_fault(vcpu, &fault);
13388}
13389EXPORT_SYMBOL_GPL(kvm_fixup_and_inject_pf_error);
Carsten Otte043405e2007-10-10 17:16:19 +020013390
Babu Moger3f3393b2020-09-11 14:29:05 -050013391/*
13392 * Handles kvm_read/write_guest_virt*() result and either injects #PF or returns
13393 * KVM_EXIT_INTERNAL_ERROR for cases not currently handled by KVM. Return value
13394 * indicates whether exit to userspace is needed.
13395 */
13396int kvm_handle_memory_failure(struct kvm_vcpu *vcpu, int r,
13397 struct x86_exception *e)
13398{
13399 if (r == X86EMUL_PROPAGATE_FAULT) {
13400 kvm_inject_emulated_page_fault(vcpu, e);
13401 return 1;
13402 }
13403
13404 /*
13405 * In case kvm_read/write_guest_virt*() failed with X86EMUL_IO_NEEDED
13406 * while handling a VMX instruction KVM could've handled the request
13407 * correctly by exiting to userspace and performing I/O but there
13408 * doesn't seem to be a real use-case behind such requests, just return
13409 * KVM_EXIT_INTERNAL_ERROR for now.
13410 */
David Edmondsone615e352021-09-20 11:37:36 +010013411 kvm_prepare_emulation_failure_exit(vcpu);
Babu Moger3f3393b2020-09-11 14:29:05 -050013412
13413 return 0;
13414}
13415EXPORT_SYMBOL_GPL(kvm_handle_memory_failure);
13416
Babu Moger97150922020-09-11 14:29:12 -050013417int kvm_handle_invpcid(struct kvm_vcpu *vcpu, unsigned long type, gva_t gva)
13418{
13419 bool pcid_enabled;
13420 struct x86_exception e;
Babu Moger97150922020-09-11 14:29:12 -050013421 struct {
13422 u64 pcid;
13423 u64 gla;
13424 } operand;
13425 int r;
13426
13427 r = kvm_read_guest_virt(vcpu, gva, &operand, sizeof(operand), &e);
13428 if (r != X86EMUL_CONTINUE)
13429 return kvm_handle_memory_failure(vcpu, r, &e);
13430
13431 if (operand.pcid >> 12 != 0) {
13432 kvm_inject_gp(vcpu, 0);
13433 return 1;
13434 }
13435
13436 pcid_enabled = kvm_read_cr4_bits(vcpu, X86_CR4_PCIDE);
13437
13438 switch (type) {
13439 case INVPCID_TYPE_INDIV_ADDR:
13440 if ((!pcid_enabled && (operand.pcid != 0)) ||
13441 is_noncanonical_address(operand.gla, vcpu)) {
13442 kvm_inject_gp(vcpu, 0);
13443 return 1;
13444 }
13445 kvm_mmu_invpcid_gva(vcpu, operand.gla, operand.pcid);
13446 return kvm_skip_emulated_instruction(vcpu);
13447
13448 case INVPCID_TYPE_SINGLE_CTXT:
13449 if (!pcid_enabled && (operand.pcid != 0)) {
13450 kvm_inject_gp(vcpu, 0);
13451 return 1;
13452 }
13453
Sean Christopherson21823fb2021-06-09 16:42:24 -070013454 kvm_invalidate_pcid(vcpu, operand.pcid);
Babu Moger97150922020-09-11 14:29:12 -050013455 return kvm_skip_emulated_instruction(vcpu);
13456
13457 case INVPCID_TYPE_ALL_NON_GLOBAL:
13458 /*
13459 * Currently, KVM doesn't mark global entries in the shadow
13460 * page tables, so a non-global flush just degenerates to a
13461 * global flush. If needed, we could optimize this later by
13462 * keeping track of global entries in shadow page tables.
13463 */
13464
13465 fallthrough;
13466 case INVPCID_TYPE_ALL_INCL_GLOBAL:
Sean Christopherson28f28d42021-06-09 16:42:30 -070013467 kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu);
Babu Moger97150922020-09-11 14:29:12 -050013468 return kvm_skip_emulated_instruction(vcpu);
13469
13470 default:
Vipin Sharma796c83c2021-11-09 17:44:26 +000013471 kvm_inject_gp(vcpu, 0);
13472 return 1;
Babu Moger97150922020-09-11 14:29:12 -050013473 }
13474}
13475EXPORT_SYMBOL_GPL(kvm_handle_invpcid);
13476
Tom Lendacky8f423a82020-12-10 11:09:53 -060013477static int complete_sev_es_emulated_mmio(struct kvm_vcpu *vcpu)
13478{
13479 struct kvm_run *run = vcpu->run;
13480 struct kvm_mmio_fragment *frag;
13481 unsigned int len;
13482
13483 BUG_ON(!vcpu->mmio_needed);
13484
13485 /* Complete previous fragment */
13486 frag = &vcpu->mmio_fragments[vcpu->mmio_cur_fragment];
13487 len = min(8u, frag->len);
13488 if (!vcpu->mmio_is_write)
13489 memcpy(frag->data, run->mmio.data, len);
13490
13491 if (frag->len <= 8) {
13492 /* Switch to the next fragment. */
13493 frag++;
13494 vcpu->mmio_cur_fragment++;
13495 } else {
13496 /* Go forward to the next mmio piece. */
13497 frag->data += len;
13498 frag->gpa += len;
13499 frag->len -= len;
13500 }
13501
13502 if (vcpu->mmio_cur_fragment >= vcpu->mmio_nr_fragments) {
13503 vcpu->mmio_needed = 0;
13504
13505 // VMG change, at this point, we're always done
13506 // RIP has already been advanced
13507 return 1;
13508 }
13509
13510 // More MMIO is needed
13511 run->mmio.phys_addr = frag->gpa;
13512 run->mmio.len = min(8u, frag->len);
13513 run->mmio.is_write = vcpu->mmio_is_write;
13514 if (run->mmio.is_write)
13515 memcpy(run->mmio.data, frag->data, min(8u, frag->len));
13516 run->exit_reason = KVM_EXIT_MMIO;
13517
13518 vcpu->arch.complete_userspace_io = complete_sev_es_emulated_mmio;
13519
13520 return 0;
13521}
13522
13523int kvm_sev_es_mmio_write(struct kvm_vcpu *vcpu, gpa_t gpa, unsigned int bytes,
13524 void *data)
13525{
13526 int handled;
13527 struct kvm_mmio_fragment *frag;
13528
13529 if (!data)
13530 return -EINVAL;
13531
13532 handled = write_emultor.read_write_mmio(vcpu, gpa, bytes, data);
13533 if (handled == bytes)
13534 return 1;
13535
13536 bytes -= handled;
13537 gpa += handled;
13538 data += handled;
13539
13540 /*TODO: Check if need to increment number of frags */
13541 frag = vcpu->mmio_fragments;
13542 vcpu->mmio_nr_fragments = 1;
13543 frag->len = bytes;
13544 frag->gpa = gpa;
13545 frag->data = data;
13546
13547 vcpu->mmio_needed = 1;
13548 vcpu->mmio_cur_fragment = 0;
13549
13550 vcpu->run->mmio.phys_addr = gpa;
13551 vcpu->run->mmio.len = min(8u, frag->len);
13552 vcpu->run->mmio.is_write = 1;
13553 memcpy(vcpu->run->mmio.data, frag->data, min(8u, frag->len));
13554 vcpu->run->exit_reason = KVM_EXIT_MMIO;
13555
13556 vcpu->arch.complete_userspace_io = complete_sev_es_emulated_mmio;
13557
13558 return 0;
13559}
13560EXPORT_SYMBOL_GPL(kvm_sev_es_mmio_write);
13561
13562int kvm_sev_es_mmio_read(struct kvm_vcpu *vcpu, gpa_t gpa, unsigned int bytes,
13563 void *data)
13564{
13565 int handled;
13566 struct kvm_mmio_fragment *frag;
13567
13568 if (!data)
13569 return -EINVAL;
13570
13571 handled = read_emultor.read_write_mmio(vcpu, gpa, bytes, data);
13572 if (handled == bytes)
13573 return 1;
13574
13575 bytes -= handled;
13576 gpa += handled;
13577 data += handled;
13578
13579 /*TODO: Check if need to increment number of frags */
13580 frag = vcpu->mmio_fragments;
13581 vcpu->mmio_nr_fragments = 1;
13582 frag->len = bytes;
13583 frag->gpa = gpa;
13584 frag->data = data;
13585
13586 vcpu->mmio_needed = 1;
13587 vcpu->mmio_cur_fragment = 0;
13588
13589 vcpu->run->mmio.phys_addr = gpa;
13590 vcpu->run->mmio.len = min(8u, frag->len);
13591 vcpu->run->mmio.is_write = 0;
13592 vcpu->run->exit_reason = KVM_EXIT_MMIO;
13593
13594 vcpu->arch.complete_userspace_io = complete_sev_es_emulated_mmio;
13595
13596 return 0;
13597}
13598EXPORT_SYMBOL_GPL(kvm_sev_es_mmio_read);
13599
Paolo Bonzinidb209362021-10-22 08:47:56 -040013600static void advance_sev_es_emulated_pio(struct kvm_vcpu *vcpu, unsigned count, int size)
13601{
13602 vcpu->arch.sev_pio_count -= count;
13603 vcpu->arch.sev_pio_data += count * size;
13604}
13605
Tom Lendacky7ed9abf2020-12-10 11:09:54 -060013606static int kvm_sev_es_outs(struct kvm_vcpu *vcpu, unsigned int size,
Paolo Bonzini95e16b42021-10-12 11:33:03 -040013607 unsigned int port);
Tom Lendacky7ed9abf2020-12-10 11:09:54 -060013608
Paolo Bonzini95e16b42021-10-12 11:33:03 -040013609static int complete_sev_es_emulated_outs(struct kvm_vcpu *vcpu)
13610{
13611 int size = vcpu->arch.pio.size;
13612 int port = vcpu->arch.pio.port;
Tom Lendacky7ed9abf2020-12-10 11:09:54 -060013613
13614 vcpu->arch.pio.count = 0;
Paolo Bonzini95e16b42021-10-12 11:33:03 -040013615 if (vcpu->arch.sev_pio_count)
13616 return kvm_sev_es_outs(vcpu, size, port);
13617 return 1;
13618}
13619
13620static int kvm_sev_es_outs(struct kvm_vcpu *vcpu, unsigned int size,
13621 unsigned int port)
13622{
13623 for (;;) {
13624 unsigned int count =
13625 min_t(unsigned int, PAGE_SIZE / size, vcpu->arch.sev_pio_count);
13626 int ret = emulator_pio_out(vcpu, size, port, vcpu->arch.sev_pio_data, count);
13627
13628 /* memcpy done already by emulator_pio_out. */
Paolo Bonzinidb209362021-10-22 08:47:56 -040013629 advance_sev_es_emulated_pio(vcpu, count, size);
Paolo Bonzini95e16b42021-10-12 11:33:03 -040013630 if (!ret)
13631 break;
13632
13633 /* Emulation done by the kernel. */
13634 if (!vcpu->arch.sev_pio_count)
13635 return 1;
13636 }
13637
13638 vcpu->arch.complete_userspace_io = complete_sev_es_emulated_outs;
Tom Lendacky7ed9abf2020-12-10 11:09:54 -060013639 return 0;
13640}
13641
Paolo Bonzini95e16b42021-10-12 11:33:03 -040013642static int kvm_sev_es_ins(struct kvm_vcpu *vcpu, unsigned int size,
13643 unsigned int port);
13644
Paolo Bonzini4fa4b382021-10-12 11:25:45 -040013645static int complete_sev_es_emulated_ins(struct kvm_vcpu *vcpu)
13646{
Paolo Bonzini0c05e102022-06-15 10:24:01 -040013647 unsigned count = vcpu->arch.pio.count;
Paolo Bonzini95e16b42021-10-12 11:33:03 -040013648 int size = vcpu->arch.pio.size;
13649 int port = vcpu->arch.pio.port;
Paolo Bonzini4fa4b382021-10-12 11:25:45 -040013650
Paolo Bonzini0c05e102022-06-15 10:24:01 -040013651 complete_emulator_pio_in(vcpu, vcpu->arch.sev_pio_data);
Paolo Bonzinidb209362021-10-22 08:47:56 -040013652 advance_sev_es_emulated_pio(vcpu, count, size);
Paolo Bonzini95e16b42021-10-12 11:33:03 -040013653 if (vcpu->arch.sev_pio_count)
13654 return kvm_sev_es_ins(vcpu, size, port);
Paolo Bonzini4fa4b382021-10-12 11:25:45 -040013655 return 1;
13656}
13657
Tom Lendacky7ed9abf2020-12-10 11:09:54 -060013658static int kvm_sev_es_ins(struct kvm_vcpu *vcpu, unsigned int size,
Paolo Bonzini95e16b42021-10-12 11:33:03 -040013659 unsigned int port)
Tom Lendacky7ed9abf2020-12-10 11:09:54 -060013660{
Paolo Bonzini95e16b42021-10-12 11:33:03 -040013661 for (;;) {
13662 unsigned int count =
13663 min_t(unsigned int, PAGE_SIZE / size, vcpu->arch.sev_pio_count);
Paolo Bonzinif35cee42021-10-22 08:19:48 -040013664 if (!emulator_pio_in(vcpu, size, port, vcpu->arch.sev_pio_data, count))
Paolo Bonzini95e16b42021-10-12 11:33:03 -040013665 break;
Tom Lendacky7ed9abf2020-12-10 11:09:54 -060013666
Paolo Bonziniea724ea2021-10-12 10:51:55 -040013667 /* Emulation done by the kernel. */
Paolo Bonzinidb209362021-10-22 08:47:56 -040013668 advance_sev_es_emulated_pio(vcpu, count, size);
Paolo Bonzini95e16b42021-10-12 11:33:03 -040013669 if (!vcpu->arch.sev_pio_count)
13670 return 1;
Tom Lendacky7ed9abf2020-12-10 11:09:54 -060013671 }
13672
Paolo Bonziniea724ea2021-10-12 10:51:55 -040013673 vcpu->arch.complete_userspace_io = complete_sev_es_emulated_ins;
Tom Lendacky7ed9abf2020-12-10 11:09:54 -060013674 return 0;
13675}
13676
13677int kvm_sev_es_string_io(struct kvm_vcpu *vcpu, unsigned int size,
13678 unsigned int port, void *data, unsigned int count,
13679 int in)
13680{
Paolo Bonziniea724ea2021-10-12 10:51:55 -040013681 vcpu->arch.sev_pio_data = data;
Paolo Bonzini95e16b42021-10-12 11:33:03 -040013682 vcpu->arch.sev_pio_count = count;
13683 return in ? kvm_sev_es_ins(vcpu, size, port)
13684 : kvm_sev_es_outs(vcpu, size, port);
Tom Lendacky7ed9abf2020-12-10 11:09:54 -060013685}
13686EXPORT_SYMBOL_GPL(kvm_sev_es_string_io);
13687
Lorenzo Bresciad95df952020-12-23 14:45:07 +000013688EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_entry);
Carsten Otte043405e2007-10-10 17:16:19 +020013689EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_exit);
13690EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_fast_mmio);
13691EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_inj_virq);
13692EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_page_fault);
13693EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_msr);
13694EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_cr);
Mingwei Zhang89e54ec2022-08-25 22:57:53 +000013695EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_vmenter);
Carsten Otte043405e2007-10-10 17:16:19 +020013696EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_vmexit);
13697EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_vmexit_inject);
13698EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_intr_vmexit);
Sean Christopherson5497b952019-07-11 08:58:29 -070013699EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_vmenter_failed);
Carsten Otte043405e2007-10-10 17:16:19 +020013700EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_invlpga);
13701EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_skinit);
13702EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_intercepts);
13703EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_write_tsc_offset);
Peter Xu4f75bcc2019-09-06 10:17:22 +080013704EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_ple_window_update);
Carsten Otte043405e2007-10-10 17:16:19 +020013705EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_pml_full);
13706EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_pi_irte_update);
13707EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_avic_unaccelerated_access);
13708EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_avic_incomplete_ipi);
Suravee Suthikulpanitab56f8e2020-03-12 05:39:28 -050013709EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_avic_ga_log);
Suravee Suthikulpanit9f084f72022-04-20 10:49:54 -050013710EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_avic_kick_vcpu_slowpath);
Suravee Suthikulpanit39b6b8c2022-05-19 05:27:08 -050013711EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_avic_doorbell);
Maxim Levitsky8e819d72021-12-09 13:54:36 +020013712EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_apicv_accept_irq);
Tom Lendackyd523ab6b2020-12-10 11:09:48 -060013713EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_vmgexit_enter);
13714EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_vmgexit_exit);
Tom Lendacky59e38b52020-12-10 11:09:52 -060013715EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_vmgexit_msr_protocol_enter);
13716EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_vmgexit_msr_protocol_exit);
Sean Christopherson1d0e8482022-03-31 22:13:59 +000013717
13718static int __init kvm_x86_init(void)
13719{
13720 kvm_mmu_x86_module_init();
13721 return 0;
13722}
13723module_init(kvm_x86_init);
13724
13725static void __exit kvm_x86_exit(void)
13726{
13727 /*
13728 * If module_init() is implemented, module_exit() must also be
13729 * implemented to allow module unload.
13730 */
13731}
13732module_exit(kvm_x86_exit);