blob: 11cb6651097e68982554c9c585c1ca729ff79aa2 [file] [log] [blame]
#ifndef X86_VMX_H
#define X86_VMX_H
#include "libcflat.h"
#include "processor.h"
#include "bitops.h"
#include "asm/page.h"
#include "asm/io.h"
void __abort_test(void);
#define __TEST_ASSERT(cond) \
do { \
if (!(cond)) { \
report_fail("%s:%d: Assertion failed: %s", \
__FILE__, __LINE__, #cond); \
dump_stack(); \
__abort_test(); \
} \
} while (0)
#define TEST_ASSERT(cond) \
do { \
__TEST_ASSERT(cond); \
report_passed(); \
} while (0)
#define TEST_ASSERT_MSG(cond, fmt, args...) \
do { \
if (!(cond)) { \
report_fail("%s:%d: Assertion failed: %s\n" fmt,\
__FILE__, __LINE__, #cond, ##args); \
dump_stack(); \
__abort_test(); \
} \
report_passed(); \
} while (0)
#define __TEST_EQ(a, b, a_str, b_str, assertion, fmt, args...) \
do { \
typeof(a) _a = a; \
typeof(b) _b = b; \
if (_a != _b) { \
char _bin_a[BINSTR_SZ]; \
char _bin_b[BINSTR_SZ]; \
binstr(_a, _bin_a); \
binstr(_b, _bin_b); \
report_fail("%s:%d: %s failed: (%s) == (%s)\n" \
"\tLHS: %#018lx - %s - %lu\n" \
"\tRHS: %#018lx - %s - %lu%s" fmt, \
__FILE__, __LINE__, \
assertion ? "Assertion" : "Expectation", a_str, b_str, \
(unsigned long) _a, _bin_a, (unsigned long) _a, \
(unsigned long) _b, _bin_b, (unsigned long) _b, \
fmt[0] == '\0' ? "" : "\n", ## args); \
dump_stack(); \
if (assertion) \
__abort_test(); \
} \
report_passed(); \
} while (0)
#define TEST_ASSERT_EQ(a, b) __TEST_EQ(a, b, #a, #b, 1, "")
#define TEST_ASSERT_EQ_MSG(a, b, fmt, args...) \
__TEST_EQ(a, b, #a, #b, 1, fmt, ## args)
#define TEST_EXPECT_EQ(a, b) __TEST_EQ(a, b, #a, #b, 0, "")
#define TEST_EXPECT_EQ_MSG(a, b, fmt, args...) \
__TEST_EQ(a, b, #a, #b, 0, fmt, ## args)
struct vmcs_hdr {
u32 revision_id:31;
u32 shadow_vmcs:1;
};
struct vmcs {
struct vmcs_hdr hdr;
u32 abort; /* VMX-abort indicator */
/* VMCS data */
char data[0];
};
struct invvpid_operand {
u64 vpid;
u64 gla;
};
struct regs {
u64 rax;
u64 rcx;
u64 rdx;
u64 rbx;
u64 cr2;
u64 rbp;
u64 rsi;
u64 rdi;
u64 r8;
u64 r9;
u64 r10;
u64 r11;
u64 r12;
u64 r13;
u64 r14;
u64 r15;
u64 rflags;
};
union exit_reason {
struct {
u32 basic : 16;
u32 reserved16 : 1;
u32 reserved17 : 1;
u32 reserved18 : 1;
u32 reserved19 : 1;
u32 reserved20 : 1;
u32 reserved21 : 1;
u32 reserved22 : 1;
u32 reserved23 : 1;
u32 reserved24 : 1;
u32 reserved25 : 1;
u32 reserved26 : 1;
u32 enclave_mode : 1;
u32 smi_pending_mtf : 1;
u32 smi_from_vmx_root : 1;
u32 reserved30 : 1;
u32 failed_vmentry : 1;
};
u32 full;
};
struct vmentry_result {
/* Instruction mnemonic (for convenience). */
const char *instr;
/* Did the test attempt vmlaunch or vmresume? */
bool vmlaunch;
/* Did the instruction VM-Fail? */
bool vm_fail;
/* Did the VM-Entry fully enter the guest? */
bool entered;
/* VM-Exit reason, valid iff !vm_fail */
union exit_reason exit_reason;
/* Contents of [re]flags after failed entry. */
unsigned long flags;
};
struct vmx_test {
const char *name;
int (*init)(struct vmcs *vmcs);
void (*guest_main)(void);
int (*exit_handler)(union exit_reason exit_reason);
void (*syscall_handler)(u64 syscall_no);
struct regs guest_regs;
int (*entry_failure_handler)(struct vmentry_result *result);
struct vmcs *vmcs;
int exits;
/* Alternative test interface. */
void (*v2)(void);
};
union vmx_basic {
u64 val;
struct {
u32 revision;
u32 size:13,
reserved1: 3,
width:1,
dual:1,
type:4,
insouts:1,
ctrl:1,
reserved2:8;
};
};
union vmx_ctrl_msr {
u64 val;
struct {
u32 set, clr;
};
};
union vmx_misc {
u64 val;
struct {
u32 pt_bit:5,
stores_lma:1,
act_hlt:1,
act_shutdown:1,
act_wfsipi:1,
:5,
vmx_pt:1,
smm_smbase:1,
cr3_targets:9,
msr_list_size:3,
smm_mon_ctl:1,
vmwrite_any:1,
inject_len0:1,
:1;
u32 mseg_revision;
};
};
union vmx_ept_vpid {
u64 val;
struct {
u32:16,
super:2,
: 2,
invept:1,
: 11;
u32 invvpid:1;
};
};
enum Encoding {
/* 16-Bit Control Fields */
VPID = 0x0000ul,
/* Posted-interrupt notification vector */
PINV = 0x0002ul,
/* EPTP index */
EPTP_IDX = 0x0004ul,
/* 16-Bit Guest State Fields */
GUEST_SEL_ES = 0x0800ul,
GUEST_SEL_CS = 0x0802ul,
GUEST_SEL_SS = 0x0804ul,
GUEST_SEL_DS = 0x0806ul,
GUEST_SEL_FS = 0x0808ul,
GUEST_SEL_GS = 0x080aul,
GUEST_SEL_LDTR = 0x080cul,
GUEST_SEL_TR = 0x080eul,
GUEST_INT_STATUS = 0x0810ul,
GUEST_PML_INDEX = 0x0812ul,
/* 16-Bit Host State Fields */
HOST_SEL_ES = 0x0c00ul,
HOST_SEL_CS = 0x0c02ul,
HOST_SEL_SS = 0x0c04ul,
HOST_SEL_DS = 0x0c06ul,
HOST_SEL_FS = 0x0c08ul,
HOST_SEL_GS = 0x0c0aul,
HOST_SEL_TR = 0x0c0cul,
/* 64-Bit Control Fields */
IO_BITMAP_A = 0x2000ul,
IO_BITMAP_B = 0x2002ul,
MSR_BITMAP = 0x2004ul,
EXIT_MSR_ST_ADDR = 0x2006ul,
EXIT_MSR_LD_ADDR = 0x2008ul,
ENTER_MSR_LD_ADDR = 0x200aul,
VMCS_EXEC_PTR = 0x200cul,
TSC_OFFSET = 0x2010ul,
TSC_OFFSET_HI = 0x2011ul,
APIC_VIRT_ADDR = 0x2012ul,
APIC_ACCS_ADDR = 0x2014ul,
POSTED_INTR_DESC_ADDR = 0x2016ul,
EPTP = 0x201aul,
EPTP_HI = 0x201bul,
VMREAD_BITMAP = 0x2026ul,
VMREAD_BITMAP_HI = 0x2027ul,
VMWRITE_BITMAP = 0x2028ul,
VMWRITE_BITMAP_HI = 0x2029ul,
EOI_EXIT_BITMAP0 = 0x201cul,
EOI_EXIT_BITMAP1 = 0x201eul,
EOI_EXIT_BITMAP2 = 0x2020ul,
EOI_EXIT_BITMAP3 = 0x2022ul,
PMLADDR = 0x200eul,
PMLADDR_HI = 0x200ful,
/* 64-Bit Readonly Data Field */
INFO_PHYS_ADDR = 0x2400ul,
/* 64-Bit Guest State */
VMCS_LINK_PTR = 0x2800ul,
VMCS_LINK_PTR_HI = 0x2801ul,
GUEST_DEBUGCTL = 0x2802ul,
GUEST_DEBUGCTL_HI = 0x2803ul,
GUEST_EFER = 0x2806ul,
GUEST_PAT = 0x2804ul,
GUEST_PERF_GLOBAL_CTRL = 0x2808ul,
GUEST_PDPTE = 0x280aul,
GUEST_BNDCFGS = 0x2812ul,
/* 64-Bit Host State */
HOST_PAT = 0x2c00ul,
HOST_EFER = 0x2c02ul,
HOST_PERF_GLOBAL_CTRL = 0x2c04ul,
/* 32-Bit Control Fields */
PIN_CONTROLS = 0x4000ul,
CPU_EXEC_CTRL0 = 0x4002ul,
EXC_BITMAP = 0x4004ul,
PF_ERROR_MASK = 0x4006ul,
PF_ERROR_MATCH = 0x4008ul,
CR3_TARGET_COUNT = 0x400aul,
EXI_CONTROLS = 0x400cul,
EXI_MSR_ST_CNT = 0x400eul,
EXI_MSR_LD_CNT = 0x4010ul,
ENT_CONTROLS = 0x4012ul,
ENT_MSR_LD_CNT = 0x4014ul,
ENT_INTR_INFO = 0x4016ul,
ENT_INTR_ERROR = 0x4018ul,
ENT_INST_LEN = 0x401aul,
TPR_THRESHOLD = 0x401cul,
CPU_EXEC_CTRL1 = 0x401eul,
/* 32-Bit R/O Data Fields */
VMX_INST_ERROR = 0x4400ul,
EXI_REASON = 0x4402ul,
EXI_INTR_INFO = 0x4404ul,
EXI_INTR_ERROR = 0x4406ul,
IDT_VECT_INFO = 0x4408ul,
IDT_VECT_ERROR = 0x440aul,
EXI_INST_LEN = 0x440cul,
EXI_INST_INFO = 0x440eul,
/* 32-Bit Guest State Fields */
GUEST_LIMIT_ES = 0x4800ul,
GUEST_LIMIT_CS = 0x4802ul,
GUEST_LIMIT_SS = 0x4804ul,
GUEST_LIMIT_DS = 0x4806ul,
GUEST_LIMIT_FS = 0x4808ul,
GUEST_LIMIT_GS = 0x480aul,
GUEST_LIMIT_LDTR = 0x480cul,
GUEST_LIMIT_TR = 0x480eul,
GUEST_LIMIT_GDTR = 0x4810ul,
GUEST_LIMIT_IDTR = 0x4812ul,
GUEST_AR_ES = 0x4814ul,
GUEST_AR_CS = 0x4816ul,
GUEST_AR_SS = 0x4818ul,
GUEST_AR_DS = 0x481aul,
GUEST_AR_FS = 0x481cul,
GUEST_AR_GS = 0x481eul,
GUEST_AR_LDTR = 0x4820ul,
GUEST_AR_TR = 0x4822ul,
GUEST_INTR_STATE = 0x4824ul,
GUEST_ACTV_STATE = 0x4826ul,
GUEST_SMBASE = 0x4828ul,
GUEST_SYSENTER_CS = 0x482aul,
PREEMPT_TIMER_VALUE = 0x482eul,
/* 32-Bit Host State Fields */
HOST_SYSENTER_CS = 0x4c00ul,
/* Natural-Width Control Fields */
CR0_MASK = 0x6000ul,
CR4_MASK = 0x6002ul,
CR0_READ_SHADOW = 0x6004ul,
CR4_READ_SHADOW = 0x6006ul,
CR3_TARGET_0 = 0x6008ul,
CR3_TARGET_1 = 0x600aul,
CR3_TARGET_2 = 0x600cul,
CR3_TARGET_3 = 0x600eul,
/* Natural-Width R/O Data Fields */
EXI_QUALIFICATION = 0x6400ul,
IO_RCX = 0x6402ul,
IO_RSI = 0x6404ul,
IO_RDI = 0x6406ul,
IO_RIP = 0x6408ul,
GUEST_LINEAR_ADDRESS = 0x640aul,
/* Natural-Width Guest State Fields */
GUEST_CR0 = 0x6800ul,
GUEST_CR3 = 0x6802ul,
GUEST_CR4 = 0x6804ul,
GUEST_BASE_ES = 0x6806ul,
GUEST_BASE_CS = 0x6808ul,
GUEST_BASE_SS = 0x680aul,
GUEST_BASE_DS = 0x680cul,
GUEST_BASE_FS = 0x680eul,
GUEST_BASE_GS = 0x6810ul,
GUEST_BASE_LDTR = 0x6812ul,
GUEST_BASE_TR = 0x6814ul,
GUEST_BASE_GDTR = 0x6816ul,
GUEST_BASE_IDTR = 0x6818ul,
GUEST_DR7 = 0x681aul,
GUEST_RSP = 0x681cul,
GUEST_RIP = 0x681eul,
GUEST_RFLAGS = 0x6820ul,
GUEST_PENDING_DEBUG = 0x6822ul,
GUEST_SYSENTER_ESP = 0x6824ul,
GUEST_SYSENTER_EIP = 0x6826ul,
/* Natural-Width Host State Fields */
HOST_CR0 = 0x6c00ul,
HOST_CR3 = 0x6c02ul,
HOST_CR4 = 0x6c04ul,
HOST_BASE_FS = 0x6c06ul,
HOST_BASE_GS = 0x6c08ul,
HOST_BASE_TR = 0x6c0aul,
HOST_BASE_GDTR = 0x6c0cul,
HOST_BASE_IDTR = 0x6c0eul,
HOST_SYSENTER_ESP = 0x6c10ul,
HOST_SYSENTER_EIP = 0x6c12ul,
HOST_RSP = 0x6c14ul,
HOST_RIP = 0x6c16ul
};
#define VMX_ENTRY_FAILURE (1ul << 31)
#define VMX_ENTRY_FLAGS (X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF | \
X86_EFLAGS_ZF | X86_EFLAGS_SF | X86_EFLAGS_OF)
enum Reason {
VMX_EXC_NMI = 0,
VMX_EXTINT = 1,
VMX_TRIPLE_FAULT = 2,
VMX_INIT = 3,
VMX_SIPI = 4,
VMX_SMI_IO = 5,
VMX_SMI_OTHER = 6,
VMX_INTR_WINDOW = 7,
VMX_NMI_WINDOW = 8,
VMX_TASK_SWITCH = 9,
VMX_CPUID = 10,
VMX_GETSEC = 11,
VMX_HLT = 12,
VMX_INVD = 13,
VMX_INVLPG = 14,
VMX_RDPMC = 15,
VMX_RDTSC = 16,
VMX_RSM = 17,
VMX_VMCALL = 18,
VMX_VMCLEAR = 19,
VMX_VMLAUNCH = 20,
VMX_VMPTRLD = 21,
VMX_VMPTRST = 22,
VMX_VMREAD = 23,
VMX_VMRESUME = 24,
VMX_VMWRITE = 25,
VMX_VMXOFF = 26,
VMX_VMXON = 27,
VMX_CR = 28,
VMX_DR = 29,
VMX_IO = 30,
VMX_RDMSR = 31,
VMX_WRMSR = 32,
VMX_FAIL_STATE = 33,
VMX_FAIL_MSR = 34,
VMX_MWAIT = 36,
VMX_MTF = 37,
VMX_MONITOR = 39,
VMX_PAUSE = 40,
VMX_FAIL_MCHECK = 41,
VMX_TPR_THRESHOLD = 43,
VMX_APIC_ACCESS = 44,
VMX_EOI_INDUCED = 45,
VMX_GDTR_IDTR = 46,
VMX_LDTR_TR = 47,
VMX_EPT_VIOLATION = 48,
VMX_EPT_MISCONFIG = 49,
VMX_INVEPT = 50,
VMX_PREEMPT = 52,
VMX_INVVPID = 53,
VMX_WBINVD = 54,
VMX_XSETBV = 55,
VMX_APIC_WRITE = 56,
VMX_RDRAND = 57,
VMX_INVPCID = 58,
VMX_VMFUNC = 59,
VMX_RDSEED = 61,
VMX_PML_FULL = 62,
VMX_XSAVES = 63,
VMX_XRSTORS = 64,
};
enum Ctrl_exi {
EXI_SAVE_DBGCTLS = 1UL << 2,
EXI_HOST_64 = 1UL << 9,
EXI_LOAD_PERF = 1UL << 12,
EXI_INTA = 1UL << 15,
EXI_SAVE_PAT = 1UL << 18,
EXI_LOAD_PAT = 1UL << 19,
EXI_SAVE_EFER = 1UL << 20,
EXI_LOAD_EFER = 1UL << 21,
EXI_SAVE_PREEMPT = 1UL << 22,
};
enum Ctrl_ent {
ENT_LOAD_DBGCTLS = 1UL << 2,
ENT_GUEST_64 = 1UL << 9,
ENT_LOAD_PERF = 1UL << 13,
ENT_LOAD_PAT = 1UL << 14,
ENT_LOAD_EFER = 1UL << 15,
ENT_LOAD_BNDCFGS = 1UL << 16
};
enum Ctrl_pin {
PIN_EXTINT = 1ul << 0,
PIN_NMI = 1ul << 3,
PIN_VIRT_NMI = 1ul << 5,
PIN_PREEMPT = 1ul << 6,
PIN_POST_INTR = 1ul << 7,
};
enum Ctrl0 {
CPU_INTR_WINDOW = 1ul << 2,
CPU_USE_TSC_OFFSET = 1ul << 3,
CPU_HLT = 1ul << 7,
CPU_INVLPG = 1ul << 9,
CPU_MWAIT = 1ul << 10,
CPU_RDPMC = 1ul << 11,
CPU_RDTSC = 1ul << 12,
CPU_CR3_LOAD = 1ul << 15,
CPU_CR3_STORE = 1ul << 16,
CPU_CR8_LOAD = 1ul << 19,
CPU_CR8_STORE = 1ul << 20,
CPU_TPR_SHADOW = 1ul << 21,
CPU_NMI_WINDOW = 1ul << 22,
CPU_IO = 1ul << 24,
CPU_IO_BITMAP = 1ul << 25,
CPU_MTF = 1ul << 27,
CPU_MSR_BITMAP = 1ul << 28,
CPU_MONITOR = 1ul << 29,
CPU_PAUSE = 1ul << 30,
CPU_SECONDARY = 1ul << 31,
};
enum Ctrl1 {
CPU_VIRT_APIC_ACCESSES = 1ul << 0,
CPU_EPT = 1ul << 1,
CPU_DESC_TABLE = 1ul << 2,
CPU_RDTSCP = 1ul << 3,
CPU_VIRT_X2APIC = 1ul << 4,
CPU_VPID = 1ul << 5,
CPU_WBINVD = 1ul << 6,
CPU_URG = 1ul << 7,
CPU_APIC_REG_VIRT = 1ul << 8,
CPU_VINTD = 1ul << 9,
CPU_RDRAND = 1ul << 11,
CPU_SHADOW_VMCS = 1ul << 14,
CPU_RDSEED = 1ul << 16,
CPU_PML = 1ul << 17,
CPU_USE_TSC_SCALING = 1ul << 25,
};
enum Intr_type {
VMX_INTR_TYPE_EXT_INTR = 0,
VMX_INTR_TYPE_NMI_INTR = 2,
VMX_INTR_TYPE_HARD_EXCEPTION = 3,
VMX_INTR_TYPE_SOFT_INTR = 4,
VMX_INTR_TYPE_SOFT_EXCEPTION = 6,
};
/*
* Interruption-information format
*/
#define INTR_INFO_VECTOR_MASK 0xff /* 7:0 */
#define INTR_INFO_INTR_TYPE_MASK 0x700 /* 10:8 */
#define INTR_INFO_DELIVER_CODE_MASK 0x800 /* 11 */
#define INTR_INFO_UNBLOCK_NMI_MASK 0x1000 /* 12 */
#define INTR_INFO_VALID_MASK 0x80000000 /* 31 */
#define INTR_INFO_INTR_TYPE_SHIFT 8
#define INTR_TYPE_EXT_INTR (0 << 8) /* external interrupt */
#define INTR_TYPE_RESERVED (1 << 8) /* reserved */
#define INTR_TYPE_NMI_INTR (2 << 8) /* NMI */
#define INTR_TYPE_HARD_EXCEPTION (3 << 8) /* processor exception */
#define INTR_TYPE_SOFT_INTR (4 << 8) /* software interrupt */
#define INTR_TYPE_PRIV_SW_EXCEPTION (5 << 8) /* priv. software exception */
#define INTR_TYPE_SOFT_EXCEPTION (6 << 8) /* software exception */
#define INTR_TYPE_OTHER_EVENT (7 << 8) /* other event */
/*
* Guest interruptibility state
*/
#define GUEST_INTR_STATE_STI (1 << 0)
#define GUEST_INTR_STATE_MOVSS (1 << 1)
#define GUEST_INTR_STATE_SMI (1 << 2)
#define GUEST_INTR_STATE_NMI (1 << 3)
#define GUEST_INTR_STATE_ENCLAVE (1 << 4)
/*
* VM-instruction error numbers
*/
enum vm_instruction_error_number {
VMXERR_VMCALL_IN_VMX_ROOT_OPERATION = 1,
VMXERR_VMCLEAR_INVALID_ADDRESS = 2,
VMXERR_VMCLEAR_VMXON_POINTER = 3,
VMXERR_VMLAUNCH_NONCLEAR_VMCS = 4,
VMXERR_VMRESUME_NONLAUNCHED_VMCS = 5,
VMXERR_VMRESUME_AFTER_VMXOFF = 6,
VMXERR_ENTRY_INVALID_CONTROL_FIELD = 7,
VMXERR_ENTRY_INVALID_HOST_STATE_FIELD = 8,
VMXERR_VMPTRLD_INVALID_ADDRESS = 9,
VMXERR_VMPTRLD_VMXON_POINTER = 10,
VMXERR_VMPTRLD_INCORRECT_VMCS_REVISION_ID = 11,
VMXERR_UNSUPPORTED_VMCS_COMPONENT = 12,
VMXERR_VMWRITE_READ_ONLY_VMCS_COMPONENT = 13,
VMXERR_VMXON_IN_VMX_ROOT_OPERATION = 15,
VMXERR_ENTRY_INVALID_EXECUTIVE_VMCS_POINTER = 16,
VMXERR_ENTRY_NONLAUNCHED_EXECUTIVE_VMCS = 17,
VMXERR_ENTRY_EXECUTIVE_VMCS_POINTER_NOT_VMXON_POINTER = 18,
VMXERR_VMCALL_NONCLEAR_VMCS = 19,
VMXERR_VMCALL_INVALID_VM_EXIT_CONTROL_FIELDS = 20,
VMXERR_VMCALL_INCORRECT_MSEG_REVISION_ID = 22,
VMXERR_VMXOFF_UNDER_DUAL_MONITOR_TREATMENT_OF_SMIS_AND_SMM = 23,
VMXERR_VMCALL_INVALID_SMM_MONITOR_FEATURES = 24,
VMXERR_ENTRY_INVALID_VM_EXECUTION_CONTROL_FIELDS_IN_EXECUTIVE_VMCS = 25,
VMXERR_ENTRY_EVENTS_BLOCKED_BY_MOV_SS = 26,
VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID = 28,
};
enum vm_entry_failure_code {
ENTRY_FAIL_DEFAULT = 0,
ENTRY_FAIL_PDPTE = 2,
ENTRY_FAIL_NMI = 3,
ENTRY_FAIL_VMCS_LINK_PTR = 4,
};
#define SAVE_GPR \
"xchg %rax, regs\n\t" \
"xchg %rcx, regs+0x8\n\t" \
"xchg %rdx, regs+0x10\n\t" \
"xchg %rbx, regs+0x18\n\t" \
"xchg %rbp, regs+0x28\n\t" \
"xchg %rsi, regs+0x30\n\t" \
"xchg %rdi, regs+0x38\n\t" \
"xchg %r8, regs+0x40\n\t" \
"xchg %r9, regs+0x48\n\t" \
"xchg %r10, regs+0x50\n\t" \
"xchg %r11, regs+0x58\n\t" \
"xchg %r12, regs+0x60\n\t" \
"xchg %r13, regs+0x68\n\t" \
"xchg %r14, regs+0x70\n\t" \
"xchg %r15, regs+0x78\n\t"
#define LOAD_GPR SAVE_GPR
#define SAVE_GPR_C \
"xchg %%rax, regs\n\t" \
"xchg %%rcx, regs+0x8\n\t" \
"xchg %%rdx, regs+0x10\n\t" \
"xchg %%rbx, regs+0x18\n\t" \
"xchg %%rbp, regs+0x28\n\t" \
"xchg %%rsi, regs+0x30\n\t" \
"xchg %%rdi, regs+0x38\n\t" \
"xchg %%r8, regs+0x40\n\t" \
"xchg %%r9, regs+0x48\n\t" \
"xchg %%r10, regs+0x50\n\t" \
"xchg %%r11, regs+0x58\n\t" \
"xchg %%r12, regs+0x60\n\t" \
"xchg %%r13, regs+0x68\n\t" \
"xchg %%r14, regs+0x70\n\t" \
"xchg %%r15, regs+0x78\n\t"
#define LOAD_GPR_C SAVE_GPR_C
#define VMX_IO_SIZE_MASK 0x7
#define _VMX_IO_BYTE 0
#define _VMX_IO_WORD 1
#define _VMX_IO_LONG 3
#define VMX_IO_DIRECTION_MASK (1ul << 3)
#define VMX_IO_IN (1ul << 3)
#define VMX_IO_OUT 0
#define VMX_IO_STRING (1ul << 4)
#define VMX_IO_REP (1ul << 5)
#define VMX_IO_OPRAND_IMM (1ul << 6)
#define VMX_IO_PORT_MASK 0xFFFF0000
#define VMX_IO_PORT_SHIFT 16
#define VMX_TEST_START 0
#define VMX_TEST_VMEXIT 1
#define VMX_TEST_EXIT 2
#define VMX_TEST_RESUME 3
#define VMX_TEST_VMABORT 4
#define VMX_TEST_VMSKIP 5
#define HYPERCALL_BIT (1ul << 12)
#define HYPERCALL_MASK 0xFFF
#define HYPERCALL_VMEXIT 0x1
#define HYPERCALL_VMABORT 0x2
#define HYPERCALL_VMSKIP 0x3
#define EPTP_PG_WALK_LEN_SHIFT 3ul
#define EPTP_PG_WALK_LEN_MASK 0x38ul
#define EPTP_RESERV_BITS_MASK 0x1ful
#define EPTP_RESERV_BITS_SHIFT 0x7ul
#define EPTP_AD_FLAG (1ul << 6)
#define EPT_MEM_TYPE_UC 0ul
#define EPT_MEM_TYPE_WC 1ul
#define EPT_MEM_TYPE_WT 4ul
#define EPT_MEM_TYPE_WP 5ul
#define EPT_MEM_TYPE_WB 6ul
#define EPT_RA 1ul
#define EPT_WA 2ul
#define EPT_EA 4ul
#define EPT_PRESENT (EPT_RA | EPT_WA | EPT_EA)
#define EPT_ACCESS_FLAG (1ul << 8)
#define EPT_DIRTY_FLAG (1ul << 9)
#define EPT_LARGE_PAGE (1ul << 7)
#define EPT_MEM_TYPE_SHIFT 3ul
#define EPT_MEM_TYPE_MASK 0x7ul
#define EPT_IGNORE_PAT (1ul << 6)
#define EPT_SUPPRESS_VE (1ull << 63)
#define EPT_CAP_EXEC_ONLY (1ull << 0)
#define EPT_CAP_PWL4 (1ull << 6)
#define EPT_CAP_PWL5 (1ull << 7)
#define EPT_CAP_UC (1ull << 8)
#define EPT_CAP_WB (1ull << 14)
#define EPT_CAP_2M_PAGE (1ull << 16)
#define EPT_CAP_1G_PAGE (1ull << 17)
#define EPT_CAP_INVEPT (1ull << 20)
#define EPT_CAP_AD_FLAG (1ull << 21)
#define EPT_CAP_ADV_EPT_INFO (1ull << 22)
#define EPT_CAP_INVEPT_SINGLE (1ull << 25)
#define EPT_CAP_INVEPT_ALL (1ull << 26)
#define VPID_CAP_INVVPID (1ull << 32)
#define VPID_CAP_INVVPID_ADDR (1ull << 40)
#define VPID_CAP_INVVPID_CXTGLB (1ull << 41)
#define VPID_CAP_INVVPID_ALL (1ull << 42)
#define VPID_CAP_INVVPID_CXTLOC (1ull << 43)
#define PAGE_SIZE_2M (512 * PAGE_SIZE)
#define PAGE_SIZE_1G (512 * PAGE_SIZE_2M)
#define EPT_PAGE_LEVEL 4
#define EPT_PGDIR_WIDTH 9
#define EPT_PGDIR_MASK 511
#define EPT_PGDIR_ENTRIES (1 << EPT_PGDIR_WIDTH)
#define EPT_LEVEL_SHIFT(level) (((level)-1) * EPT_PGDIR_WIDTH + 12)
#define EPT_ADDR_MASK GENMASK_ULL(51, 12)
#define PAGE_MASK_2M (~(PAGE_SIZE_2M-1))
#define EPT_VLT_RD (1ull << 0)
#define EPT_VLT_WR (1ull << 1)
#define EPT_VLT_FETCH (1ull << 2)
#define EPT_VLT_PERM_RD (1ull << 3)
#define EPT_VLT_PERM_WR (1ull << 4)
#define EPT_VLT_PERM_EX (1ull << 5)
#define EPT_VLT_PERM_USER_EX (1ull << 6)
#define EPT_VLT_PERMS (EPT_VLT_PERM_RD | EPT_VLT_PERM_WR | \
EPT_VLT_PERM_EX)
#define EPT_VLT_LADDR_VLD (1ull << 7)
#define EPT_VLT_PADDR (1ull << 8)
#define EPT_VLT_GUEST_USER (1ull << 9)
#define EPT_VLT_GUEST_RW (1ull << 10)
#define EPT_VLT_GUEST_EX (1ull << 11)
#define EPT_VLT_GUEST_MASK (EPT_VLT_GUEST_USER | EPT_VLT_GUEST_RW | \
EPT_VLT_GUEST_EX)
#define MAGIC_VAL_1 0x12345678ul
#define MAGIC_VAL_2 0x87654321ul
#define MAGIC_VAL_3 0xfffffffful
#define MAGIC_VAL_4 0xdeadbeeful
#define INVEPT_SINGLE 1
#define INVEPT_GLOBAL 2
#define INVVPID_ADDR 0
#define INVVPID_CONTEXT_GLOBAL 1
#define INVVPID_ALL 2
#define INVVPID_CONTEXT_LOCAL 3
#define ACTV_ACTIVE 0
#define ACTV_HLT 1
#define ACTV_SHUTDOWN 2
#define ACTV_WAIT_SIPI 3
/*
* VMCS field encoding:
* Bit 0: High-access
* Bits 1-9: Index
* Bits 10-12: Type
* Bits 13-15: Width
* Bits 15-64: Reserved
*/
#define VMCS_FIELD_HIGH_SHIFT (0)
#define VMCS_FIELD_INDEX_SHIFT (1)
#define VMCS_FIELD_INDEX_MASK GENMASK(9, 1)
#define VMCS_FIELD_TYPE_SHIFT (10)
#define VMCS_FIELD_WIDTH_SHIFT (13)
#define VMCS_FIELD_RESERVED_SHIFT (15)
#define VMCS_FIELD_BIT_SIZE (BITS_PER_LONG)
extern struct regs regs;
extern union vmx_basic basic;
extern union vmx_ctrl_msr ctrl_pin_rev;
extern union vmx_ctrl_msr ctrl_cpu_rev[2];
extern union vmx_ctrl_msr ctrl_exit_rev;
extern union vmx_ctrl_msr ctrl_enter_rev;
extern union vmx_ept_vpid ept_vpid;
static inline bool ept_2m_supported(void)
{
return ept_vpid.val & EPT_CAP_2M_PAGE;
}
static inline bool ept_1g_supported(void)
{
return ept_vpid.val & EPT_CAP_1G_PAGE;
}
static inline bool ept_huge_pages_supported(int level)
{
if (level == 2)
return ept_2m_supported();
else if (level == 3)
return ept_1g_supported();
else
return false;
}
static inline bool ept_execute_only_supported(void)
{
return ept_vpid.val & EPT_CAP_EXEC_ONLY;
}
static inline bool ept_ad_bits_supported(void)
{
return ept_vpid.val & EPT_CAP_AD_FLAG;
}
static inline bool is_4_level_ept_supported(void)
{
return ept_vpid.val & EPT_CAP_PWL4;
}
static inline bool is_5_level_ept_supported(void)
{
return ept_vpid.val & EPT_CAP_PWL5;
}
static inline bool is_ept_memtype_supported(int type)
{
if (type == EPT_MEM_TYPE_UC)
return ept_vpid.val & EPT_CAP_UC;
if (type == EPT_MEM_TYPE_WB)
return ept_vpid.val & EPT_CAP_WB;
return false;
}
static inline bool is_invept_type_supported(u64 type)
{
if (type < INVEPT_SINGLE || type > INVEPT_GLOBAL)
return false;
return ept_vpid.val & (EPT_CAP_INVEPT_SINGLE << (type - INVEPT_SINGLE));
}
static inline bool is_vpid_supported(void)
{
return (ctrl_cpu_rev[0].clr & CPU_SECONDARY) &&
(ctrl_cpu_rev[1].clr & CPU_VPID);
}
static inline bool is_invvpid_supported(void)
{
return ept_vpid.val & VPID_CAP_INVVPID;
}
static inline bool is_invvpid_type_supported(unsigned long type)
{
if (type < INVVPID_ADDR || type > INVVPID_CONTEXT_LOCAL)
return false;
return ept_vpid.val & (VPID_CAP_INVVPID_ADDR << (type - INVVPID_ADDR));
}
extern u64 *bsp_vmxon_region;
extern bool launched;
void vmx_set_test_stage(u32 s);
u32 vmx_get_test_stage(void);
void vmx_inc_test_stage(void);
static int _vmx_on(u64 *vmxon_region)
{
bool ret;
u64 rflags = read_rflags() | X86_EFLAGS_CF | X86_EFLAGS_ZF;
asm volatile ("push %1; popf; vmxon %2; setbe %0\n\t"
: "=q" (ret) : "q" (rflags), "m" (vmxon_region) : "cc");
return ret;
}
static int vmx_on(void)
{
return _vmx_on(bsp_vmxon_region);
}
static int vmx_off(void)
{
bool ret;
u64 rflags = read_rflags() | X86_EFLAGS_CF | X86_EFLAGS_ZF;
asm volatile("push %1; popf; vmxoff; setbe %0\n\t"
: "=q"(ret) : "q" (rflags) : "cc");
return ret;
}
static inline int make_vmcs_current(struct vmcs *vmcs)
{
bool ret;
u64 rflags = read_rflags() | X86_EFLAGS_CF | X86_EFLAGS_ZF;
asm volatile ("push %1; popf; vmptrld %2; setbe %0"
: "=q" (ret) : "q" (rflags), "m" (vmcs) : "cc");
return ret;
}
static inline int vmcs_clear(struct vmcs *vmcs)
{
bool ret;
u64 rflags = read_rflags() | X86_EFLAGS_CF | X86_EFLAGS_ZF;
asm volatile ("push %1; popf; vmclear %2; setbe %0"
: "=q" (ret) : "q" (rflags), "m" (vmcs) : "cc");
return ret;
}
static inline u64 vmcs_read(enum Encoding enc)
{
u64 val;
asm volatile ("vmread %1, %0" : "=rm" (val) : "r" ((u64)enc) : "cc");
return val;
}
/*
* VMREAD with a guaranteed memory operand, used to test KVM's MMU by forcing
* KVM to translate GVA->GPA.
*/
static inline u64 vmcs_readm(enum Encoding enc)
{
u64 val;
asm volatile ("vmread %1, %0" : "=m" (val) : "r" ((u64)enc) : "cc");
return val;
}
static inline int vmcs_read_checking(enum Encoding enc, u64 *value)
{
u64 rflags = read_rflags() | X86_EFLAGS_CF | X86_EFLAGS_ZF;
u64 encoding = enc;
u64 val;
asm volatile ("shl $8, %%rax;"
"sahf;"
"vmread %[encoding], %[val];"
"lahf;"
"shr $8, %%rax"
: /* output */ [val]"=rm"(val), "+a"(rflags)
: /* input */ [encoding]"r"(encoding)
: /* clobber */ "cc");
*value = val;
return rflags & (X86_EFLAGS_CF | X86_EFLAGS_ZF);
}
static inline int vmcs_write(enum Encoding enc, u64 val)
{
bool ret;
asm volatile ("vmwrite %1, %2; setbe %0"
: "=q"(ret) : "rm" (val), "r" ((u64)enc) : "cc");
return ret;
}
static inline int vmcs_set_bits(enum Encoding enc, u64 val)
{
return vmcs_write(enc, vmcs_read(enc) | val);
}
static inline int vmcs_clear_bits(enum Encoding enc, u64 val)
{
return vmcs_write(enc, vmcs_read(enc) & ~val);
}
static inline int vmcs_save(struct vmcs **vmcs)
{
bool ret;
unsigned long pa;
u64 rflags = read_rflags() | X86_EFLAGS_CF | X86_EFLAGS_ZF;
asm volatile ("push %2; popf; vmptrst %1; setbe %0"
: "=q" (ret), "=m" (pa) : "r" (rflags) : "cc");
*vmcs = (pa == -1ull) ? NULL : phys_to_virt(pa);
return ret;
}
static inline int __invept(unsigned long type, u64 eptp)
{
bool failed = false;
u64 rflags = read_rflags() | X86_EFLAGS_CF | X86_EFLAGS_ZF;
struct {
u64 eptp, gpa;
} operand = {eptp, 0};
asm volatile("push %1; popf; invept %2, %3; setbe %0"
: "=q" (failed) : "r" (rflags), "m"(operand),"r"(type) : "cc");
return failed ? -1: 0;
}
static inline void invept(unsigned long type, u64 eptp)
{
__TEST_ASSERT(!__invept(type, eptp));
}
static inline int __invvpid(unsigned long type, u64 vpid, u64 gla)
{
bool failed = false;
u64 rflags = read_rflags() | X86_EFLAGS_CF | X86_EFLAGS_ZF;
struct invvpid_operand operand = {vpid, gla};
asm volatile("push %1; popf; invvpid %2, %3; setbe %0"
: "=q" (failed) : "r" (rflags), "m"(operand),"r"(type) : "cc");
return failed ? -1: 0;
}
static inline void invvpid(unsigned long type, u64 vpid, u64 gla)
{
__TEST_ASSERT(!__invvpid(type, vpid, gla));
}
void enable_vmx(void);
void init_vmx(u64 *vmxon_region);
int init_vmcs(struct vmcs **vmcs);
const char *exit_reason_description(u64 reason);
void print_vmexit_info(union exit_reason exit_reason);
void print_vmentry_failure_info(struct vmentry_result *result);
void install_ept_entry(unsigned long *pml4, int pte_level,
unsigned long guest_addr, unsigned long pte,
unsigned long *pt_page);
void install_1g_ept(unsigned long *pml4, unsigned long phys,
unsigned long guest_addr, u64 perm);
void install_2m_ept(unsigned long *pml4, unsigned long phys,
unsigned long guest_addr, u64 perm);
void install_ept(unsigned long *pml4, unsigned long phys,
unsigned long guest_addr, u64 perm);
void setup_ept_range(unsigned long *pml4, unsigned long start,
unsigned long len, int map_1g, int map_2m, u64 perm);
bool get_ept_pte(unsigned long *pml4, unsigned long guest_addr, int level,
unsigned long *pte);
void set_ept_pte(unsigned long *pml4, unsigned long guest_addr,
int level, u64 pte_val);
void check_ept_ad(unsigned long *pml4, u64 guest_cr3,
unsigned long guest_addr, int expected_gpa_ad,
int expected_pt_ad);
void clear_ept_ad(unsigned long *pml4, u64 guest_cr3,
unsigned long guest_addr);
#define ABORT_ON_EARLY_VMENTRY_FAIL 0x1
#define ABORT_ON_INVALID_GUEST_STATE 0x2
void __enter_guest(u8 abort_flag, struct vmentry_result *result);
void enter_guest(void);
void enter_guest_with_bad_controls(void);
void hypercall(u32 hypercall_no);
typedef void (*test_guest_func)(void);
typedef void (*test_teardown_func)(void *data);
void test_set_guest(test_guest_func func);
void test_override_guest(test_guest_func func);
void test_add_teardown(test_teardown_func func, void *data);
void test_skip(const char *msg);
void test_set_guest_finished(void);
#endif