blob: 00268290dcbd85a75ec2f17cc2caa5431c03208b [file] [log] [blame]
Thomas Gleixner20c8ccb2019-06-04 10:11:32 +02001// SPDX-License-Identifier: GPL-2.0-only
Avi Kivity6aa8b732006-12-10 02:21:36 -08002/*
3 * Kernel-based Virtual Machine driver for Linux
4 *
5 * This module enables machines with Intel VT-x extensions to run virtual
6 * machines without emulation or binary translation.
7 *
8 * Copyright (C) 2006 Qumranet, Inc.
Nicolas Kaiser9611c182010-10-06 14:23:22 +02009 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
Avi Kivity6aa8b732006-12-10 02:21:36 -080010 *
11 * Authors:
12 * Avi Kivity <avi@qumranet.com>
13 * Yaniv Kamay <yaniv@qumranet.com>
Avi Kivity6aa8b732006-12-10 02:21:36 -080014 */
15
Andre Przywaraaf669ac2015-03-26 14:39:29 +000016#include <kvm/iodev.h>
Avi Kivity6aa8b732006-12-10 02:21:36 -080017
Avi Kivityedf88412007-12-16 11:02:48 +020018#include <linux/kvm_host.h>
Avi Kivity6aa8b732006-12-10 02:21:36 -080019#include <linux/kvm.h>
20#include <linux/module.h>
21#include <linux/errno.h>
Avi Kivity6aa8b732006-12-10 02:21:36 -080022#include <linux/percpu.h>
Avi Kivity6aa8b732006-12-10 02:21:36 -080023#include <linux/mm.h>
24#include <linux/miscdevice.h>
25#include <linux/vmalloc.h>
Avi Kivity6aa8b732006-12-10 02:21:36 -080026#include <linux/reboot.h>
Avi Kivity6aa8b732006-12-10 02:21:36 -080027#include <linux/debugfs.h>
28#include <linux/highmem.h>
29#include <linux/file.h>
Rafael J. Wysockifb3600c2011-03-23 22:16:23 +010030#include <linux/syscore_ops.h>
Avi Kivity774c47f2007-02-12 00:54:47 -080031#include <linux/cpu.h>
Ingo Molnar174cd4b2017-02-02 19:15:33 +010032#include <linux/sched/signal.h>
Ingo Molnar6e84f312017-02-08 18:51:29 +010033#include <linux/sched/mm.h>
Ingo Molnar03441a32017-02-08 18:51:35 +010034#include <linux/sched/stat.h>
Avi Kivityd9e368d2007-06-07 19:18:30 +030035#include <linux/cpumask.h>
36#include <linux/smp.h>
Avi Kivityd6d28162007-06-28 08:38:16 -040037#include <linux/anon_inodes.h>
Avi Kivity04d2cc72007-09-10 18:10:54 +030038#include <linux/profile.h>
Anthony Liguori7aa81cc2007-09-17 14:57:50 -050039#include <linux/kvm_para.h>
Izik Eidus6fc138d2007-10-09 19:20:39 +020040#include <linux/pagemap.h>
Anthony Liguori8d4e1282007-10-18 09:59:34 -050041#include <linux/mman.h>
Anthony Liguori35149e22008-04-02 14:46:56 -050042#include <linux/swap.h>
Sheng Yange56d5322009-03-12 21:45:39 +080043#include <linux/bitops.h>
Marcelo Tosatti547de292009-05-07 17:55:13 -030044#include <linux/spinlock.h>
Arnd Bergmann6ff58942009-10-22 14:19:27 +020045#include <linux/compat.h>
Marcelo Tosattibc6678a2009-12-23 14:35:21 -020046#include <linux/srcu.h>
Joerg Roedel8f0b1ab2010-01-28 12:37:56 +010047#include <linux/hugetlb.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090048#include <linux/slab.h>
Sasha Levin743eeb02011-07-27 16:00:48 +030049#include <linux/sort.h>
50#include <linux/bsearch.h>
Paolo Bonzinic011d232019-05-17 14:08:53 +020051#include <linux/io.h>
Wanpeng Li2eb06c32019-05-17 16:49:49 +080052#include <linux/lockdep.h>
Junaid Shahidc57c8042019-11-04 12:22:02 +010053#include <linux/kthread.h>
Avi Kivity6aa8b732006-12-10 02:21:36 -080054
Avi Kivitye4956062007-06-28 14:15:57 -040055#include <asm/processor.h>
David Matlack2ea75be2014-09-19 16:03:25 -070056#include <asm/ioctl.h>
Linus Torvalds7c0f6ba2016-12-24 11:46:01 -080057#include <linux/uaccess.h>
Izik Eidus3e021bf2007-11-19 11:16:57 +020058#include <asm/pgtable.h>
Avi Kivity6aa8b732006-12-10 02:21:36 -080059
Laurent Vivier5f94c172008-05-30 16:05:54 +020060#include "coalesced_mmio.h"
Gleb Natapovaf585b92010-10-14 11:22:46 +020061#include "async_pf.h"
Paolo Bonzini3c3c29f2014-09-24 13:02:46 +020062#include "vfio.h"
Laurent Vivier5f94c172008-05-30 16:05:54 +020063
Marcelo Tosatti229456f2009-06-17 09:22:14 -030064#define CREATE_TRACE_POINTS
65#include <trace/events/kvm.h>
66
Janosch Frank536a6f82016-05-18 13:26:23 +020067/* Worst case buffer size needed for holding an integer. */
68#define ITOA_MAX_LEN 12
69
Avi Kivity6aa8b732006-12-10 02:21:36 -080070MODULE_AUTHOR("Qumranet");
71MODULE_LICENSE("GPL");
72
David Hildenbrand920552b2015-09-18 12:34:53 +020073/* Architectures should define their poll value according to the halt latency */
Suraj Jitindar Singhec76d812016-10-14 11:53:19 +110074unsigned int halt_poll_ns = KVM_HALT_POLL_NS_DEFAULT;
Roman Storozhenko039c5d12017-06-27 12:51:18 +030075module_param(halt_poll_ns, uint, 0644);
Suraj Jitindar Singhec76d812016-10-14 11:53:19 +110076EXPORT_SYMBOL_GPL(halt_poll_ns);
Paolo Bonzinif7819512015-02-04 18:20:58 +010077
Wanpeng Liaca6ff22015-09-03 22:07:38 +080078/* Default doubles per-vcpu halt_poll_ns. */
Suraj Jitindar Singhec76d812016-10-14 11:53:19 +110079unsigned int halt_poll_ns_grow = 2;
Roman Storozhenko039c5d12017-06-27 12:51:18 +030080module_param(halt_poll_ns_grow, uint, 0644);
Suraj Jitindar Singhec76d812016-10-14 11:53:19 +110081EXPORT_SYMBOL_GPL(halt_poll_ns_grow);
Wanpeng Liaca6ff22015-09-03 22:07:38 +080082
Nir Weiner49113d32019-01-27 12:17:15 +020083/* The start value to grow halt_poll_ns from */
84unsigned int halt_poll_ns_grow_start = 10000; /* 10us */
85module_param(halt_poll_ns_grow_start, uint, 0644);
86EXPORT_SYMBOL_GPL(halt_poll_ns_grow_start);
87
Wanpeng Liaca6ff22015-09-03 22:07:38 +080088/* Default resets per-vcpu halt_poll_ns . */
Suraj Jitindar Singhec76d812016-10-14 11:53:19 +110089unsigned int halt_poll_ns_shrink;
Roman Storozhenko039c5d12017-06-27 12:51:18 +030090module_param(halt_poll_ns_shrink, uint, 0644);
Suraj Jitindar Singhec76d812016-10-14 11:53:19 +110091EXPORT_SYMBOL_GPL(halt_poll_ns_shrink);
Wanpeng Liaca6ff22015-09-03 22:07:38 +080092
Marcelo Tosattifa40a822009-06-04 15:08:24 -030093/*
94 * Ordering of locks:
95 *
Xiubo Lib7d409d2015-02-26 14:58:24 +080096 * kvm->lock --> kvm->slots_lock --> kvm->irq_lock
Marcelo Tosattifa40a822009-06-04 15:08:24 -030097 */
98
Junaid Shahid0d9ce162019-01-03 17:14:28 -080099DEFINE_MUTEX(kvm_lock);
Paolo Bonzini4a937f92013-09-10 12:58:35 +0200100static DEFINE_RAW_SPINLOCK(kvm_count_lock);
Zhang Xiantaoe9b11c12007-11-14 20:38:21 +0800101LIST_HEAD(vm_list);
Avi Kivity133de902007-02-12 00:54:44 -0800102
Rusty Russell7f59f492008-12-07 21:25:45 +1030103static cpumask_var_t cpus_hardware_enabled;
Xiubo Lif4fee932015-02-26 14:58:21 +0800104static int kvm_usage_count;
Alexander Graf10474ae2009-09-15 11:37:46 +0200105static atomic_t hardware_enable_failed;
Avi Kivity1b6c0162007-05-24 13:03:52 +0300106
Rusty Russellc16f8622007-07-30 21:12:19 +1000107struct kmem_cache *kvm_vcpu_cache;
108EXPORT_SYMBOL_GPL(kvm_vcpu_cache);
Avi Kivity1165f5f2007-04-19 17:27:43 +0300109
Avi Kivity15ad7142007-07-11 18:17:21 +0300110static __read_mostly struct preempt_ops kvm_preempt_ops;
111
Hollis Blanchard76f7c872008-04-15 16:05:42 -0500112struct dentry *kvm_debugfs_dir;
Paul Mackerrase23a8082015-03-28 14:21:01 +1100113EXPORT_SYMBOL_GPL(kvm_debugfs_dir);
Avi Kivity6aa8b732006-12-10 02:21:36 -0800114
Janosch Frank536a6f82016-05-18 13:26:23 +0200115static int kvm_debugfs_num_entries;
116static const struct file_operations *stat_fops_per_vm[];
117
Avi Kivitybccf2152007-02-21 18:04:26 +0200118static long kvm_vcpu_ioctl(struct file *file, unsigned int ioctl,
119 unsigned long arg);
Christian Borntraegerde8e5d72015-02-03 09:35:15 +0100120#ifdef CONFIG_KVM_COMPAT
Alexander Graf1dda6062011-06-08 02:45:37 +0200121static long kvm_vcpu_compat_ioctl(struct file *file, unsigned int ioctl,
122 unsigned long arg);
Marc Zyngier7ddfd3e2018-06-17 10:16:21 +0100123#define KVM_COMPAT(c) .compat_ioctl = (c)
124#else
Marc Zyngier9cb09e72019-11-14 13:17:39 +0000125/*
126 * For architectures that don't implement a compat infrastructure,
127 * adopt a double line of defense:
128 * - Prevent a compat task from opening /dev/kvm
129 * - If the open has been done by a 64bit task, and the KVM fd
130 * passed to a compat task, let the ioctls fail.
131 */
Marc Zyngier7ddfd3e2018-06-17 10:16:21 +0100132static long kvm_no_compat_ioctl(struct file *file, unsigned int ioctl,
133 unsigned long arg) { return -EINVAL; }
Marc Zyngierb9876e62019-11-13 16:05:23 +0000134
135static int kvm_no_compat_open(struct inode *inode, struct file *file)
136{
137 return is_compat_task() ? -ENODEV : 0;
138}
139#define KVM_COMPAT(c) .compat_ioctl = kvm_no_compat_ioctl, \
140 .open = kvm_no_compat_open
Alexander Graf1dda6062011-06-08 02:45:37 +0200141#endif
Alexander Graf10474ae2009-09-15 11:37:46 +0200142static int hardware_enable_all(void);
143static void hardware_disable_all(void);
Avi Kivitybccf2152007-02-21 18:04:26 +0200144
Marcelo Tosattie93f8a02009-12-23 14:35:24 -0200145static void kvm_io_bus_destroy(struct kvm_io_bus *bus);
Stephen Hemminger79408762013-12-29 12:12:29 -0800146
Paolo Bonzinibc009e42015-05-26 12:43:41 +0200147static void mark_page_dirty_in_slot(struct kvm_memory_slot *memslot, gfn_t gfn);
Marcelo Tosattie93f8a02009-12-23 14:35:24 -0200148
Andi Kleen52480132014-02-08 08:51:57 +0100149__visible bool kvm_rebooting;
Avi Kivityb7c41452010-12-02 17:52:50 +0200150EXPORT_SYMBOL_GPL(kvm_rebooting);
Avi Kivity4ecac3f2008-05-13 13:23:38 +0300151
Marcelo Tosatti54dee992009-06-11 12:07:44 -0300152static bool largepages_enabled = true;
153
Claudio Imbrenda286de8f2017-07-12 17:56:44 +0200154#define KVM_EVENT_CREATE_VM 0
155#define KVM_EVENT_DESTROY_VM 1
156static void kvm_uevent_notify_change(unsigned int type, struct kvm *kvm);
157static unsigned long long kvm_createvm_count;
158static unsigned long long kvm_active_vms;
159
Michal Hocko93065ac2018-08-21 21:52:33 -0700160__weak int kvm_arch_mmu_notifier_invalidate_range(struct kvm *kvm,
161 unsigned long start, unsigned long end, bool blockable)
Radim Krčmářb1394e72017-11-30 19:05:45 +0100162{
Michal Hocko93065ac2018-08-21 21:52:33 -0700163 return 0;
Radim Krčmářb1394e72017-11-30 19:05:45 +0100164}
165
Sean Christophersona78986a2019-11-11 14:12:27 -0800166bool kvm_is_zone_device_pfn(kvm_pfn_t pfn)
167{
168 /*
169 * The metadata used by is_zone_device_page() to determine whether or
170 * not a page is ZONE_DEVICE is guaranteed to be valid if and only if
171 * the device has been pinned, e.g. by get_user_pages(). WARN if the
172 * page_count() is zero to help detect bad usage of this helper.
173 */
174 if (!pfn_valid(pfn) || WARN_ON_ONCE(!page_count(pfn_to_page(pfn))))
175 return false;
176
177 return is_zone_device_page(pfn_to_page(pfn));
178}
179
Dan Williamsba049e92016-01-15 16:56:11 -0800180bool kvm_is_reserved_pfn(kvm_pfn_t pfn)
Ben-Ami Yassourcbff90a2008-07-28 19:26:24 +0300181{
Sean Christophersona78986a2019-11-11 14:12:27 -0800182 /*
183 * ZONE_DEVICE pages currently set PG_reserved, but from a refcounting
184 * perspective they are "normal" pages, albeit with slightly different
185 * usage rules.
186 */
Andrea Arcangeli11feeb42013-07-25 03:04:38 +0200187 if (pfn_valid(pfn))
Sean Christophersona78986a2019-11-11 14:12:27 -0800188 return PageReserved(pfn_to_page(pfn)) &&
189 !kvm_is_zone_device_pfn(pfn);
Ben-Ami Yassourcbff90a2008-07-28 19:26:24 +0300190
191 return true;
192}
193
Avi Kivity6aa8b732006-12-10 02:21:36 -0800194/*
195 * Switches to specified vcpu, until a matching vcpu_put()
196 */
Christoffer Dallec7660c2017-12-04 21:35:23 +0100197void vcpu_load(struct kvm_vcpu *vcpu)
Avi Kivity6aa8b732006-12-10 02:21:36 -0800198{
Christoffer Dallec7660c2017-12-04 21:35:23 +0100199 int cpu = get_cpu();
Avi Kivity15ad7142007-07-11 18:17:21 +0300200 preempt_notifier_register(&vcpu->preempt_notifier);
Carsten Otte313a3dc2007-10-11 19:16:52 +0200201 kvm_arch_vcpu_load(vcpu, cpu);
Avi Kivity15ad7142007-07-11 18:17:21 +0300202 put_cpu();
Avi Kivitybccf2152007-02-21 18:04:26 +0200203}
Jim Mattson2f1fe812016-07-08 15:36:06 -0700204EXPORT_SYMBOL_GPL(vcpu_load);
Avi Kivitybccf2152007-02-21 18:04:26 +0200205
Carsten Otte313a3dc2007-10-11 19:16:52 +0200206void vcpu_put(struct kvm_vcpu *vcpu)
Avi Kivity6aa8b732006-12-10 02:21:36 -0800207{
Avi Kivity15ad7142007-07-11 18:17:21 +0300208 preempt_disable();
Carsten Otte313a3dc2007-10-11 19:16:52 +0200209 kvm_arch_vcpu_put(vcpu);
Avi Kivity15ad7142007-07-11 18:17:21 +0300210 preempt_notifier_unregister(&vcpu->preempt_notifier);
211 preempt_enable();
Avi Kivity6aa8b732006-12-10 02:21:36 -0800212}
Jim Mattson2f1fe812016-07-08 15:36:06 -0700213EXPORT_SYMBOL_GPL(vcpu_put);
Avi Kivity6aa8b732006-12-10 02:21:36 -0800214
Paolo Bonzini7a97cec2017-04-27 14:33:43 +0200215/* TODO: merge with kvm_arch_vcpu_should_kick */
216static bool kvm_request_needs_ipi(struct kvm_vcpu *vcpu, unsigned req)
217{
218 int mode = kvm_vcpu_exiting_guest_mode(vcpu);
219
220 /*
221 * We need to wait for the VCPU to reenable interrupts and get out of
222 * READING_SHADOW_PAGE_TABLES mode.
223 */
224 if (req & KVM_REQUEST_WAIT)
225 return mode != OUTSIDE_GUEST_MODE;
226
227 /*
228 * Need to kick a running VCPU, but otherwise there is nothing to do.
229 */
230 return mode == IN_GUEST_MODE;
231}
232
Avi Kivityd9e368d2007-06-07 19:18:30 +0300233static void ack_flush(void *_completed)
234{
Avi Kivityd9e368d2007-06-07 19:18:30 +0300235}
236
Paolo Bonzinib49defe2017-06-30 13:25:45 +0200237static inline bool kvm_kick_many_cpus(const struct cpumask *cpus, bool wait)
238{
239 if (unlikely(!cpus))
240 cpus = cpu_online_mask;
241
242 if (cpumask_empty(cpus))
243 return false;
244
245 smp_call_function_many(cpus, ack_flush, NULL, wait);
246 return true;
247}
248
Vitaly Kuznetsov7053df42018-05-16 17:21:28 +0200249bool kvm_make_vcpus_request_mask(struct kvm *kvm, unsigned int req,
250 unsigned long *vcpu_bitmap, cpumask_var_t tmp)
Avi Kivityd9e368d2007-06-07 19:18:30 +0300251{
Avi Kivity597a5f52008-07-20 14:24:22 +0300252 int i, cpu, me;
Avi Kivityd9e368d2007-06-07 19:18:30 +0300253 struct kvm_vcpu *vcpu;
Vitaly Kuznetsov7053df42018-05-16 17:21:28 +0200254 bool called;
Rusty Russell6ef7a1b2008-12-08 20:28:04 +1030255
Xiao Guangrong3cba4132011-01-12 15:41:22 +0800256 me = get_cpu();
Vitaly Kuznetsov7053df42018-05-16 17:21:28 +0200257
Gleb Natapov988a2ca2009-06-09 15:56:29 +0300258 kvm_for_each_vcpu(i, vcpu, kvm) {
Vitaly Kuznetsova8122972018-08-22 12:18:29 +0200259 if (vcpu_bitmap && !test_bit(i, vcpu_bitmap))
Vitaly Kuznetsov7053df42018-05-16 17:21:28 +0200260 continue;
261
Xiao Guangrong3cba4132011-01-12 15:41:22 +0800262 kvm_make_request(req, vcpu);
Avi Kivityd9e368d2007-06-07 19:18:30 +0300263 cpu = vcpu->cpu;
Xiao Guangrong6b7e2d02011-01-12 15:40:31 +0800264
Radim Krčmář178f02f2017-04-26 22:32:26 +0200265 if (!(req & KVM_REQUEST_NO_WAKEUP) && kvm_vcpu_wake_up(vcpu))
266 continue;
Radim Krčmář6c6e8362017-04-26 22:32:23 +0200267
Vitaly Kuznetsov7053df42018-05-16 17:21:28 +0200268 if (tmp != NULL && cpu != -1 && cpu != me &&
Paolo Bonzini7a97cec2017-04-27 14:33:43 +0200269 kvm_request_needs_ipi(vcpu, req))
Vitaly Kuznetsov7053df42018-05-16 17:21:28 +0200270 __cpumask_set_cpu(cpu, tmp);
Avi Kivityd9e368d2007-06-07 19:18:30 +0300271 }
Vitaly Kuznetsov7053df42018-05-16 17:21:28 +0200272
273 called = kvm_kick_many_cpus(tmp, !!(req & KVM_REQUEST_WAIT));
Xiao Guangrong3cba4132011-01-12 15:41:22 +0800274 put_cpu();
Vitaly Kuznetsov7053df42018-05-16 17:21:28 +0200275
276 return called;
277}
278
279bool kvm_make_all_cpus_request(struct kvm *kvm, unsigned int req)
280{
281 cpumask_var_t cpus;
282 bool called;
Vitaly Kuznetsov7053df42018-05-16 17:21:28 +0200283
284 zalloc_cpumask_var(&cpus, GFP_ATOMIC);
285
Vitaly Kuznetsova8122972018-08-22 12:18:29 +0200286 called = kvm_make_vcpus_request_mask(kvm, req, NULL, cpus);
Vitaly Kuznetsov7053df42018-05-16 17:21:28 +0200287
Rusty Russell6ef7a1b2008-12-08 20:28:04 +1030288 free_cpumask_var(cpus);
Rusty Russell49846892008-12-08 20:26:24 +1030289 return called;
290}
291
Mario Smarducha6d51012015-01-15 15:58:52 -0800292#ifndef CONFIG_HAVE_KVM_ARCH_TLB_FLUSH_ALL
Rusty Russell49846892008-12-08 20:26:24 +1030293void kvm_flush_remote_tlbs(struct kvm *kvm)
294{
Lan Tianyu4ae3cb32016-03-13 11:10:28 +0800295 /*
296 * Read tlbs_dirty before setting KVM_REQ_TLB_FLUSH in
297 * kvm_make_all_cpus_request.
298 */
299 long dirty_count = smp_load_acquire(&kvm->tlbs_dirty);
Xiao Guangronga086f6a2014-04-17 17:06:12 +0800300
Lan Tianyu4ae3cb32016-03-13 11:10:28 +0800301 /*
302 * We want to publish modifications to the page tables before reading
303 * mode. Pairs with a memory barrier in arch-specific code.
304 * - x86: smp_mb__after_srcu_read_unlock in vcpu_enter_guest
305 * and smp_mb in walk_shadow_page_lockless_begin/end.
306 * - powerpc: smp_mb in kvmppc_prepare_to_enter.
307 *
308 * There is already an smp_mb__after_atomic() before
309 * kvm_make_all_cpus_request() reads vcpu->mode. We reuse that
310 * barrier here.
311 */
Tianyu Lanb08660e2018-07-19 08:40:17 +0000312 if (!kvm_arch_flush_remote_tlb(kvm)
313 || kvm_make_all_cpus_request(kvm, KVM_REQ_TLB_FLUSH))
Rusty Russell49846892008-12-08 20:26:24 +1030314 ++kvm->stat.remote_tlb_flush;
Xiao Guangronga086f6a2014-04-17 17:06:12 +0800315 cmpxchg(&kvm->tlbs_dirty, dirty_count, 0);
Avi Kivityd9e368d2007-06-07 19:18:30 +0300316}
Aneesh Kumar K.V2ba9f0d2013-10-07 22:17:59 +0530317EXPORT_SYMBOL_GPL(kvm_flush_remote_tlbs);
Mario Smarducha6d51012015-01-15 15:58:52 -0800318#endif
Avi Kivityd9e368d2007-06-07 19:18:30 +0300319
Marcelo Tosatti2e53d632008-02-20 14:47:24 -0500320void kvm_reload_remote_mmus(struct kvm *kvm)
321{
Tang Chen445b8232014-09-24 15:57:55 +0800322 kvm_make_all_cpus_request(kvm, KVM_REQ_MMU_RELOAD);
Marcelo Tosatti2e53d632008-02-20 14:47:24 -0500323}
324
Rusty Russellfb3f0f52007-07-27 17:16:56 +1000325int kvm_vcpu_init(struct kvm_vcpu *vcpu, struct kvm *kvm, unsigned id)
326{
327 struct page *page;
328 int r;
329
330 mutex_init(&vcpu->mutex);
331 vcpu->cpu = -1;
Rusty Russellfb3f0f52007-07-27 17:16:56 +1000332 vcpu->kvm = kvm;
333 vcpu->vcpu_id = id;
Rik van Riel34bb10b2011-02-01 09:52:41 -0500334 vcpu->pid = NULL;
Marcelo Tosatti85773702016-02-19 09:46:39 +0100335 init_swait_queue_head(&vcpu->wq);
Gleb Natapovaf585b92010-10-14 11:22:46 +0200336 kvm_async_pf_vcpu_init(vcpu);
Rusty Russellfb3f0f52007-07-27 17:16:56 +1000337
Feng Wubf9f6ac2015-09-18 22:29:55 +0800338 vcpu->pre_pcpu = -1;
339 INIT_LIST_HEAD(&vcpu->blocked_vcpu_list);
340
Rusty Russellfb3f0f52007-07-27 17:16:56 +1000341 page = alloc_page(GFP_KERNEL | __GFP_ZERO);
342 if (!page) {
343 r = -ENOMEM;
344 goto fail;
345 }
346 vcpu->run = page_address(page);
347
Raghavendra K T4c088492012-07-18 19:07:46 +0530348 kvm_vcpu_set_in_spin_loop(vcpu, false);
349 kvm_vcpu_set_dy_eligible(vcpu, false);
Raghavendra K T3a08a8f2013-03-04 23:32:07 +0530350 vcpu->preempted = false;
Wanpeng Lid73eb572019-07-18 19:39:06 +0800351 vcpu->ready = false;
Raghavendra K T4c088492012-07-18 19:07:46 +0530352
Zhang Xiantaoe9b11c12007-11-14 20:38:21 +0800353 r = kvm_arch_vcpu_init(vcpu);
Rusty Russellfb3f0f52007-07-27 17:16:56 +1000354 if (r < 0)
Zhang Xiantaoe9b11c12007-11-14 20:38:21 +0800355 goto fail_free_run;
Rusty Russellfb3f0f52007-07-27 17:16:56 +1000356 return 0;
357
Rusty Russellfb3f0f52007-07-27 17:16:56 +1000358fail_free_run:
359 free_page((unsigned long)vcpu->run);
360fail:
Rusty Russell76fafa52007-10-08 10:50:48 +1000361 return r;
Rusty Russellfb3f0f52007-07-27 17:16:56 +1000362}
363EXPORT_SYMBOL_GPL(kvm_vcpu_init);
364
365void kvm_vcpu_uninit(struct kvm_vcpu *vcpu)
366{
Christian Borntraeger0e4524a2017-07-06 14:44:28 +0200367 /*
368 * no need for rcu_read_lock as VCPU_RUN is the only place that
369 * will change the vcpu->pid pointer and on uninit all file
370 * descriptors are already gone.
371 */
372 put_pid(rcu_dereference_protected(vcpu->pid, 1));
Zhang Xiantaoe9b11c12007-11-14 20:38:21 +0800373 kvm_arch_vcpu_uninit(vcpu);
Rusty Russellfb3f0f52007-07-27 17:16:56 +1000374 free_page((unsigned long)vcpu->run);
375}
376EXPORT_SYMBOL_GPL(kvm_vcpu_uninit);
377
Andrea Arcangelie930bff2008-07-25 16:24:52 +0200378#if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER)
379static inline struct kvm *mmu_notifier_to_kvm(struct mmu_notifier *mn)
380{
381 return container_of(mn, struct kvm, mmu_notifier);
382}
383
Izik Eidus3da0dd42009-09-23 21:47:18 +0300384static void kvm_mmu_notifier_change_pte(struct mmu_notifier *mn,
385 struct mm_struct *mm,
386 unsigned long address,
387 pte_t pte)
388{
389 struct kvm *kvm = mmu_notifier_to_kvm(mn);
Marcelo Tosattibc6678a2009-12-23 14:35:21 -0200390 int idx;
Izik Eidus3da0dd42009-09-23 21:47:18 +0300391
Marcelo Tosattibc6678a2009-12-23 14:35:21 -0200392 idx = srcu_read_lock(&kvm->srcu);
Izik Eidus3da0dd42009-09-23 21:47:18 +0300393 spin_lock(&kvm->mmu_lock);
394 kvm->mmu_notifier_seq++;
Lan Tianyu0cf853c2018-12-06 21:21:11 +0800395
396 if (kvm_set_spte_hva(kvm, address, pte))
397 kvm_flush_remote_tlbs(kvm);
398
Izik Eidus3da0dd42009-09-23 21:47:18 +0300399 spin_unlock(&kvm->mmu_lock);
Marcelo Tosattibc6678a2009-12-23 14:35:21 -0200400 srcu_read_unlock(&kvm->srcu, idx);
Izik Eidus3da0dd42009-09-23 21:47:18 +0300401}
402
Michal Hocko93065ac2018-08-21 21:52:33 -0700403static int kvm_mmu_notifier_invalidate_range_start(struct mmu_notifier *mn,
Jérôme Glisse5d6527a2018-12-28 00:38:05 -0800404 const struct mmu_notifier_range *range)
Andrea Arcangelie930bff2008-07-25 16:24:52 +0200405{
406 struct kvm *kvm = mmu_notifier_to_kvm(mn);
Marcelo Tosattibc6678a2009-12-23 14:35:21 -0200407 int need_tlb_flush = 0, idx;
Michal Hocko93065ac2018-08-21 21:52:33 -0700408 int ret;
Andrea Arcangelie930bff2008-07-25 16:24:52 +0200409
Marcelo Tosattibc6678a2009-12-23 14:35:21 -0200410 idx = srcu_read_lock(&kvm->srcu);
Andrea Arcangelie930bff2008-07-25 16:24:52 +0200411 spin_lock(&kvm->mmu_lock);
412 /*
413 * The count increase must become visible at unlock time as no
414 * spte can be established without taking the mmu_lock and
415 * count is also read inside the mmu_lock critical section.
416 */
417 kvm->mmu_notifier_count++;
Jérôme Glisse5d6527a2018-12-28 00:38:05 -0800418 need_tlb_flush = kvm_unmap_hva_range(kvm, range->start, range->end);
Xiao Guangronga4ee1ca2010-11-23 11:13:00 +0800419 need_tlb_flush |= kvm->tlbs_dirty;
Andrea Arcangelie930bff2008-07-25 16:24:52 +0200420 /* we've to flush the tlb before the pages can be freed */
421 if (need_tlb_flush)
422 kvm_flush_remote_tlbs(kvm);
Takuya Yoshikawa565f3be2012-02-10 15:28:31 +0900423
424 spin_unlock(&kvm->mmu_lock);
Radim Krčmářb1394e72017-11-30 19:05:45 +0100425
Jérôme Glisse5d6527a2018-12-28 00:38:05 -0800426 ret = kvm_arch_mmu_notifier_invalidate_range(kvm, range->start,
Jérôme Glissedfcd6662019-05-13 17:20:38 -0700427 range->end,
428 mmu_notifier_range_blockable(range));
Radim Krčmářb1394e72017-11-30 19:05:45 +0100429
Takuya Yoshikawa565f3be2012-02-10 15:28:31 +0900430 srcu_read_unlock(&kvm->srcu, idx);
Michal Hocko93065ac2018-08-21 21:52:33 -0700431
432 return ret;
Andrea Arcangelie930bff2008-07-25 16:24:52 +0200433}
434
435static void kvm_mmu_notifier_invalidate_range_end(struct mmu_notifier *mn,
Jérôme Glisse5d6527a2018-12-28 00:38:05 -0800436 const struct mmu_notifier_range *range)
Andrea Arcangelie930bff2008-07-25 16:24:52 +0200437{
438 struct kvm *kvm = mmu_notifier_to_kvm(mn);
439
440 spin_lock(&kvm->mmu_lock);
441 /*
442 * This sequence increase will notify the kvm page fault that
443 * the page that is going to be mapped in the spte could have
444 * been freed.
445 */
446 kvm->mmu_notifier_seq++;
Paul Mackerrasa355aa52011-12-12 12:37:21 +0000447 smp_wmb();
Andrea Arcangelie930bff2008-07-25 16:24:52 +0200448 /*
449 * The above sequence increase must be visible before the
Paul Mackerrasa355aa52011-12-12 12:37:21 +0000450 * below count decrease, which is ensured by the smp_wmb above
451 * in conjunction with the smp_rmb in mmu_notifier_retry().
Andrea Arcangelie930bff2008-07-25 16:24:52 +0200452 */
453 kvm->mmu_notifier_count--;
454 spin_unlock(&kvm->mmu_lock);
455
456 BUG_ON(kvm->mmu_notifier_count < 0);
457}
458
459static int kvm_mmu_notifier_clear_flush_young(struct mmu_notifier *mn,
460 struct mm_struct *mm,
Andres Lagar-Cavilla57128462014-09-22 14:54:42 -0700461 unsigned long start,
462 unsigned long end)
Andrea Arcangelie930bff2008-07-25 16:24:52 +0200463{
464 struct kvm *kvm = mmu_notifier_to_kvm(mn);
Marcelo Tosattibc6678a2009-12-23 14:35:21 -0200465 int young, idx;
Andrea Arcangelie930bff2008-07-25 16:24:52 +0200466
Marcelo Tosattibc6678a2009-12-23 14:35:21 -0200467 idx = srcu_read_lock(&kvm->srcu);
Andrea Arcangelie930bff2008-07-25 16:24:52 +0200468 spin_lock(&kvm->mmu_lock);
Andrea Arcangelie930bff2008-07-25 16:24:52 +0200469
Andres Lagar-Cavilla57128462014-09-22 14:54:42 -0700470 young = kvm_age_hva(kvm, start, end);
Andrea Arcangelie930bff2008-07-25 16:24:52 +0200471 if (young)
472 kvm_flush_remote_tlbs(kvm);
473
Takuya Yoshikawa565f3be2012-02-10 15:28:31 +0900474 spin_unlock(&kvm->mmu_lock);
475 srcu_read_unlock(&kvm->srcu, idx);
476
Andrea Arcangelie930bff2008-07-25 16:24:52 +0200477 return young;
478}
479
Vladimir Davydov1d7715c2015-09-09 15:35:41 -0700480static int kvm_mmu_notifier_clear_young(struct mmu_notifier *mn,
481 struct mm_struct *mm,
482 unsigned long start,
483 unsigned long end)
484{
485 struct kvm *kvm = mmu_notifier_to_kvm(mn);
486 int young, idx;
487
488 idx = srcu_read_lock(&kvm->srcu);
489 spin_lock(&kvm->mmu_lock);
490 /*
491 * Even though we do not flush TLB, this will still adversely
492 * affect performance on pre-Haswell Intel EPT, where there is
493 * no EPT Access Bit to clear so that we have to tear down EPT
494 * tables instead. If we find this unacceptable, we can always
495 * add a parameter to kvm_age_hva so that it effectively doesn't
496 * do anything on clear_young.
497 *
498 * Also note that currently we never issue secondary TLB flushes
499 * from clear_young, leaving this job up to the regular system
500 * cadence. If we find this inaccurate, we might come up with a
501 * more sophisticated heuristic later.
502 */
503 young = kvm_age_hva(kvm, start, end);
504 spin_unlock(&kvm->mmu_lock);
505 srcu_read_unlock(&kvm->srcu, idx);
506
507 return young;
508}
509
Andrea Arcangeli8ee53822011-01-13 15:47:10 -0800510static int kvm_mmu_notifier_test_young(struct mmu_notifier *mn,
511 struct mm_struct *mm,
512 unsigned long address)
513{
514 struct kvm *kvm = mmu_notifier_to_kvm(mn);
515 int young, idx;
516
517 idx = srcu_read_lock(&kvm->srcu);
518 spin_lock(&kvm->mmu_lock);
519 young = kvm_test_age_hva(kvm, address);
520 spin_unlock(&kvm->mmu_lock);
521 srcu_read_unlock(&kvm->srcu, idx);
522
523 return young;
524}
525
Marcelo Tosatti85db06e2008-12-10 21:23:26 +0100526static void kvm_mmu_notifier_release(struct mmu_notifier *mn,
527 struct mm_struct *mm)
528{
529 struct kvm *kvm = mmu_notifier_to_kvm(mn);
Lai Jiangshaneda2bed2010-04-20 14:29:29 +0800530 int idx;
531
532 idx = srcu_read_lock(&kvm->srcu);
Marcelo Tosatti2df72e92012-08-24 15:54:57 -0300533 kvm_arch_flush_shadow_all(kvm);
Lai Jiangshaneda2bed2010-04-20 14:29:29 +0800534 srcu_read_unlock(&kvm->srcu, idx);
Marcelo Tosatti85db06e2008-12-10 21:23:26 +0100535}
536
Andrea Arcangelie930bff2008-07-25 16:24:52 +0200537static const struct mmu_notifier_ops kvm_mmu_notifier_ops = {
Andrea Arcangelie930bff2008-07-25 16:24:52 +0200538 .invalidate_range_start = kvm_mmu_notifier_invalidate_range_start,
539 .invalidate_range_end = kvm_mmu_notifier_invalidate_range_end,
540 .clear_flush_young = kvm_mmu_notifier_clear_flush_young,
Vladimir Davydov1d7715c2015-09-09 15:35:41 -0700541 .clear_young = kvm_mmu_notifier_clear_young,
Andrea Arcangeli8ee53822011-01-13 15:47:10 -0800542 .test_young = kvm_mmu_notifier_test_young,
Izik Eidus3da0dd42009-09-23 21:47:18 +0300543 .change_pte = kvm_mmu_notifier_change_pte,
Marcelo Tosatti85db06e2008-12-10 21:23:26 +0100544 .release = kvm_mmu_notifier_release,
Andrea Arcangelie930bff2008-07-25 16:24:52 +0200545};
Avi Kivity4c07b0a2009-12-20 14:54:04 +0200546
547static int kvm_init_mmu_notifier(struct kvm *kvm)
548{
549 kvm->mmu_notifier.ops = &kvm_mmu_notifier_ops;
550 return mmu_notifier_register(&kvm->mmu_notifier, current->mm);
551}
552
553#else /* !(CONFIG_MMU_NOTIFIER && KVM_ARCH_WANT_MMU_NOTIFIER) */
554
555static int kvm_init_mmu_notifier(struct kvm *kvm)
556{
557 return 0;
558}
559
Andrea Arcangelie930bff2008-07-25 16:24:52 +0200560#endif /* CONFIG_MMU_NOTIFIER && KVM_ARCH_WANT_MMU_NOTIFIER */
561
Paolo Bonzinia47d2b02015-05-17 11:41:37 +0200562static struct kvm_memslots *kvm_alloc_memslots(void)
Xiao Guangrongbf3e05b2011-11-24 17:40:57 +0800563{
564 int i;
Paolo Bonzinia47d2b02015-05-17 11:41:37 +0200565 struct kvm_memslots *slots;
Xiao Guangrongbf3e05b2011-11-24 17:40:57 +0800566
Ben Gardonb12ce362019-02-11 11:02:49 -0800567 slots = kvzalloc(sizeof(struct kvm_memslots), GFP_KERNEL_ACCOUNT);
Paolo Bonzinia47d2b02015-05-17 11:41:37 +0200568 if (!slots)
569 return NULL;
570
Xiao Guangrongbf3e05b2011-11-24 17:40:57 +0800571 for (i = 0; i < KVM_MEM_SLOTS_NUM; i++)
Xiao Guangrongf85e2cb2011-11-24 17:41:54 +0800572 slots->id_to_index[i] = slots->memslots[i].id = i;
Paolo Bonzinia47d2b02015-05-17 11:41:37 +0200573
574 return slots;
575}
576
577static void kvm_destroy_dirty_bitmap(struct kvm_memory_slot *memslot)
578{
579 if (!memslot->dirty_bitmap)
580 return;
581
582 kvfree(memslot->dirty_bitmap);
583 memslot->dirty_bitmap = NULL;
584}
585
586/*
587 * Free any memory in @free but not in @dont.
588 */
589static void kvm_free_memslot(struct kvm *kvm, struct kvm_memory_slot *free,
590 struct kvm_memory_slot *dont)
591{
592 if (!dont || free->dirty_bitmap != dont->dirty_bitmap)
593 kvm_destroy_dirty_bitmap(free);
594
595 kvm_arch_free_memslot(kvm, free, dont);
596
597 free->npages = 0;
598}
599
600static void kvm_free_memslots(struct kvm *kvm, struct kvm_memslots *slots)
601{
602 struct kvm_memory_slot *memslot;
603
604 if (!slots)
605 return;
606
607 kvm_for_each_memslot(memslot, slots)
608 kvm_free_memslot(kvm, memslot, NULL);
609
610 kvfree(slots);
Xiao Guangrongbf3e05b2011-11-24 17:40:57 +0800611}
612
Janosch Frank536a6f82016-05-18 13:26:23 +0200613static void kvm_destroy_vm_debugfs(struct kvm *kvm)
614{
615 int i;
616
617 if (!kvm->debugfs_dentry)
618 return;
619
620 debugfs_remove_recursive(kvm->debugfs_dentry);
621
Luiz Capitulino9d5a1dc2016-09-07 14:47:21 -0400622 if (kvm->debugfs_stat_data) {
623 for (i = 0; i < kvm_debugfs_num_entries; i++)
624 kfree(kvm->debugfs_stat_data[i]);
625 kfree(kvm->debugfs_stat_data);
626 }
Janosch Frank536a6f82016-05-18 13:26:23 +0200627}
628
629static int kvm_create_vm_debugfs(struct kvm *kvm, int fd)
630{
631 char dir_name[ITOA_MAX_LEN * 2];
632 struct kvm_stat_data *stat_data;
633 struct kvm_stats_debugfs_item *p;
634
635 if (!debugfs_initialized())
636 return 0;
637
638 snprintf(dir_name, sizeof(dir_name), "%d-%d", task_pid_nr(current), fd);
Greg Kroah-Hartman929f45e2018-05-29 18:22:04 +0200639 kvm->debugfs_dentry = debugfs_create_dir(dir_name, kvm_debugfs_dir);
Janosch Frank536a6f82016-05-18 13:26:23 +0200640
641 kvm->debugfs_stat_data = kcalloc(kvm_debugfs_num_entries,
642 sizeof(*kvm->debugfs_stat_data),
Ben Gardonb12ce362019-02-11 11:02:49 -0800643 GFP_KERNEL_ACCOUNT);
Janosch Frank536a6f82016-05-18 13:26:23 +0200644 if (!kvm->debugfs_stat_data)
645 return -ENOMEM;
646
647 for (p = debugfs_entries; p->name; p++) {
Ben Gardonb12ce362019-02-11 11:02:49 -0800648 stat_data = kzalloc(sizeof(*stat_data), GFP_KERNEL_ACCOUNT);
Janosch Frank536a6f82016-05-18 13:26:23 +0200649 if (!stat_data)
650 return -ENOMEM;
651
652 stat_data->kvm = kvm;
653 stat_data->offset = p->offset;
Paolo Bonzini833b45d2019-09-30 18:48:44 +0200654 stat_data->mode = p->mode ? p->mode : 0644;
Janosch Frank536a6f82016-05-18 13:26:23 +0200655 kvm->debugfs_stat_data[p - debugfs_entries] = stat_data;
Paolo Bonzini833b45d2019-09-30 18:48:44 +0200656 debugfs_create_file(p->name, stat_data->mode, kvm->debugfs_dentry,
Greg Kroah-Hartman929f45e2018-05-29 18:22:04 +0200657 stat_data, stat_fops_per_vm[p->kind]);
Janosch Frank536a6f82016-05-18 13:26:23 +0200658 }
659 return 0;
660}
661
Junaid Shahid1aa9b952019-11-04 20:26:00 +0100662/*
663 * Called after the VM is otherwise initialized, but just before adding it to
664 * the vm_list.
665 */
666int __weak kvm_arch_post_init_vm(struct kvm *kvm)
667{
668 return 0;
669}
670
671/*
672 * Called just after removing the VM from the vm_list, but before doing any
673 * other destruction.
674 */
675void __weak kvm_arch_pre_destroy_vm(struct kvm *kvm)
676{
677}
678
Carsten Ottee08b9632012-01-04 10:25:20 +0100679static struct kvm *kvm_create_vm(unsigned long type)
Avi Kivity6aa8b732006-12-10 02:21:36 -0800680{
Jan Kiszkad89f5ef2010-11-09 17:02:49 +0100681 struct kvm *kvm = kvm_arch_alloc_vm();
Jim Mattson91219232019-10-24 16:03:26 -0700682 int r = -ENOMEM;
683 int i;
Avi Kivity6aa8b732006-12-10 02:21:36 -0800684
Jan Kiszkad89f5ef2010-11-09 17:02:49 +0100685 if (!kvm)
686 return ERR_PTR(-ENOMEM);
687
Paolo Bonzinie9ad4ec2016-03-21 10:15:25 +0100688 spin_lock_init(&kvm->mmu_lock);
Vegard Nossumf1f10072017-02-27 14:30:07 -0800689 mmgrab(current->mm);
Paolo Bonzinie9ad4ec2016-03-21 10:15:25 +0100690 kvm->mm = current->mm;
691 kvm_eventfd_init(kvm);
692 mutex_init(&kvm->lock);
693 mutex_init(&kvm->irq_lock);
694 mutex_init(&kvm->slots_lock);
Paolo Bonzinie9ad4ec2016-03-21 10:15:25 +0100695 INIT_LIST_HEAD(&kvm->devices);
696
Jim Mattson91219232019-10-24 16:03:26 -0700697 BUILD_BUG_ON(KVM_MEM_SLOTS_NUM > SHRT_MAX);
698
Paolo Bonzini8a44119a2019-11-04 12:16:49 +0100699 if (init_srcu_struct(&kvm->srcu))
700 goto out_err_no_srcu;
701 if (init_srcu_struct(&kvm->irq_srcu))
702 goto out_err_no_irq_srcu;
703
Paolo Bonzinie2d3fca2019-11-04 13:23:53 +0100704 refcount_set(&kvm->users_count, 1);
Jim Mattson91219232019-10-24 16:03:26 -0700705 for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) {
706 struct kvm_memslots *slots = kvm_alloc_memslots();
707
708 if (!slots)
Jim Mattsona97b0e72019-10-25 13:34:58 +0200709 goto out_err_no_arch_destroy_vm;
Jim Mattson91219232019-10-24 16:03:26 -0700710 /* Generations must be different for each address space. */
711 slots->generation = i;
712 rcu_assign_pointer(kvm->memslots[i], slots);
713 }
714
715 for (i = 0; i < KVM_NR_BUSES; i++) {
716 rcu_assign_pointer(kvm->buses[i],
717 kzalloc(sizeof(struct kvm_io_bus), GFP_KERNEL_ACCOUNT));
718 if (!kvm->buses[i])
Jim Mattsona97b0e72019-10-25 13:34:58 +0200719 goto out_err_no_arch_destroy_vm;
Jim Mattson91219232019-10-24 16:03:26 -0700720 }
721
Carsten Ottee08b9632012-01-04 10:25:20 +0100722 r = kvm_arch_init_vm(kvm, type);
Jan Kiszkad89f5ef2010-11-09 17:02:49 +0100723 if (r)
Jim Mattsona97b0e72019-10-25 13:34:58 +0200724 goto out_err_no_arch_destroy_vm;
Alexander Graf10474ae2009-09-15 11:37:46 +0200725
726 r = hardware_enable_all();
727 if (r)
Christian Borntraeger719d93c2014-01-16 13:44:20 +0100728 goto out_err_no_disable;
Alexander Graf10474ae2009-09-15 11:37:46 +0200729
Paolo Bonzinic77dcac2014-08-06 14:24:45 +0200730#ifdef CONFIG_HAVE_KVM_IRQFD
Gleb Natapov136bdfe2009-08-24 11:54:23 +0300731 INIT_HLIST_HEAD(&kvm->irq_ack_notifier_list);
Avi Kivity75858a82009-01-04 17:10:50 +0200732#endif
Avi Kivity6aa8b732006-12-10 02:21:36 -0800733
Mike Waychison74b5c5b2011-06-03 13:04:53 -0700734 r = kvm_init_mmu_notifier(kvm);
735 if (r)
Junaid Shahid1aa9b952019-11-04 20:26:00 +0100736 goto out_err_no_mmu_notifier;
737
738 r = kvm_arch_post_init_vm(kvm);
739 if (r)
Mike Waychison74b5c5b2011-06-03 13:04:53 -0700740 goto out_err;
741
Junaid Shahid0d9ce162019-01-03 17:14:28 -0800742 mutex_lock(&kvm_lock);
Rusty Russell5e58cfe2007-07-23 17:08:21 +1000743 list_add(&kvm->vm_list, &vm_list);
Junaid Shahid0d9ce162019-01-03 17:14:28 -0800744 mutex_unlock(&kvm_lock);
Jan Kiszkad89f5ef2010-11-09 17:02:49 +0100745
Peter Zijlstra2ecd9d22015-07-03 18:53:58 +0200746 preempt_notifier_inc();
747
Avi Kivityf17abe92007-02-21 19:28:04 +0200748 return kvm;
Alexander Graf10474ae2009-09-15 11:37:46 +0200749
750out_err:
Junaid Shahid1aa9b952019-11-04 20:26:00 +0100751#if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER)
752 if (kvm->mmu_notifier.ops)
753 mmu_notifier_unregister(&kvm->mmu_notifier, current->mm);
754#endif
755out_err_no_mmu_notifier:
Alexander Graf10474ae2009-09-15 11:37:46 +0200756 hardware_disable_all();
Christian Borntraeger719d93c2014-01-16 13:44:20 +0100757out_err_no_disable:
Jim Mattsona97b0e72019-10-25 13:34:58 +0200758 kvm_arch_destroy_vm(kvm);
Jim Mattsona97b0e72019-10-25 13:34:58 +0200759out_err_no_arch_destroy_vm:
Paolo Bonzinie2d3fca2019-11-04 13:23:53 +0100760 WARN_ON_ONCE(!refcount_dec_and_test(&kvm->users_count));
Marcelo Tosattie93f8a02009-12-23 14:35:24 -0200761 for (i = 0; i < KVM_NR_BUSES; i++)
Paolo Bonzini3898da92017-08-02 17:55:54 +0200762 kfree(kvm_get_bus(kvm, i));
Paolo Bonzinif481b062015-05-17 17:30:37 +0200763 for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++)
Paolo Bonzini3898da92017-08-02 17:55:54 +0200764 kvm_free_memslots(kvm, __kvm_memslots(kvm, i));
Paolo Bonzini8a44119a2019-11-04 12:16:49 +0100765 cleanup_srcu_struct(&kvm->irq_srcu);
766out_err_no_irq_srcu:
767 cleanup_srcu_struct(&kvm->srcu);
768out_err_no_srcu:
Jan Kiszkad89f5ef2010-11-09 17:02:49 +0100769 kvm_arch_free_vm(kvm);
Paolo Bonzinie9ad4ec2016-03-21 10:15:25 +0100770 mmdrop(current->mm);
Alexander Graf10474ae2009-09-15 11:37:46 +0200771 return ERR_PTR(r);
Avi Kivityf17abe92007-02-21 19:28:04 +0200772}
773
Scott Wood07f0a7b2013-04-25 14:11:23 +0000774static void kvm_destroy_devices(struct kvm *kvm)
775{
Geliang Tange6e3b5a2016-01-01 19:47:12 +0800776 struct kvm_device *dev, *tmp;
Scott Wood07f0a7b2013-04-25 14:11:23 +0000777
Christoffer Dalla28ebea2016-08-09 19:13:01 +0200778 /*
779 * We do not need to take the kvm->lock here, because nobody else
780 * has a reference to the struct kvm at this point and therefore
781 * cannot access the devices list anyhow.
782 */
Geliang Tange6e3b5a2016-01-01 19:47:12 +0800783 list_for_each_entry_safe(dev, tmp, &kvm->devices, vm_node) {
784 list_del(&dev->vm_node);
Scott Wood07f0a7b2013-04-25 14:11:23 +0000785 dev->ops->destroy(dev);
786 }
787}
788
Avi Kivityf17abe92007-02-21 19:28:04 +0200789static void kvm_destroy_vm(struct kvm *kvm)
790{
Marcelo Tosattie93f8a02009-12-23 14:35:24 -0200791 int i;
Avi Kivity6d4e4c42007-11-21 16:41:05 +0200792 struct mm_struct *mm = kvm->mm;
793
Claudio Imbrenda286de8f2017-07-12 17:56:44 +0200794 kvm_uevent_notify_change(KVM_EVENT_DESTROY_VM, kvm);
Janosch Frank536a6f82016-05-18 13:26:23 +0200795 kvm_destroy_vm_debugfs(kvm);
Sheng Yangad8ba2c2009-01-06 10:03:02 +0800796 kvm_arch_sync_events(kvm);
Junaid Shahid0d9ce162019-01-03 17:14:28 -0800797 mutex_lock(&kvm_lock);
Avi Kivity133de902007-02-12 00:54:44 -0800798 list_del(&kvm->vm_list);
Junaid Shahid0d9ce162019-01-03 17:14:28 -0800799 mutex_unlock(&kvm_lock);
Junaid Shahid1aa9b952019-11-04 20:26:00 +0100800 kvm_arch_pre_destroy_vm(kvm);
801
Avi Kivity399ec802008-11-19 13:58:46 +0200802 kvm_free_irq_routing(kvm);
Peter Xudf630b82017-03-15 16:01:17 +0800803 for (i = 0; i < KVM_NR_BUSES; i++) {
Paolo Bonzini3898da92017-08-02 17:55:54 +0200804 struct kvm_io_bus *bus = kvm_get_bus(kvm, i);
Christian Borntraeger4a12f952017-07-07 10:51:38 +0200805
Christian Borntraeger4a12f952017-07-07 10:51:38 +0200806 if (bus)
807 kvm_io_bus_destroy(bus);
Peter Xudf630b82017-03-15 16:01:17 +0800808 kvm->buses[i] = NULL;
809 }
Avi Kivity980da6c2009-12-20 15:13:43 +0200810 kvm_coalesced_mmio_free(kvm);
Andrea Arcangelie930bff2008-07-25 16:24:52 +0200811#if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER)
812 mmu_notifier_unregister(&kvm->mmu_notifier, kvm->mm);
Gleb Natapovf00be0c2009-03-19 12:20:36 +0200813#else
Marcelo Tosatti2df72e92012-08-24 15:54:57 -0300814 kvm_arch_flush_shadow_all(kvm);
Andrea Arcangelie930bff2008-07-25 16:24:52 +0200815#endif
Zhang Xiantaod19a9cd2007-11-18 18:43:45 +0800816 kvm_arch_destroy_vm(kvm);
Scott Wood07f0a7b2013-04-25 14:11:23 +0000817 kvm_destroy_devices(kvm);
Paolo Bonzinif481b062015-05-17 17:30:37 +0200818 for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++)
Paolo Bonzini3898da92017-08-02 17:55:54 +0200819 kvm_free_memslots(kvm, __kvm_memslots(kvm, i));
Paolo Bonzini820b3fc2014-06-03 13:44:17 +0200820 cleanup_srcu_struct(&kvm->irq_srcu);
Jan Kiszkad89f5ef2010-11-09 17:02:49 +0100821 cleanup_srcu_struct(&kvm->srcu);
822 kvm_arch_free_vm(kvm);
Peter Zijlstra2ecd9d22015-07-03 18:53:58 +0200823 preempt_notifier_dec();
Alexander Graf10474ae2009-09-15 11:37:46 +0200824 hardware_disable_all();
Avi Kivity6d4e4c42007-11-21 16:41:05 +0200825 mmdrop(mm);
Avi Kivityf17abe92007-02-21 19:28:04 +0200826}
827
Izik Eidusd39f13b2008-03-30 16:01:25 +0300828void kvm_get_kvm(struct kvm *kvm)
829{
Elena Reshetovae3736c32017-02-20 13:06:21 +0200830 refcount_inc(&kvm->users_count);
Izik Eidusd39f13b2008-03-30 16:01:25 +0300831}
832EXPORT_SYMBOL_GPL(kvm_get_kvm);
833
834void kvm_put_kvm(struct kvm *kvm)
835{
Elena Reshetovae3736c32017-02-20 13:06:21 +0200836 if (refcount_dec_and_test(&kvm->users_count))
Izik Eidusd39f13b2008-03-30 16:01:25 +0300837 kvm_destroy_vm(kvm);
838}
839EXPORT_SYMBOL_GPL(kvm_put_kvm);
840
Sean Christopherson149487b2019-10-21 15:58:42 -0700841/*
842 * Used to put a reference that was taken on behalf of an object associated
843 * with a user-visible file descriptor, e.g. a vcpu or device, if installation
844 * of the new file descriptor fails and the reference cannot be transferred to
845 * its final owner. In such cases, the caller is still actively using @kvm and
846 * will fail miserably if the refcount unexpectedly hits zero.
847 */
848void kvm_put_kvm_no_destroy(struct kvm *kvm)
849{
850 WARN_ON(refcount_dec_and_test(&kvm->users_count));
851}
852EXPORT_SYMBOL_GPL(kvm_put_kvm_no_destroy);
Izik Eidusd39f13b2008-03-30 16:01:25 +0300853
Avi Kivityf17abe92007-02-21 19:28:04 +0200854static int kvm_vm_release(struct inode *inode, struct file *filp)
855{
856 struct kvm *kvm = filp->private_data;
857
Gregory Haskins721eecbf2009-05-20 10:30:49 -0400858 kvm_irqfd_release(kvm);
859
Izik Eidusd39f13b2008-03-30 16:01:25 +0300860 kvm_put_kvm(kvm);
Avi Kivity6aa8b732006-12-10 02:21:36 -0800861 return 0;
862}
863
Takuya Yoshikawa515a0122010-10-27 18:23:54 +0900864/*
865 * Allocation size is twice as large as the actual dirty bitmap size.
Takuya Yoshikawa93474b22012-03-01 19:34:45 +0900866 * See x86's kvm_vm_ioctl_get_dirty_log() why this is needed.
Takuya Yoshikawa515a0122010-10-27 18:23:54 +0900867 */
Takuya Yoshikawaa36a57b12010-10-27 18:22:19 +0900868static int kvm_create_dirty_bitmap(struct kvm_memory_slot *memslot)
869{
Takuya Yoshikawa515a0122010-10-27 18:23:54 +0900870 unsigned long dirty_bytes = 2 * kvm_dirty_bitmap_bytes(memslot);
Takuya Yoshikawaa36a57b12010-10-27 18:22:19 +0900871
Ben Gardonb12ce362019-02-11 11:02:49 -0800872 memslot->dirty_bitmap = kvzalloc(dirty_bytes, GFP_KERNEL_ACCOUNT);
Takuya Yoshikawaa36a57b12010-10-27 18:22:19 +0900873 if (!memslot->dirty_bitmap)
874 return -ENOMEM;
875
Takuya Yoshikawaa36a57b12010-10-27 18:22:19 +0900876 return 0;
877}
878
Xiao Guangrongbf3e05b2011-11-24 17:40:57 +0800879/*
Igor Mammedov0e60b072014-12-01 17:29:26 +0000880 * Insert memslot and re-sort memslots based on their GFN,
881 * so binary search could be used to lookup GFN.
882 * Sorting algorithm takes advantage of having initially
883 * sorted array and known changed memslot position.
Xiao Guangrongbf3e05b2011-11-24 17:40:57 +0800884 */
Paolo Bonzini5cc15022014-11-14 10:55:31 +0100885static void update_memslots(struct kvm_memslots *slots,
Wei Yang31fc4f92018-08-22 21:57:11 +0800886 struct kvm_memory_slot *new,
887 enum kvm_mr_change change)
Xiao Guangrongbf3e05b2011-11-24 17:40:57 +0800888{
Paolo Bonzini85931762014-11-14 10:22:07 +0100889 int id = new->id;
890 int i = slots->id_to_index[id];
Igor Mammedov063584d2014-11-13 23:00:13 +0000891 struct kvm_memory_slot *mslots = slots->memslots;
Xiao Guangrongf85e2cb2011-11-24 17:41:54 +0800892
Paolo Bonzini85931762014-11-14 10:22:07 +0100893 WARN_ON(mslots[i].id != id);
Wei Yang31fc4f92018-08-22 21:57:11 +0800894 switch (change) {
895 case KVM_MR_CREATE:
896 slots->used_slots++;
897 WARN_ON(mslots[i].npages || !new->npages);
898 break;
899 case KVM_MR_DELETE:
900 slots->used_slots--;
901 WARN_ON(new->npages || !mslots[i].npages);
902 break;
903 default:
904 break;
Igor Mammedov9c1a5d382014-12-01 17:29:27 +0000905 }
Igor Mammedov0e60b072014-12-01 17:29:26 +0000906
Igor Mammedov7f379cf2014-12-01 17:29:24 +0000907 while (i < KVM_MEM_SLOTS_NUM - 1 &&
Igor Mammedov0e60b072014-12-01 17:29:26 +0000908 new->base_gfn <= mslots[i + 1].base_gfn) {
909 if (!mslots[i + 1].npages)
910 break;
Igor Mammedov7f379cf2014-12-01 17:29:24 +0000911 mslots[i] = mslots[i + 1];
912 slots->id_to_index[mslots[i].id] = i;
913 i++;
914 }
Paolo Bonziniefbeec72014-12-27 18:01:00 +0100915
916 /*
917 * The ">=" is needed when creating a slot with base_gfn == 0,
918 * so that it moves before all those with base_gfn == npages == 0.
919 *
920 * On the other hand, if new->npages is zero, the above loop has
921 * already left i pointing to the beginning of the empty part of
922 * mslots, and the ">=" would move the hole backwards in this
923 * case---which is wrong. So skip the loop when deleting a slot.
924 */
925 if (new->npages) {
926 while (i > 0 &&
927 new->base_gfn >= mslots[i - 1].base_gfn) {
928 mslots[i] = mslots[i - 1];
929 slots->id_to_index[mslots[i].id] = i;
930 i--;
931 }
Paolo Bonzinidbaff302014-12-27 21:08:16 +0100932 } else
933 WARN_ON_ONCE(i != slots->used_slots);
Xiao Guangrongf85e2cb2011-11-24 17:41:54 +0800934
Paolo Bonzini85931762014-11-14 10:22:07 +0100935 mslots[i] = *new;
936 slots->id_to_index[mslots[i].id] = i;
Xiao Guangrongbf3e05b2011-11-24 17:40:57 +0800937}
938
Paolo Bonzini09170a42015-05-18 13:59:39 +0200939static int check_memory_region_flags(const struct kvm_userspace_memory_region *mem)
Xiao Guangronga50d64d2012-08-21 10:58:13 +0800940{
Xiao Guangrong4d8b81a2012-08-21 11:02:51 +0800941 u32 valid_flags = KVM_MEM_LOG_DIRTY_PAGES;
942
Christoffer Dall0f8a4de2014-08-26 14:00:37 +0200943#ifdef __KVM_HAVE_READONLY_MEM
Xiao Guangrong4d8b81a2012-08-21 11:02:51 +0800944 valid_flags |= KVM_MEM_READONLY;
945#endif
946
947 if (mem->flags & ~valid_flags)
Xiao Guangronga50d64d2012-08-21 10:58:13 +0800948 return -EINVAL;
949
950 return 0;
951}
952
Gleb Natapov7ec4fb42012-12-24 17:49:30 +0200953static struct kvm_memslots *install_new_memslots(struct kvm *kvm,
Paolo Bonzinif481b062015-05-17 17:30:37 +0200954 int as_id, struct kvm_memslots *slots)
Gleb Natapov7ec4fb42012-12-24 17:49:30 +0200955{
Paolo Bonzinif481b062015-05-17 17:30:37 +0200956 struct kvm_memslots *old_memslots = __kvm_memslots(kvm, as_id);
Sean Christopherson361209e2019-02-05 13:01:14 -0800957 u64 gen = old_memslots->generation;
Gleb Natapov7ec4fb42012-12-24 17:49:30 +0200958
Sean Christopherson361209e2019-02-05 13:01:14 -0800959 WARN_ON(gen & KVM_MEMSLOT_GEN_UPDATE_IN_PROGRESS);
960 slots->generation = gen | KVM_MEMSLOT_GEN_UPDATE_IN_PROGRESS;
David Matlackee3d1572014-08-18 15:46:06 -0700961
Paolo Bonzinif481b062015-05-17 17:30:37 +0200962 rcu_assign_pointer(kvm->memslots[as_id], slots);
Gleb Natapov7ec4fb42012-12-24 17:49:30 +0200963 synchronize_srcu_expedited(&kvm->srcu);
Takuya Yoshikawae59dbe02013-07-04 13:40:29 +0900964
David Matlackee3d1572014-08-18 15:46:06 -0700965 /*
Sean Christopherson361209e2019-02-05 13:01:14 -0800966 * Increment the new memslot generation a second time, dropping the
967 * update in-progress flag and incrementing then generation based on
968 * the number of address spaces. This provides a unique and easily
969 * identifiable generation number while the memslots are in flux.
970 */
971 gen = slots->generation & ~KVM_MEMSLOT_GEN_UPDATE_IN_PROGRESS;
972
973 /*
Paolo Bonzini4bd518f2017-02-03 20:44:51 -0800974 * Generations must be unique even across address spaces. We do not need
975 * a global counter for that, instead the generation space is evenly split
976 * across address spaces. For example, with two address spaces, address
Sean Christopherson164bf7e2019-02-05 13:01:18 -0800977 * space 0 will use generations 0, 2, 4, ... while address space 1 will
978 * use generations 1, 3, 5, ...
David Matlackee3d1572014-08-18 15:46:06 -0700979 */
Sean Christopherson164bf7e2019-02-05 13:01:18 -0800980 gen += KVM_ADDRESS_SPACE_NUM;
David Matlackee3d1572014-08-18 15:46:06 -0700981
Sean Christopherson15248252019-02-05 12:54:17 -0800982 kvm_arch_memslots_updated(kvm, gen);
983
984 slots->generation = gen;
Takuya Yoshikawae59dbe02013-07-04 13:40:29 +0900985
986 return old_memslots;
Gleb Natapov7ec4fb42012-12-24 17:49:30 +0200987}
988
Avi Kivity6aa8b732006-12-10 02:21:36 -0800989/*
Avi Kivity6aa8b732006-12-10 02:21:36 -0800990 * Allocate some memory and give it an address in the guest physical address
991 * space.
992 *
993 * Discontiguous memory is allowed, mostly for framebuffers.
Sheng Yangf78e0e22007-10-29 09:40:42 +0800994 *
Dominik Dingel02d5d552014-10-27 16:22:56 +0100995 * Must be called holding kvm->slots_lock for write.
Avi Kivity6aa8b732006-12-10 02:21:36 -0800996 */
Sheng Yangf78e0e22007-10-29 09:40:42 +0800997int __kvm_set_memory_region(struct kvm *kvm,
Paolo Bonzini09170a42015-05-18 13:59:39 +0200998 const struct kvm_userspace_memory_region *mem)
Avi Kivity6aa8b732006-12-10 02:21:36 -0800999{
Avi Kivity8234b222010-12-27 12:08:45 +02001000 int r;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001001 gfn_t base_gfn;
Heiko Carstens28bcb112009-09-03 17:35:35 +02001002 unsigned long npages;
Takuya Yoshikawaa843fac2013-01-11 18:27:43 +09001003 struct kvm_memory_slot *slot;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001004 struct kvm_memory_slot old, new;
Alex Williamsonb7f69c52012-12-10 10:33:03 -07001005 struct kvm_memslots *slots = NULL, *old_memslots;
Paolo Bonzinif481b062015-05-17 17:30:37 +02001006 int as_id, id;
Takuya Yoshikawaf64c0392013-01-29 11:00:07 +09001007 enum kvm_mr_change change;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001008
Xiao Guangronga50d64d2012-08-21 10:58:13 +08001009 r = check_memory_region_flags(mem);
1010 if (r)
1011 goto out;
1012
Avi Kivity6aa8b732006-12-10 02:21:36 -08001013 r = -EINVAL;
Paolo Bonzinif481b062015-05-17 17:30:37 +02001014 as_id = mem->slot >> 16;
1015 id = (u16)mem->slot;
1016
Avi Kivity6aa8b732006-12-10 02:21:36 -08001017 /* General sanity checks */
1018 if (mem->memory_size & (PAGE_SIZE - 1))
1019 goto out;
1020 if (mem->guest_phys_addr & (PAGE_SIZE - 1))
1021 goto out;
Takuya Yoshikawafa3d3152011-05-07 16:35:38 +09001022 /* We can read the guest memory with __xxx_user() later on. */
Paolo Bonzinif481b062015-05-17 17:30:37 +02001023 if ((id < KVM_USER_MEM_SLOTS) &&
Takuya Yoshikawafa3d3152011-05-07 16:35:38 +09001024 ((mem->userspace_addr & (PAGE_SIZE - 1)) ||
Linus Torvalds96d4f262019-01-03 18:57:57 -08001025 !access_ok((void __user *)(unsigned long)mem->userspace_addr,
Heiko Carstens9e3bb6b2011-05-24 07:51:27 +02001026 mem->memory_size)))
Hollis Blanchard78749802008-11-07 13:32:12 -06001027 goto out;
Paolo Bonzinif481b062015-05-17 17:30:37 +02001028 if (as_id >= KVM_ADDRESS_SPACE_NUM || id >= KVM_MEM_SLOTS_NUM)
Avi Kivity6aa8b732006-12-10 02:21:36 -08001029 goto out;
1030 if (mem->guest_phys_addr + mem->memory_size < mem->guest_phys_addr)
1031 goto out;
1032
Paolo Bonzinif481b062015-05-17 17:30:37 +02001033 slot = id_to_memslot(__kvm_memslots(kvm, as_id), id);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001034 base_gfn = mem->guest_phys_addr >> PAGE_SHIFT;
1035 npages = mem->memory_size >> PAGE_SHIFT;
1036
Takuya Yoshikawa660c22c2010-04-13 22:47:24 +09001037 if (npages > KVM_MEM_MAX_NR_PAGES)
1038 goto out;
1039
Takuya Yoshikawaa843fac2013-01-11 18:27:43 +09001040 new = old = *slot;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001041
Paolo Bonzinif481b062015-05-17 17:30:37 +02001042 new.id = id;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001043 new.base_gfn = base_gfn;
1044 new.npages = npages;
1045 new.flags = mem->flags;
1046
Takuya Yoshikawaf64c0392013-01-29 11:00:07 +09001047 if (npages) {
1048 if (!old.npages)
1049 change = KVM_MR_CREATE;
1050 else { /* Modify an existing slot. */
1051 if ((mem->userspace_addr != old.userspace_addr) ||
Takuya Yoshikawa75d61fb2013-01-30 19:40:41 +09001052 (npages != old.npages) ||
1053 ((new.flags ^ old.flags) & KVM_MEM_READONLY))
Takuya Yoshikawaf64c0392013-01-29 11:00:07 +09001054 goto out;
1055
1056 if (base_gfn != old.base_gfn)
1057 change = KVM_MR_MOVE;
1058 else if (new.flags != old.flags)
1059 change = KVM_MR_FLAGS_ONLY;
1060 else { /* Nothing to change. */
1061 r = 0;
1062 goto out;
1063 }
1064 }
Paolo Bonzini09170a42015-05-18 13:59:39 +02001065 } else {
1066 if (!old.npages)
1067 goto out;
1068
Takuya Yoshikawaf64c0392013-01-29 11:00:07 +09001069 change = KVM_MR_DELETE;
Paolo Bonzini09170a42015-05-18 13:59:39 +02001070 new.base_gfn = 0;
1071 new.flags = 0;
1072 }
Avi Kivity6aa8b732006-12-10 02:21:36 -08001073
Takuya Yoshikawaf64c0392013-01-29 11:00:07 +09001074 if ((change == KVM_MR_CREATE) || (change == KVM_MR_MOVE)) {
Takuya Yoshikawa0a706be2013-01-11 18:26:55 +09001075 /* Check for overlaps */
1076 r = -EEXIST;
Paolo Bonzinif481b062015-05-17 17:30:37 +02001077 kvm_for_each_memslot(slot, __kvm_memslots(kvm, as_id)) {
Wanpeng Lib28676b2018-02-13 15:36:00 +01001078 if (slot->id == id)
Takuya Yoshikawa0a706be2013-01-11 18:26:55 +09001079 continue;
1080 if (!((base_gfn + npages <= slot->base_gfn) ||
1081 (base_gfn >= slot->base_gfn + slot->npages)))
1082 goto out;
1083 }
Avi Kivity6aa8b732006-12-10 02:21:36 -08001084 }
Avi Kivity6aa8b732006-12-10 02:21:36 -08001085
Avi Kivity6aa8b732006-12-10 02:21:36 -08001086 /* Free page dirty bitmap if unneeded */
1087 if (!(new.flags & KVM_MEM_LOG_DIRTY_PAGES))
Al Viro8b6d44c2007-02-09 16:38:40 +00001088 new.dirty_bitmap = NULL;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001089
1090 r = -ENOMEM;
Takuya Yoshikawaf64c0392013-01-29 11:00:07 +09001091 if (change == KVM_MR_CREATE) {
Takuya Yoshikawa189a2f72012-02-08 13:01:09 +09001092 new.userspace_addr = mem->userspace_addr;
Takuya Yoshikawad89cc612012-08-01 18:03:28 +09001093
Aneesh Kumar K.V55870272013-10-07 22:18:00 +05301094 if (kvm_arch_create_memslot(kvm, &new, npages))
Takuya Yoshikawadb3fe4e2012-02-08 13:02:18 +09001095 goto out_free;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001096 }
Joerg Roedelec04b262009-06-19 15:16:23 +02001097
Avi Kivity6aa8b732006-12-10 02:21:36 -08001098 /* Allocate page dirty bitmap if needed */
1099 if ((new.flags & KVM_MEM_LOG_DIRTY_PAGES) && !new.dirty_bitmap) {
Takuya Yoshikawaa36a57b12010-10-27 18:22:19 +09001100 if (kvm_create_dirty_bitmap(&new) < 0)
Sheng Yangf78e0e22007-10-29 09:40:42 +08001101 goto out_free;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001102 }
1103
Ben Gardonb12ce362019-02-11 11:02:49 -08001104 slots = kvzalloc(sizeof(struct kvm_memslots), GFP_KERNEL_ACCOUNT);
Paolo Bonzinif2a81032014-11-14 10:46:45 +01001105 if (!slots)
1106 goto out_free;
Paolo Bonzinif481b062015-05-17 17:30:37 +02001107 memcpy(slots, __kvm_memslots(kvm, as_id), sizeof(struct kvm_memslots));
Paolo Bonzinif2a81032014-11-14 10:46:45 +01001108
Takuya Yoshikawaf64c0392013-01-29 11:00:07 +09001109 if ((change == KVM_MR_DELETE) || (change == KVM_MR_MOVE)) {
Paolo Bonzinif481b062015-05-17 17:30:37 +02001110 slot = id_to_memslot(slots, id);
Xiao Guangrong28a37542011-11-24 19:04:35 +08001111 slot->flags |= KVM_MEMSLOT_INVALID;
1112
Paolo Bonzinif481b062015-05-17 17:30:37 +02001113 old_memslots = install_new_memslots(kvm, as_id, slots);
Marcelo Tosattibc6678a2009-12-23 14:35:21 -02001114
Marcelo Tosatti12d6e752012-08-24 15:54:58 -03001115 /* From this point no new shadow pages pointing to a deleted,
1116 * or moved, memslot will be created.
Marcelo Tosattibc6678a2009-12-23 14:35:21 -02001117 *
1118 * validation of sp->gfn happens in:
Xiubo Lib7d409d2015-02-26 14:58:24 +08001119 * - gfn_to_hva (kvm_read_guest, gfn_to_pfn)
1120 * - kvm_is_visible_gfn (mmu_check_roots)
Marcelo Tosattibc6678a2009-12-23 14:35:21 -02001121 */
Marcelo Tosatti2df72e92012-08-24 15:54:57 -03001122 kvm_arch_flush_shadow_memslot(kvm, slot);
Paolo Bonzinif2a81032014-11-14 10:46:45 +01001123
1124 /*
1125 * We can re-use the old_memslots from above, the only difference
1126 * from the currently installed memslots is the invalid flag. This
1127 * will get overwritten by update_memslots anyway.
1128 */
Alex Williamsonb7f69c52012-12-10 10:33:03 -07001129 slots = old_memslots;
Marcelo Tosattibc6678a2009-12-23 14:35:21 -02001130 }
Marcelo Tosatti34d4cb82008-07-10 20:49:31 -03001131
Takuya Yoshikawa7b6195a2013-02-27 19:44:34 +09001132 r = kvm_arch_prepare_memory_region(kvm, &new, mem, change);
Marcelo Tosattif7784b82009-12-23 14:35:18 -02001133 if (r)
Alex Williamsonb7f69c52012-12-10 10:33:03 -07001134 goto out_slots;
Marcelo Tosattif7784b82009-12-23 14:35:18 -02001135
Paolo Bonzinia47d2b02015-05-17 11:41:37 +02001136 /* actual memory is freed via old in kvm_free_memslot below */
Takuya Yoshikawaf64c0392013-01-29 11:00:07 +09001137 if (change == KVM_MR_DELETE) {
Marcelo Tosattibc6678a2009-12-23 14:35:21 -02001138 new.dirty_bitmap = NULL;
Takuya Yoshikawadb3fe4e2012-02-08 13:02:18 +09001139 memset(&new.arch, 0, sizeof(new.arch));
Marcelo Tosattibc6678a2009-12-23 14:35:21 -02001140 }
1141
Wei Yang31fc4f92018-08-22 21:57:11 +08001142 update_memslots(slots, &new, change);
Paolo Bonzinif481b062015-05-17 17:30:37 +02001143 old_memslots = install_new_memslots(kvm, as_id, slots);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001144
Paolo Bonzinif36f3f22015-05-18 13:20:23 +02001145 kvm_arch_commit_memory_region(kvm, mem, &old, &new, change);
Zhang Xiantao3ad82a72007-11-20 13:11:38 +08001146
Paolo Bonzinia47d2b02015-05-17 11:41:37 +02001147 kvm_free_memslot(kvm, &old, &new);
Igor Mammedov74496132015-03-20 12:21:37 +00001148 kvfree(old_memslots);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001149 return 0;
1150
Alex Williamsone40f1932012-12-10 10:32:57 -07001151out_slots:
Igor Mammedov74496132015-03-20 12:21:37 +00001152 kvfree(slots);
Sheng Yangf78e0e22007-10-29 09:40:42 +08001153out_free:
Paolo Bonzinia47d2b02015-05-17 11:41:37 +02001154 kvm_free_memslot(kvm, &new, &old);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001155out:
1156 return r;
Izik Eidus210c7c42007-10-24 23:52:57 +02001157}
Sheng Yangf78e0e22007-10-29 09:40:42 +08001158EXPORT_SYMBOL_GPL(__kvm_set_memory_region);
1159
1160int kvm_set_memory_region(struct kvm *kvm,
Paolo Bonzini09170a42015-05-18 13:59:39 +02001161 const struct kvm_userspace_memory_region *mem)
Sheng Yangf78e0e22007-10-29 09:40:42 +08001162{
1163 int r;
1164
Marcelo Tosatti79fac952009-12-23 14:35:26 -02001165 mutex_lock(&kvm->slots_lock);
Takuya Yoshikawa47ae31e2013-02-27 19:43:00 +09001166 r = __kvm_set_memory_region(kvm, mem);
Marcelo Tosatti79fac952009-12-23 14:35:26 -02001167 mutex_unlock(&kvm->slots_lock);
Sheng Yangf78e0e22007-10-29 09:40:42 +08001168 return r;
1169}
Izik Eidus210c7c42007-10-24 23:52:57 +02001170EXPORT_SYMBOL_GPL(kvm_set_memory_region);
1171
Stephen Hemminger79408762013-12-29 12:12:29 -08001172static int kvm_vm_ioctl_set_memory_region(struct kvm *kvm,
1173 struct kvm_userspace_memory_region *mem)
Izik Eidus210c7c42007-10-24 23:52:57 +02001174{
Paolo Bonzinif481b062015-05-17 17:30:37 +02001175 if ((u16)mem->slot >= KVM_USER_MEM_SLOTS)
Izik Eiduse0d62c72007-10-24 23:57:46 +02001176 return -EINVAL;
Paolo Bonzini09170a42015-05-18 13:59:39 +02001177
Takuya Yoshikawa47ae31e2013-02-27 19:43:00 +09001178 return kvm_set_memory_region(kvm, mem);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001179}
1180
Zhang Xiantao5bb064d2007-11-18 20:29:43 +08001181int kvm_get_dirty_log(struct kvm *kvm,
1182 struct kvm_dirty_log *log, int *is_dirty)
Avi Kivity6aa8b732006-12-10 02:21:36 -08001183{
Paolo Bonzini9f6b8022015-05-17 16:20:07 +02001184 struct kvm_memslots *slots;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001185 struct kvm_memory_slot *memslot;
Markus Elfring843574a2017-01-22 17:41:07 +01001186 int i, as_id, id;
Takuya Yoshikawa87bf6e72010-04-12 19:35:35 +09001187 unsigned long n;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001188 unsigned long any = 0;
1189
Paolo Bonzinif481b062015-05-17 17:30:37 +02001190 as_id = log->slot >> 16;
1191 id = (u16)log->slot;
1192 if (as_id >= KVM_ADDRESS_SPACE_NUM || id >= KVM_USER_MEM_SLOTS)
Markus Elfring843574a2017-01-22 17:41:07 +01001193 return -EINVAL;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001194
Paolo Bonzinif481b062015-05-17 17:30:37 +02001195 slots = __kvm_memslots(kvm, as_id);
1196 memslot = id_to_memslot(slots, id);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001197 if (!memslot->dirty_bitmap)
Markus Elfring843574a2017-01-22 17:41:07 +01001198 return -ENOENT;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001199
Takuya Yoshikawa87bf6e72010-04-12 19:35:35 +09001200 n = kvm_dirty_bitmap_bytes(memslot);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001201
Uri Lublincd1a4a92007-02-22 16:43:09 +02001202 for (i = 0; !any && i < n/sizeof(long); ++i)
Avi Kivity6aa8b732006-12-10 02:21:36 -08001203 any = memslot->dirty_bitmap[i];
1204
Avi Kivity6aa8b732006-12-10 02:21:36 -08001205 if (copy_to_user(log->dirty_bitmap, memslot->dirty_bitmap, n))
Markus Elfring843574a2017-01-22 17:41:07 +01001206 return -EFAULT;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001207
Zhang Xiantao5bb064d2007-11-18 20:29:43 +08001208 if (any)
1209 *is_dirty = 1;
Markus Elfring843574a2017-01-22 17:41:07 +01001210 return 0;
Avi Kivity6aa8b732006-12-10 02:21:36 -08001211}
Aneesh Kumar K.V2ba9f0d2013-10-07 22:17:59 +05301212EXPORT_SYMBOL_GPL(kvm_get_dirty_log);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001213
Mario Smarduchba0513b2015-01-15 15:58:53 -08001214#ifdef CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT
1215/**
Jiang Biaob8b00222019-04-23 19:40:30 +08001216 * kvm_get_dirty_log_protect - get a snapshot of dirty pages
Paolo Bonzini2a31b9d2018-10-23 02:36:47 +02001217 * and reenable dirty page tracking for the corresponding pages.
Mario Smarduchba0513b2015-01-15 15:58:53 -08001218 * @kvm: pointer to kvm instance
1219 * @log: slot id and address to which we copy the log
Jiang Biaob8b00222019-04-23 19:40:30 +08001220 * @flush: true if TLB flush is needed by caller
Mario Smarduchba0513b2015-01-15 15:58:53 -08001221 *
1222 * We need to keep it in mind that VCPU threads can write to the bitmap
1223 * concurrently. So, to avoid losing track of dirty pages we keep the
1224 * following order:
1225 *
1226 * 1. Take a snapshot of the bit and clear it if needed.
1227 * 2. Write protect the corresponding page.
1228 * 3. Copy the snapshot to the userspace.
1229 * 4. Upon return caller flushes TLB's if needed.
1230 *
1231 * Between 2 and 4, the guest may write to the page using the remaining TLB
1232 * entry. This is not a problem because the page is reported dirty using
1233 * the snapshot taken before and step 4 ensures that writes done after
1234 * exiting to userspace will be logged for the next call.
1235 *
1236 */
1237int kvm_get_dirty_log_protect(struct kvm *kvm,
Paolo Bonzini8fe65a82018-10-23 02:18:42 +02001238 struct kvm_dirty_log *log, bool *flush)
Mario Smarduchba0513b2015-01-15 15:58:53 -08001239{
Paolo Bonzini9f6b8022015-05-17 16:20:07 +02001240 struct kvm_memslots *slots;
Mario Smarduchba0513b2015-01-15 15:58:53 -08001241 struct kvm_memory_slot *memslot;
Markus Elfring58d6db32017-01-22 17:30:16 +01001242 int i, as_id, id;
Mario Smarduchba0513b2015-01-15 15:58:53 -08001243 unsigned long n;
1244 unsigned long *dirty_bitmap;
1245 unsigned long *dirty_bitmap_buffer;
1246
Paolo Bonzinif481b062015-05-17 17:30:37 +02001247 as_id = log->slot >> 16;
1248 id = (u16)log->slot;
1249 if (as_id >= KVM_ADDRESS_SPACE_NUM || id >= KVM_USER_MEM_SLOTS)
Markus Elfring58d6db32017-01-22 17:30:16 +01001250 return -EINVAL;
Mario Smarduchba0513b2015-01-15 15:58:53 -08001251
Paolo Bonzinif481b062015-05-17 17:30:37 +02001252 slots = __kvm_memslots(kvm, as_id);
1253 memslot = id_to_memslot(slots, id);
Mario Smarduchba0513b2015-01-15 15:58:53 -08001254
1255 dirty_bitmap = memslot->dirty_bitmap;
Mario Smarduchba0513b2015-01-15 15:58:53 -08001256 if (!dirty_bitmap)
Markus Elfring58d6db32017-01-22 17:30:16 +01001257 return -ENOENT;
Mario Smarduchba0513b2015-01-15 15:58:53 -08001258
1259 n = kvm_dirty_bitmap_bytes(memslot);
Paolo Bonzini8fe65a82018-10-23 02:18:42 +02001260 *flush = false;
Paolo Bonzini2a31b9d2018-10-23 02:36:47 +02001261 if (kvm->manual_dirty_log_protect) {
1262 /*
1263 * Unlike kvm_get_dirty_log, we always return false in *flush,
1264 * because no flush is needed until KVM_CLEAR_DIRTY_LOG. There
1265 * is some code duplication between this function and
1266 * kvm_get_dirty_log, but hopefully all architecture
1267 * transition to kvm_get_dirty_log_protect and kvm_get_dirty_log
1268 * can be eliminated.
1269 */
1270 dirty_bitmap_buffer = dirty_bitmap;
1271 } else {
1272 dirty_bitmap_buffer = kvm_second_dirty_bitmap(memslot);
1273 memset(dirty_bitmap_buffer, 0, n);
Mario Smarduchba0513b2015-01-15 15:58:53 -08001274
Paolo Bonzini2a31b9d2018-10-23 02:36:47 +02001275 spin_lock(&kvm->mmu_lock);
1276 for (i = 0; i < n / sizeof(long); i++) {
1277 unsigned long mask;
1278 gfn_t offset;
Mario Smarduchba0513b2015-01-15 15:58:53 -08001279
Paolo Bonzini2a31b9d2018-10-23 02:36:47 +02001280 if (!dirty_bitmap[i])
1281 continue;
Mario Smarduchba0513b2015-01-15 15:58:53 -08001282
Paolo Bonzini2a31b9d2018-10-23 02:36:47 +02001283 *flush = true;
1284 mask = xchg(&dirty_bitmap[i], 0);
1285 dirty_bitmap_buffer[i] = mask;
Mario Smarduchba0513b2015-01-15 15:58:53 -08001286
Lan Tianyua67794c2019-02-02 17:20:27 +08001287 offset = i * BITS_PER_LONG;
1288 kvm_arch_mmu_enable_log_dirty_pt_masked(kvm, memslot,
1289 offset, mask);
Takuya Yoshikawa58d29302015-03-17 16:19:58 +09001290 }
Paolo Bonzini2a31b9d2018-10-23 02:36:47 +02001291 spin_unlock(&kvm->mmu_lock);
Mario Smarduchba0513b2015-01-15 15:58:53 -08001292 }
1293
Mario Smarduchba0513b2015-01-15 15:58:53 -08001294 if (copy_to_user(log->dirty_bitmap, dirty_bitmap_buffer, n))
Markus Elfring58d6db32017-01-22 17:30:16 +01001295 return -EFAULT;
1296 return 0;
Mario Smarduchba0513b2015-01-15 15:58:53 -08001297}
1298EXPORT_SYMBOL_GPL(kvm_get_dirty_log_protect);
Paolo Bonzini2a31b9d2018-10-23 02:36:47 +02001299
1300/**
1301 * kvm_clear_dirty_log_protect - clear dirty bits in the bitmap
1302 * and reenable dirty page tracking for the corresponding pages.
1303 * @kvm: pointer to kvm instance
1304 * @log: slot id and address from which to fetch the bitmap of dirty pages
Jiang Biaob8b00222019-04-23 19:40:30 +08001305 * @flush: true if TLB flush is needed by caller
Paolo Bonzini2a31b9d2018-10-23 02:36:47 +02001306 */
1307int kvm_clear_dirty_log_protect(struct kvm *kvm,
1308 struct kvm_clear_dirty_log *log, bool *flush)
1309{
1310 struct kvm_memslots *slots;
1311 struct kvm_memory_slot *memslot;
Tomas Bortoli98938aa2019-01-02 18:29:37 +01001312 int as_id, id;
Paolo Bonzini2a31b9d2018-10-23 02:36:47 +02001313 gfn_t offset;
Tomas Bortoli98938aa2019-01-02 18:29:37 +01001314 unsigned long i, n;
Paolo Bonzini2a31b9d2018-10-23 02:36:47 +02001315 unsigned long *dirty_bitmap;
1316 unsigned long *dirty_bitmap_buffer;
1317
1318 as_id = log->slot >> 16;
1319 id = (u16)log->slot;
1320 if (as_id >= KVM_ADDRESS_SPACE_NUM || id >= KVM_USER_MEM_SLOTS)
1321 return -EINVAL;
1322
Paolo Bonzini76d58e02019-04-17 15:28:44 +02001323 if (log->first_page & 63)
Paolo Bonzini2a31b9d2018-10-23 02:36:47 +02001324 return -EINVAL;
1325
1326 slots = __kvm_memslots(kvm, as_id);
1327 memslot = id_to_memslot(slots, id);
1328
1329 dirty_bitmap = memslot->dirty_bitmap;
1330 if (!dirty_bitmap)
1331 return -ENOENT;
1332
Peter Xu4ddc9202019-05-08 17:15:45 +08001333 n = ALIGN(log->num_pages, BITS_PER_LONG) / 8;
Tomas Bortoli98938aa2019-01-02 18:29:37 +01001334
1335 if (log->first_page > memslot->npages ||
Paolo Bonzini76d58e02019-04-17 15:28:44 +02001336 log->num_pages > memslot->npages - log->first_page ||
1337 (log->num_pages < memslot->npages - log->first_page && (log->num_pages & 63)))
1338 return -EINVAL;
Tomas Bortoli98938aa2019-01-02 18:29:37 +01001339
Paolo Bonzini2a31b9d2018-10-23 02:36:47 +02001340 *flush = false;
1341 dirty_bitmap_buffer = kvm_second_dirty_bitmap(memslot);
1342 if (copy_from_user(dirty_bitmap_buffer, log->dirty_bitmap, n))
1343 return -EFAULT;
1344
1345 spin_lock(&kvm->mmu_lock);
Peter Xu53eac7a2019-05-08 17:15:46 +08001346 for (offset = log->first_page, i = offset / BITS_PER_LONG,
1347 n = DIV_ROUND_UP(log->num_pages, BITS_PER_LONG); n--;
Paolo Bonzini2a31b9d2018-10-23 02:36:47 +02001348 i++, offset += BITS_PER_LONG) {
1349 unsigned long mask = *dirty_bitmap_buffer++;
1350 atomic_long_t *p = (atomic_long_t *) &dirty_bitmap[i];
1351 if (!mask)
1352 continue;
1353
1354 mask &= atomic_long_fetch_andnot(mask, p);
1355
1356 /*
1357 * mask contains the bits that really have been cleared. This
1358 * never includes any bits beyond the length of the memslot (if
1359 * the length is not aligned to 64 pages), therefore it is not
1360 * a problem if userspace sets them in log->dirty_bitmap.
1361 */
1362 if (mask) {
1363 *flush = true;
1364 kvm_arch_mmu_enable_log_dirty_pt_masked(kvm, memslot,
1365 offset, mask);
1366 }
1367 }
1368 spin_unlock(&kvm->mmu_lock);
1369
1370 return 0;
1371}
1372EXPORT_SYMBOL_GPL(kvm_clear_dirty_log_protect);
Mario Smarduchba0513b2015-01-15 15:58:53 -08001373#endif
1374
Takuya Yoshikawadb3fe4e2012-02-08 13:02:18 +09001375bool kvm_largepages_enabled(void)
1376{
1377 return largepages_enabled;
1378}
1379
Marcelo Tosatti54dee992009-06-11 12:07:44 -03001380void kvm_disable_largepages(void)
1381{
1382 largepages_enabled = false;
1383}
1384EXPORT_SYMBOL_GPL(kvm_disable_largepages);
1385
Gleb Natapov49c77542010-10-18 15:22:23 +02001386struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn)
1387{
1388 return __gfn_to_memslot(kvm_memslots(kvm), gfn);
1389}
Avi Kivitya1f4d3952010-06-21 11:44:20 +03001390EXPORT_SYMBOL_GPL(gfn_to_memslot);
Avi Kivity6aa8b732006-12-10 02:21:36 -08001391
Paolo Bonzini8e734852015-05-17 13:58:53 +02001392struct kvm_memory_slot *kvm_vcpu_gfn_to_memslot(struct kvm_vcpu *vcpu, gfn_t gfn)
1393{
1394 return __gfn_to_memslot(kvm_vcpu_memslots(vcpu), gfn);
1395}
1396
Yaowei Bai33e94152015-11-14 11:21:06 +08001397bool kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn)
Izik Eiduse0d62c72007-10-24 23:57:46 +02001398{
Xiao Guangrongbf3e05b2011-11-24 17:40:57 +08001399 struct kvm_memory_slot *memslot = gfn_to_memslot(kvm, gfn);
Izik Eiduse0d62c72007-10-24 23:57:46 +02001400
Alex Williamsonbbacc0c2012-12-10 10:33:09 -07001401 if (!memslot || memslot->id >= KVM_USER_MEM_SLOTS ||
Xiao Guangrongbf3e05b2011-11-24 17:40:57 +08001402 memslot->flags & KVM_MEMSLOT_INVALID)
Yaowei Bai33e94152015-11-14 11:21:06 +08001403 return false;
Izik Eiduse0d62c72007-10-24 23:57:46 +02001404
Yaowei Bai33e94152015-11-14 11:21:06 +08001405 return true;
Izik Eiduse0d62c72007-10-24 23:57:46 +02001406}
1407EXPORT_SYMBOL_GPL(kvm_is_visible_gfn);
1408
Joerg Roedel8f0b1ab2010-01-28 12:37:56 +01001409unsigned long kvm_host_page_size(struct kvm *kvm, gfn_t gfn)
1410{
1411 struct vm_area_struct *vma;
1412 unsigned long addr, size;
1413
1414 size = PAGE_SIZE;
1415
1416 addr = gfn_to_hva(kvm, gfn);
1417 if (kvm_is_error_hva(addr))
1418 return PAGE_SIZE;
1419
1420 down_read(&current->mm->mmap_sem);
1421 vma = find_vma(current->mm, addr);
1422 if (!vma)
1423 goto out;
1424
1425 size = vma_kernel_pagesize(vma);
1426
1427out:
1428 up_read(&current->mm->mmap_sem);
1429
1430 return size;
1431}
1432
Xiao Guangrong4d8b81a2012-08-21 11:02:51 +08001433static bool memslot_is_readonly(struct kvm_memory_slot *slot)
1434{
1435 return slot->flags & KVM_MEM_READONLY;
1436}
1437
Xiao Guangrong4d8b81a2012-08-21 11:02:51 +08001438static unsigned long __gfn_to_hva_many(struct kvm_memory_slot *slot, gfn_t gfn,
1439 gfn_t *nr_pages, bool write)
Izik Eidus539cb662007-11-11 22:05:04 +02001440{
Marcelo Tosattibc6678a2009-12-23 14:35:21 -02001441 if (!slot || slot->flags & KVM_MEMSLOT_INVALID)
Xiao Guangrongca3a4902012-08-21 11:01:50 +08001442 return KVM_HVA_ERR_BAD;
Xiao Guangrong48987782010-08-22 19:11:43 +08001443
Xiao Guangrong4d8b81a2012-08-21 11:02:51 +08001444 if (memslot_is_readonly(slot) && write)
1445 return KVM_HVA_ERR_RO_BAD;
Xiao Guangrong48987782010-08-22 19:11:43 +08001446
1447 if (nr_pages)
1448 *nr_pages = slot->npages - (gfn - slot->base_gfn);
1449
Xiao Guangrong4d8b81a2012-08-21 11:02:51 +08001450 return __gfn_to_hva_memslot(slot, gfn);
Izik Eidus539cb662007-11-11 22:05:04 +02001451}
Xiao Guangrong48987782010-08-22 19:11:43 +08001452
Xiao Guangrong4d8b81a2012-08-21 11:02:51 +08001453static unsigned long gfn_to_hva_many(struct kvm_memory_slot *slot, gfn_t gfn,
1454 gfn_t *nr_pages)
1455{
1456 return __gfn_to_hva_many(slot, gfn, nr_pages, true);
1457}
1458
1459unsigned long gfn_to_hva_memslot(struct kvm_memory_slot *slot,
Stephen Hemminger79408762013-12-29 12:12:29 -08001460 gfn_t gfn)
Xiao Guangrong4d8b81a2012-08-21 11:02:51 +08001461{
1462 return gfn_to_hva_many(slot, gfn, NULL);
1463}
1464EXPORT_SYMBOL_GPL(gfn_to_hva_memslot);
1465
Xiao Guangrong48987782010-08-22 19:11:43 +08001466unsigned long gfn_to_hva(struct kvm *kvm, gfn_t gfn)
1467{
Gleb Natapov49c77542010-10-18 15:22:23 +02001468 return gfn_to_hva_many(gfn_to_memslot(kvm, gfn), gfn, NULL);
Xiao Guangrong48987782010-08-22 19:11:43 +08001469}
Sheng Yang0d150292008-04-25 21:44:50 +08001470EXPORT_SYMBOL_GPL(gfn_to_hva);
Izik Eidus539cb662007-11-11 22:05:04 +02001471
Paolo Bonzini8e734852015-05-17 13:58:53 +02001472unsigned long kvm_vcpu_gfn_to_hva(struct kvm_vcpu *vcpu, gfn_t gfn)
1473{
1474 return gfn_to_hva_many(kvm_vcpu_gfn_to_memslot(vcpu, gfn), gfn, NULL);
1475}
1476EXPORT_SYMBOL_GPL(kvm_vcpu_gfn_to_hva);
1477
Xiao Guangrong86ab8cf2012-08-21 10:59:53 +08001478/*
Wei Yang970c0d42018-10-09 10:41:15 +08001479 * Return the hva of a @gfn and the R/W attribute if possible.
1480 *
1481 * @slot: the kvm_memory_slot which contains @gfn
1482 * @gfn: the gfn to be translated
1483 * @writable: used to return the read/write attribute of the @slot if the hva
1484 * is valid and @writable is not NULL
Xiao Guangrong86ab8cf2012-08-21 10:59:53 +08001485 */
Christoffer Dall64d83122014-08-19 12:15:00 +02001486unsigned long gfn_to_hva_memslot_prot(struct kvm_memory_slot *slot,
1487 gfn_t gfn, bool *writable)
Gleb Natapov80300892010-10-19 18:13:41 +02001488{
Gleb Natapova2ac07f2013-10-01 19:58:36 +03001489 unsigned long hva = __gfn_to_hva_many(slot, gfn, NULL, false);
1490
1491 if (!kvm_is_error_hva(hva) && writable)
Paolo Bonziniba6a3542013-09-09 13:52:33 +02001492 *writable = !memslot_is_readonly(slot);
1493
Gleb Natapova2ac07f2013-10-01 19:58:36 +03001494 return hva;
Xiao Guangrong86ab8cf2012-08-21 10:59:53 +08001495}
1496
Christoffer Dall64d83122014-08-19 12:15:00 +02001497unsigned long gfn_to_hva_prot(struct kvm *kvm, gfn_t gfn, bool *writable)
1498{
1499 struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn);
1500
1501 return gfn_to_hva_memslot_prot(slot, gfn, writable);
1502}
1503
Paolo Bonzini8e734852015-05-17 13:58:53 +02001504unsigned long kvm_vcpu_gfn_to_hva_prot(struct kvm_vcpu *vcpu, gfn_t gfn, bool *writable)
1505{
1506 struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
1507
1508 return gfn_to_hva_memslot_prot(slot, gfn, writable);
1509}
1510
Huang Yingfafc3db2011-01-30 11:15:49 +08001511static inline int check_user_page_hwpoison(unsigned long addr)
1512{
Lorenzo Stoakes0d731752016-10-24 10:57:25 +01001513 int rc, flags = FOLL_HWPOISON | FOLL_WRITE;
Huang Yingfafc3db2011-01-30 11:15:49 +08001514
Lorenzo Stoakes0d731752016-10-24 10:57:25 +01001515 rc = get_user_pages(addr, 1, flags, NULL, NULL);
Huang Yingfafc3db2011-01-30 11:15:49 +08001516 return rc == -EHWPOISON;
1517}
1518
Xiao Guangrong2fc843112012-08-21 11:00:22 +08001519/*
Paolo Bonzinib9b33da2018-07-27 17:44:41 +02001520 * The fast path to get the writable pfn which will be stored in @pfn,
1521 * true indicates success, otherwise false is returned. It's also the
1522 * only part that runs if we can are in atomic context.
Xiao Guangrong2fc843112012-08-21 11:00:22 +08001523 */
Paolo Bonzinib9b33da2018-07-27 17:44:41 +02001524static bool hva_to_pfn_fast(unsigned long addr, bool write_fault,
1525 bool *writable, kvm_pfn_t *pfn)
Xiao Guangrong2fc843112012-08-21 11:00:22 +08001526{
1527 struct page *page[1];
1528 int npages;
1529
Xiao Guangrong12ce13f2012-08-21 11:00:49 +08001530 /*
1531 * Fast pin a writable pfn only if it is a write fault request
1532 * or the caller allows to map a writable pfn for a read fault
1533 * request.
1534 */
1535 if (!(write_fault || writable))
1536 return false;
1537
Xiao Guangrong2fc843112012-08-21 11:00:22 +08001538 npages = __get_user_pages_fast(addr, 1, 1, page);
1539 if (npages == 1) {
1540 *pfn = page_to_pfn(page[0]);
1541
1542 if (writable)
1543 *writable = true;
1544 return true;
1545 }
1546
1547 return false;
1548}
1549
1550/*
1551 * The slow path to get the pfn of the specified host virtual address,
1552 * 1 indicates success, -errno is returned if error is detected.
1553 */
1554static int hva_to_pfn_slow(unsigned long addr, bool *async, bool write_fault,
Dan Williamsba049e92016-01-15 16:56:11 -08001555 bool *writable, kvm_pfn_t *pfn)
Avi Kivity954bbbc2007-03-30 14:02:32 +03001556{
Al Viroce530532017-11-19 17:47:33 -05001557 unsigned int flags = FOLL_HWPOISON;
1558 struct page *page;
Gleb Natapovaf585b92010-10-14 11:22:46 +02001559 int npages = 0;
Xiao Guangrong2fc843112012-08-21 11:00:22 +08001560
1561 might_sleep();
1562
1563 if (writable)
1564 *writable = write_fault;
1565
Al Viroce530532017-11-19 17:47:33 -05001566 if (write_fault)
1567 flags |= FOLL_WRITE;
1568 if (async)
1569 flags |= FOLL_NOWAIT;
Lorenzo Stoakesd4944b02016-10-13 01:20:12 +01001570
Al Viroce530532017-11-19 17:47:33 -05001571 npages = get_user_pages_unlocked(addr, 1, &page, flags);
Xiao Guangrong2fc843112012-08-21 11:00:22 +08001572 if (npages != 1)
1573 return npages;
1574
1575 /* map read fault as writable if possible */
Xiao Guangrong12ce13f2012-08-21 11:00:49 +08001576 if (unlikely(!write_fault) && writable) {
Al Viroce530532017-11-19 17:47:33 -05001577 struct page *wpage;
Xiao Guangrong2fc843112012-08-21 11:00:22 +08001578
Al Viroce530532017-11-19 17:47:33 -05001579 if (__get_user_pages_fast(addr, 1, 1, &wpage) == 1) {
Xiao Guangrong2fc843112012-08-21 11:00:22 +08001580 *writable = true;
Al Viroce530532017-11-19 17:47:33 -05001581 put_page(page);
1582 page = wpage;
Xiao Guangrong2fc843112012-08-21 11:00:22 +08001583 }
Xiao Guangrong2fc843112012-08-21 11:00:22 +08001584 }
Al Viroce530532017-11-19 17:47:33 -05001585 *pfn = page_to_pfn(page);
Xiao Guangrong2fc843112012-08-21 11:00:22 +08001586 return npages;
1587}
1588
Xiao Guangrong4d8b81a2012-08-21 11:02:51 +08001589static bool vma_is_valid(struct vm_area_struct *vma, bool write_fault)
1590{
1591 if (unlikely(!(vma->vm_flags & VM_READ)))
1592 return false;
1593
1594 if (write_fault && (unlikely(!(vma->vm_flags & VM_WRITE))))
1595 return false;
1596
1597 return true;
1598}
1599
Paolo Bonzini92176a82016-06-07 16:22:47 +02001600static int hva_to_pfn_remapped(struct vm_area_struct *vma,
1601 unsigned long addr, bool *async,
KarimAllah Ahmeda340b3e2018-01-17 19:18:56 +01001602 bool write_fault, bool *writable,
1603 kvm_pfn_t *p_pfn)
Paolo Bonzini92176a82016-06-07 16:22:47 +02001604{
Paolo Bonziniadd6a0c2016-06-07 17:51:18 +02001605 unsigned long pfn;
1606 int r;
1607
1608 r = follow_pfn(vma, addr, &pfn);
1609 if (r) {
1610 /*
1611 * get_user_pages fails for VM_IO and VM_PFNMAP vmas and does
1612 * not call the fault handler, so do it here.
1613 */
1614 bool unlocked = false;
1615 r = fixup_user_fault(current, current->mm, addr,
1616 (write_fault ? FAULT_FLAG_WRITE : 0),
1617 &unlocked);
1618 if (unlocked)
1619 return -EAGAIN;
1620 if (r)
1621 return r;
1622
1623 r = follow_pfn(vma, addr, &pfn);
1624 if (r)
1625 return r;
1626
1627 }
1628
KarimAllah Ahmeda340b3e2018-01-17 19:18:56 +01001629 if (writable)
1630 *writable = true;
Paolo Bonziniadd6a0c2016-06-07 17:51:18 +02001631
1632 /*
1633 * Get a reference here because callers of *hva_to_pfn* and
1634 * *gfn_to_pfn* ultimately call kvm_release_pfn_clean on the
1635 * returned pfn. This is only needed if the VMA has VM_MIXEDMAP
1636 * set, but the kvm_get_pfn/kvm_release_pfn_clean pair will
1637 * simply do nothing for reserved pfns.
1638 *
1639 * Whoever called remap_pfn_range is also going to call e.g.
1640 * unmap_mapping_range before the underlying pages are freed,
1641 * causing a call to our MMU notifier.
1642 */
1643 kvm_get_pfn(pfn);
1644
1645 *p_pfn = pfn;
Paolo Bonzini92176a82016-06-07 16:22:47 +02001646 return 0;
1647}
1648
Xiao Guangrong12ce13f2012-08-21 11:00:49 +08001649/*
1650 * Pin guest page in memory and return its pfn.
1651 * @addr: host virtual address which maps memory to the guest
1652 * @atomic: whether this function can sleep
1653 * @async: whether this function need to wait IO complete if the
1654 * host page is not in the memory
1655 * @write_fault: whether we should get a writable host page
1656 * @writable: whether it allows to map a writable host page for !@write_fault
1657 *
1658 * The function will map a writable host page for these two cases:
1659 * 1): @write_fault = true
1660 * 2): @write_fault = false && @writable, @writable will tell the caller
1661 * whether the mapping is writable.
1662 */
Dan Williamsba049e92016-01-15 16:56:11 -08001663static kvm_pfn_t hva_to_pfn(unsigned long addr, bool atomic, bool *async,
Xiao Guangrong2fc843112012-08-21 11:00:22 +08001664 bool write_fault, bool *writable)
1665{
1666 struct vm_area_struct *vma;
Dan Williamsba049e92016-01-15 16:56:11 -08001667 kvm_pfn_t pfn = 0;
Paolo Bonzini92176a82016-06-07 16:22:47 +02001668 int npages, r;
Avi Kivity954bbbc2007-03-30 14:02:32 +03001669
Gleb Natapovaf585b92010-10-14 11:22:46 +02001670 /* we can do it either atomically or asynchronously, not both */
1671 BUG_ON(atomic && async);
1672
Paolo Bonzinib9b33da2018-07-27 17:44:41 +02001673 if (hva_to_pfn_fast(addr, write_fault, writable, &pfn))
Xiao Guangrong2fc843112012-08-21 11:00:22 +08001674 return pfn;
Marcelo Tosatti612819c2010-10-22 14:18:18 -02001675
Xiao Guangrong2fc843112012-08-21 11:00:22 +08001676 if (atomic)
1677 return KVM_PFN_ERR_FAULT;
Marcelo Tosatti612819c2010-10-22 14:18:18 -02001678
Xiao Guangrong2fc843112012-08-21 11:00:22 +08001679 npages = hva_to_pfn_slow(addr, async, write_fault, writable, &pfn);
1680 if (npages == 1)
1681 return pfn;
Gleb Natapovaf585b92010-10-14 11:22:46 +02001682
Xiao Guangrong2fc843112012-08-21 11:00:22 +08001683 down_read(&current->mm->mmap_sem);
1684 if (npages == -EHWPOISON ||
1685 (!async && check_user_page_hwpoison(addr))) {
1686 pfn = KVM_PFN_ERR_HWPOISON;
1687 goto exit;
Xiao Guangrong887c08a2010-08-22 19:10:28 +08001688 }
Izik Eidus539cb662007-11-11 22:05:04 +02001689
Paolo Bonziniadd6a0c2016-06-07 17:51:18 +02001690retry:
Xiao Guangrong2fc843112012-08-21 11:00:22 +08001691 vma = find_vma_intersection(current->mm, addr, addr + 1);
Anthony Liguori8d4e1282007-10-18 09:59:34 -05001692
Xiao Guangrong2fc843112012-08-21 11:00:22 +08001693 if (vma == NULL)
1694 pfn = KVM_PFN_ERR_FAULT;
Paolo Bonzini92176a82016-06-07 16:22:47 +02001695 else if (vma->vm_flags & (VM_IO | VM_PFNMAP)) {
KarimAllah Ahmeda340b3e2018-01-17 19:18:56 +01001696 r = hva_to_pfn_remapped(vma, addr, async, write_fault, writable, &pfn);
Paolo Bonziniadd6a0c2016-06-07 17:51:18 +02001697 if (r == -EAGAIN)
1698 goto retry;
Paolo Bonzini92176a82016-06-07 16:22:47 +02001699 if (r < 0)
1700 pfn = KVM_PFN_ERR_FAULT;
Xiao Guangrong2fc843112012-08-21 11:00:22 +08001701 } else {
Xiao Guangrong4d8b81a2012-08-21 11:02:51 +08001702 if (async && vma_is_valid(vma, write_fault))
Xiao Guangrong2fc843112012-08-21 11:00:22 +08001703 *async = true;
1704 pfn = KVM_PFN_ERR_FAULT;
1705 }
1706exit:
1707 up_read(&current->mm->mmap_sem);
Anthony Liguori2e2e3732008-04-30 15:37:07 -05001708 return pfn;
Anthony Liguori35149e22008-04-02 14:46:56 -05001709}
1710
Dan Williamsba049e92016-01-15 16:56:11 -08001711kvm_pfn_t __gfn_to_pfn_memslot(struct kvm_memory_slot *slot, gfn_t gfn,
1712 bool atomic, bool *async, bool write_fault,
1713 bool *writable)
Xiao Guangrong887c08a2010-08-22 19:10:28 +08001714{
Xiao Guangrong4d8b81a2012-08-21 11:02:51 +08001715 unsigned long addr = __gfn_to_hva_many(slot, gfn, NULL, write_fault);
1716
Paolo Bonzinib2740d32016-02-23 15:36:01 +01001717 if (addr == KVM_HVA_ERR_RO_BAD) {
1718 if (writable)
1719 *writable = false;
Xiao Guangrong4d8b81a2012-08-21 11:02:51 +08001720 return KVM_PFN_ERR_RO_FAULT;
Paolo Bonzinib2740d32016-02-23 15:36:01 +01001721 }
Xiao Guangrong4d8b81a2012-08-21 11:02:51 +08001722
Paolo Bonzinib2740d32016-02-23 15:36:01 +01001723 if (kvm_is_error_hva(addr)) {
1724 if (writable)
1725 *writable = false;
Xiao Guangrong81c52c52012-10-16 20:10:59 +08001726 return KVM_PFN_NOSLOT;
Paolo Bonzinib2740d32016-02-23 15:36:01 +01001727 }
Xiao Guangrong4d8b81a2012-08-21 11:02:51 +08001728
1729 /* Do not map writable pfn in the readonly memslot. */
1730 if (writable && memslot_is_readonly(slot)) {
1731 *writable = false;
1732 writable = NULL;
1733 }
1734
1735 return hva_to_pfn(addr, atomic, async, write_fault,
1736 writable);
Xiao Guangrong887c08a2010-08-22 19:10:28 +08001737}
Paolo Bonzini35204692015-04-02 11:20:48 +02001738EXPORT_SYMBOL_GPL(__gfn_to_pfn_memslot);
Xiao Guangrong887c08a2010-08-22 19:10:28 +08001739
Dan Williamsba049e92016-01-15 16:56:11 -08001740kvm_pfn_t gfn_to_pfn_prot(struct kvm *kvm, gfn_t gfn, bool write_fault,
Marcelo Tosatti612819c2010-10-22 14:18:18 -02001741 bool *writable)
1742{
Paolo Bonzinie37afc62015-05-19 16:09:04 +02001743 return __gfn_to_pfn_memslot(gfn_to_memslot(kvm, gfn), gfn, false, NULL,
1744 write_fault, writable);
Marcelo Tosatti612819c2010-10-22 14:18:18 -02001745}
1746EXPORT_SYMBOL_GPL(gfn_to_pfn_prot);
1747
Dan Williamsba049e92016-01-15 16:56:11 -08001748kvm_pfn_t gfn_to_pfn_memslot(struct kvm_memory_slot *slot, gfn_t gfn)
Marcelo Tosatti506f0d62009-12-23 14:35:19 -02001749{
Xiao Guangrong4d8b81a2012-08-21 11:02:51 +08001750 return __gfn_to_pfn_memslot(slot, gfn, false, NULL, true, NULL);
Marcelo Tosatti506f0d62009-12-23 14:35:19 -02001751}
Paolo Bonzinie37afc62015-05-19 16:09:04 +02001752EXPORT_SYMBOL_GPL(gfn_to_pfn_memslot);
Marcelo Tosatti506f0d62009-12-23 14:35:19 -02001753
Dan Williamsba049e92016-01-15 16:56:11 -08001754kvm_pfn_t gfn_to_pfn_memslot_atomic(struct kvm_memory_slot *slot, gfn_t gfn)
Xiao Guangrong037d92d2012-08-21 10:59:12 +08001755{
Xiao Guangrong4d8b81a2012-08-21 11:02:51 +08001756 return __gfn_to_pfn_memslot(slot, gfn, true, NULL, true, NULL);
Xiao Guangrong037d92d2012-08-21 10:59:12 +08001757}
1758EXPORT_SYMBOL_GPL(gfn_to_pfn_memslot_atomic);
1759
Dan Williamsba049e92016-01-15 16:56:11 -08001760kvm_pfn_t gfn_to_pfn_atomic(struct kvm *kvm, gfn_t gfn)
Paolo Bonzinie37afc62015-05-19 16:09:04 +02001761{
1762 return gfn_to_pfn_memslot_atomic(gfn_to_memslot(kvm, gfn), gfn);
1763}
1764EXPORT_SYMBOL_GPL(gfn_to_pfn_atomic);
1765
Dan Williamsba049e92016-01-15 16:56:11 -08001766kvm_pfn_t kvm_vcpu_gfn_to_pfn_atomic(struct kvm_vcpu *vcpu, gfn_t gfn)
Paolo Bonzini8e734852015-05-17 13:58:53 +02001767{
1768 return gfn_to_pfn_memslot_atomic(kvm_vcpu_gfn_to_memslot(vcpu, gfn), gfn);
1769}
1770EXPORT_SYMBOL_GPL(kvm_vcpu_gfn_to_pfn_atomic);
1771
Dan Williamsba049e92016-01-15 16:56:11 -08001772kvm_pfn_t gfn_to_pfn(struct kvm *kvm, gfn_t gfn)
Paolo Bonzinie37afc62015-05-19 16:09:04 +02001773{
1774 return gfn_to_pfn_memslot(gfn_to_memslot(kvm, gfn), gfn);
1775}
1776EXPORT_SYMBOL_GPL(gfn_to_pfn);
1777
Dan Williamsba049e92016-01-15 16:56:11 -08001778kvm_pfn_t kvm_vcpu_gfn_to_pfn(struct kvm_vcpu *vcpu, gfn_t gfn)
Paolo Bonzini8e734852015-05-17 13:58:53 +02001779{
1780 return gfn_to_pfn_memslot(kvm_vcpu_gfn_to_memslot(vcpu, gfn), gfn);
1781}
1782EXPORT_SYMBOL_GPL(kvm_vcpu_gfn_to_pfn);
1783
Paolo Bonzinid9ef13c2015-05-19 16:01:50 +02001784int gfn_to_page_many_atomic(struct kvm_memory_slot *slot, gfn_t gfn,
1785 struct page **pages, int nr_pages)
Xiao Guangrong48987782010-08-22 19:11:43 +08001786{
1787 unsigned long addr;
Arnd Bergmann076b9252017-08-10 14:14:39 +02001788 gfn_t entry = 0;
Xiao Guangrong48987782010-08-22 19:11:43 +08001789
Paolo Bonzinid9ef13c2015-05-19 16:01:50 +02001790 addr = gfn_to_hva_many(slot, gfn, &entry);
Xiao Guangrong48987782010-08-22 19:11:43 +08001791 if (kvm_is_error_hva(addr))
1792 return -1;
1793
1794 if (entry < nr_pages)
1795 return 0;
1796
1797 return __get_user_pages_fast(addr, nr_pages, 1, pages);
1798}
1799EXPORT_SYMBOL_GPL(gfn_to_page_many_atomic);
1800
Dan Williamsba049e92016-01-15 16:56:11 -08001801static struct page *kvm_pfn_to_page(kvm_pfn_t pfn)
Xiao Guangronga2766322012-07-26 11:58:59 +08001802{
Xiao Guangrong81c52c52012-10-16 20:10:59 +08001803 if (is_error_noslot_pfn(pfn))
Xiao Guangrong6cede2e2012-08-03 15:41:22 +08001804 return KVM_ERR_PTR_BAD_PAGE;
Xiao Guangronga2766322012-07-26 11:58:59 +08001805
Ard Biesheuvelbf4bea82014-11-10 08:33:56 +00001806 if (kvm_is_reserved_pfn(pfn)) {
Xiao Guangrongcb9aaa32012-08-03 15:42:10 +08001807 WARN_ON(1);
1808 return KVM_ERR_PTR_BAD_PAGE;
1809 }
1810
Xiao Guangronga2766322012-07-26 11:58:59 +08001811 return pfn_to_page(pfn);
1812}
1813
Anthony Liguori35149e22008-04-02 14:46:56 -05001814struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn)
1815{
Dan Williamsba049e92016-01-15 16:56:11 -08001816 kvm_pfn_t pfn;
Anthony Liguori2e2e3732008-04-30 15:37:07 -05001817
1818 pfn = gfn_to_pfn(kvm, gfn);
Anthony Liguori2e2e3732008-04-30 15:37:07 -05001819
Xiao Guangronga2766322012-07-26 11:58:59 +08001820 return kvm_pfn_to_page(pfn);
Avi Kivity954bbbc2007-03-30 14:02:32 +03001821}
1822EXPORT_SYMBOL_GPL(gfn_to_page);
1823
KarimAllah Ahmede45adf62019-01-31 21:24:34 +01001824static int __kvm_map_gfn(struct kvm_memory_slot *slot, gfn_t gfn,
1825 struct kvm_host_map *map)
1826{
1827 kvm_pfn_t pfn;
1828 void *hva = NULL;
1829 struct page *page = KVM_UNMAPPED_PAGE;
1830
1831 if (!map)
1832 return -EINVAL;
1833
1834 pfn = gfn_to_pfn_memslot(slot, gfn);
1835 if (is_error_noslot_pfn(pfn))
1836 return -EINVAL;
1837
1838 if (pfn_valid(pfn)) {
1839 page = pfn_to_page(pfn);
1840 hva = kmap(page);
Paolo Bonzinid30b2142019-05-20 12:06:36 +02001841#ifdef CONFIG_HAS_IOMEM
KarimAllah Ahmede45adf62019-01-31 21:24:34 +01001842 } else {
1843 hva = memremap(pfn_to_hpa(pfn), PAGE_SIZE, MEMREMAP_WB);
Paolo Bonzinid30b2142019-05-20 12:06:36 +02001844#endif
KarimAllah Ahmede45adf62019-01-31 21:24:34 +01001845 }
1846
1847 if (!hva)
1848 return -EFAULT;
1849
1850 map->page = page;
1851 map->hva = hva;
1852 map->pfn = pfn;
1853 map->gfn = gfn;
1854
1855 return 0;
1856}
1857
1858int kvm_vcpu_map(struct kvm_vcpu *vcpu, gfn_t gfn, struct kvm_host_map *map)
1859{
1860 return __kvm_map_gfn(kvm_vcpu_gfn_to_memslot(vcpu, gfn), gfn, map);
1861}
1862EXPORT_SYMBOL_GPL(kvm_vcpu_map);
1863
1864void kvm_vcpu_unmap(struct kvm_vcpu *vcpu, struct kvm_host_map *map,
1865 bool dirty)
1866{
1867 if (!map)
1868 return;
1869
1870 if (!map->hva)
1871 return;
1872
KarimAllah Ahmedb614c6022019-07-10 11:13:13 +02001873 if (map->page != KVM_UNMAPPED_PAGE)
KarimAllah Ahmede45adf62019-01-31 21:24:34 +01001874 kunmap(map->page);
Christian Borntraegereb1f2f32019-05-27 10:28:25 +02001875#ifdef CONFIG_HAS_IOMEM
KarimAllah Ahmede45adf62019-01-31 21:24:34 +01001876 else
1877 memunmap(map->hva);
Christian Borntraegereb1f2f32019-05-27 10:28:25 +02001878#endif
KarimAllah Ahmede45adf62019-01-31 21:24:34 +01001879
1880 if (dirty) {
1881 kvm_vcpu_mark_page_dirty(vcpu, map->gfn);
1882 kvm_release_pfn_dirty(map->pfn);
1883 } else {
1884 kvm_release_pfn_clean(map->pfn);
1885 }
1886
1887 map->hva = NULL;
1888 map->page = NULL;
1889}
1890EXPORT_SYMBOL_GPL(kvm_vcpu_unmap);
1891
Paolo Bonzini8e734852015-05-17 13:58:53 +02001892struct page *kvm_vcpu_gfn_to_page(struct kvm_vcpu *vcpu, gfn_t gfn)
1893{
Dan Williamsba049e92016-01-15 16:56:11 -08001894 kvm_pfn_t pfn;
Paolo Bonzini8e734852015-05-17 13:58:53 +02001895
1896 pfn = kvm_vcpu_gfn_to_pfn(vcpu, gfn);
1897
1898 return kvm_pfn_to_page(pfn);
1899}
1900EXPORT_SYMBOL_GPL(kvm_vcpu_gfn_to_page);
1901
Izik Eidusb4231d612007-11-20 11:49:33 +02001902void kvm_release_page_clean(struct page *page)
1903{
Xiao Guangrong32cad842012-08-03 15:42:52 +08001904 WARN_ON(is_error_page(page));
1905
Anthony Liguori35149e22008-04-02 14:46:56 -05001906 kvm_release_pfn_clean(page_to_pfn(page));
Izik Eidusb4231d612007-11-20 11:49:33 +02001907}
1908EXPORT_SYMBOL_GPL(kvm_release_page_clean);
1909
Dan Williamsba049e92016-01-15 16:56:11 -08001910void kvm_release_pfn_clean(kvm_pfn_t pfn)
Anthony Liguori35149e22008-04-02 14:46:56 -05001911{
Ard Biesheuvelbf4bea82014-11-10 08:33:56 +00001912 if (!is_error_noslot_pfn(pfn) && !kvm_is_reserved_pfn(pfn))
Anthony Liguori2e2e3732008-04-30 15:37:07 -05001913 put_page(pfn_to_page(pfn));
Anthony Liguori35149e22008-04-02 14:46:56 -05001914}
1915EXPORT_SYMBOL_GPL(kvm_release_pfn_clean);
1916
Izik Eidusb4231d612007-11-20 11:49:33 +02001917void kvm_release_page_dirty(struct page *page)
Izik Eidus8a7ae052007-10-18 11:09:33 +02001918{
Xiao Guangronga2766322012-07-26 11:58:59 +08001919 WARN_ON(is_error_page(page));
1920
Anthony Liguori35149e22008-04-02 14:46:56 -05001921 kvm_release_pfn_dirty(page_to_pfn(page));
Izik Eidus8a7ae052007-10-18 11:09:33 +02001922}
Izik Eidusb4231d612007-11-20 11:49:33 +02001923EXPORT_SYMBOL_GPL(kvm_release_page_dirty);
Izik Eidus8a7ae052007-10-18 11:09:33 +02001924
David Hildenbrandf7a65092017-09-01 17:11:43 +02001925void kvm_release_pfn_dirty(kvm_pfn_t pfn)
Anthony Liguori35149e22008-04-02 14:46:56 -05001926{
1927 kvm_set_pfn_dirty(pfn);
1928 kvm_release_pfn_clean(pfn);
1929}
David Hildenbrandf7a65092017-09-01 17:11:43 +02001930EXPORT_SYMBOL_GPL(kvm_release_pfn_dirty);
Anthony Liguori35149e22008-04-02 14:46:56 -05001931
Dan Williamsba049e92016-01-15 16:56:11 -08001932void kvm_set_pfn_dirty(kvm_pfn_t pfn)
Anthony Liguori35149e22008-04-02 14:46:56 -05001933{
Sean Christophersona78986a2019-11-11 14:12:27 -08001934 if (!kvm_is_reserved_pfn(pfn) && !kvm_is_zone_device_pfn(pfn)) {
Anthony Liguori2e2e3732008-04-30 15:37:07 -05001935 struct page *page = pfn_to_page(pfn);
Xiubo Lif95ef0cd2015-02-26 14:58:23 +08001936
Paolo Bonzini8f946da2019-08-05 11:11:08 +02001937 SetPageDirty(page);
Anthony Liguori2e2e3732008-04-30 15:37:07 -05001938 }
Anthony Liguori35149e22008-04-02 14:46:56 -05001939}
1940EXPORT_SYMBOL_GPL(kvm_set_pfn_dirty);
1941
Dan Williamsba049e92016-01-15 16:56:11 -08001942void kvm_set_pfn_accessed(kvm_pfn_t pfn)
Anthony Liguori35149e22008-04-02 14:46:56 -05001943{
Sean Christophersona78986a2019-11-11 14:12:27 -08001944 if (!kvm_is_reserved_pfn(pfn) && !kvm_is_zone_device_pfn(pfn))
Anthony Liguori2e2e3732008-04-30 15:37:07 -05001945 mark_page_accessed(pfn_to_page(pfn));
Anthony Liguori35149e22008-04-02 14:46:56 -05001946}
1947EXPORT_SYMBOL_GPL(kvm_set_pfn_accessed);
1948
Dan Williamsba049e92016-01-15 16:56:11 -08001949void kvm_get_pfn(kvm_pfn_t pfn)
Anthony Liguori35149e22008-04-02 14:46:56 -05001950{
Ard Biesheuvelbf4bea82014-11-10 08:33:56 +00001951 if (!kvm_is_reserved_pfn(pfn))
Anthony Liguori2e2e3732008-04-30 15:37:07 -05001952 get_page(pfn_to_page(pfn));
Anthony Liguori35149e22008-04-02 14:46:56 -05001953}
1954EXPORT_SYMBOL_GPL(kvm_get_pfn);
1955
Izik Eidus195aefd2007-10-01 22:14:18 +02001956static int next_segment(unsigned long len, int offset)
1957{
1958 if (len > PAGE_SIZE - offset)
1959 return PAGE_SIZE - offset;
1960 else
1961 return len;
1962}
1963
Paolo Bonzini8e734852015-05-17 13:58:53 +02001964static int __kvm_read_guest_page(struct kvm_memory_slot *slot, gfn_t gfn,
1965 void *data, int offset, int len)
Izik Eidus195aefd2007-10-01 22:14:18 +02001966{
Izik Eiduse0506bc2007-11-11 22:10:22 +02001967 int r;
1968 unsigned long addr;
Izik Eidus195aefd2007-10-01 22:14:18 +02001969
Paolo Bonzini8e734852015-05-17 13:58:53 +02001970 addr = gfn_to_hva_memslot_prot(slot, gfn, NULL);
Izik Eiduse0506bc2007-11-11 22:10:22 +02001971 if (kvm_is_error_hva(addr))
Izik Eidus195aefd2007-10-01 22:14:18 +02001972 return -EFAULT;
Paolo Bonzini3180a7f2015-04-02 14:08:20 +02001973 r = __copy_from_user(data, (void __user *)addr + offset, len);
Izik Eiduse0506bc2007-11-11 22:10:22 +02001974 if (r)
1975 return -EFAULT;
Izik Eidus195aefd2007-10-01 22:14:18 +02001976 return 0;
1977}
Paolo Bonzini8e734852015-05-17 13:58:53 +02001978
1979int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset,
1980 int len)
1981{
1982 struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn);
1983
1984 return __kvm_read_guest_page(slot, gfn, data, offset, len);
1985}
Izik Eidus195aefd2007-10-01 22:14:18 +02001986EXPORT_SYMBOL_GPL(kvm_read_guest_page);
1987
Paolo Bonzini8e734852015-05-17 13:58:53 +02001988int kvm_vcpu_read_guest_page(struct kvm_vcpu *vcpu, gfn_t gfn, void *data,
1989 int offset, int len)
1990{
1991 struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
1992
1993 return __kvm_read_guest_page(slot, gfn, data, offset, len);
1994}
1995EXPORT_SYMBOL_GPL(kvm_vcpu_read_guest_page);
1996
Izik Eidus195aefd2007-10-01 22:14:18 +02001997int kvm_read_guest(struct kvm *kvm, gpa_t gpa, void *data, unsigned long len)
1998{
1999 gfn_t gfn = gpa >> PAGE_SHIFT;
2000 int seg;
2001 int offset = offset_in_page(gpa);
2002 int ret;
2003
2004 while ((seg = next_segment(len, offset)) != 0) {
2005 ret = kvm_read_guest_page(kvm, gfn, data, offset, seg);
2006 if (ret < 0)
2007 return ret;
2008 offset = 0;
2009 len -= seg;
2010 data += seg;
2011 ++gfn;
2012 }
2013 return 0;
2014}
2015EXPORT_SYMBOL_GPL(kvm_read_guest);
2016
Paolo Bonzini8e734852015-05-17 13:58:53 +02002017int kvm_vcpu_read_guest(struct kvm_vcpu *vcpu, gpa_t gpa, void *data, unsigned long len)
2018{
2019 gfn_t gfn = gpa >> PAGE_SHIFT;
2020 int seg;
2021 int offset = offset_in_page(gpa);
2022 int ret;
2023
2024 while ((seg = next_segment(len, offset)) != 0) {
2025 ret = kvm_vcpu_read_guest_page(vcpu, gfn, data, offset, seg);
2026 if (ret < 0)
2027 return ret;
2028 offset = 0;
2029 len -= seg;
2030 data += seg;
2031 ++gfn;
2032 }
2033 return 0;
2034}
2035EXPORT_SYMBOL_GPL(kvm_vcpu_read_guest);
2036
2037static int __kvm_read_guest_atomic(struct kvm_memory_slot *slot, gfn_t gfn,
2038 void *data, int offset, unsigned long len)
Marcelo Tosatti7ec54582007-12-20 19:18:23 -05002039{
2040 int r;
2041 unsigned long addr;
Marcelo Tosatti7ec54582007-12-20 19:18:23 -05002042
Paolo Bonzini8e734852015-05-17 13:58:53 +02002043 addr = gfn_to_hva_memslot_prot(slot, gfn, NULL);
Marcelo Tosatti7ec54582007-12-20 19:18:23 -05002044 if (kvm_is_error_hva(addr))
2045 return -EFAULT;
Andrea Arcangeli0aac03f2008-01-30 19:57:35 +01002046 pagefault_disable();
Paolo Bonzini3180a7f2015-04-02 14:08:20 +02002047 r = __copy_from_user_inatomic(data, (void __user *)addr + offset, len);
Andrea Arcangeli0aac03f2008-01-30 19:57:35 +01002048 pagefault_enable();
Marcelo Tosatti7ec54582007-12-20 19:18:23 -05002049 if (r)
2050 return -EFAULT;
2051 return 0;
2052}
Marcelo Tosatti7ec54582007-12-20 19:18:23 -05002053
Paolo Bonzini8e734852015-05-17 13:58:53 +02002054int kvm_read_guest_atomic(struct kvm *kvm, gpa_t gpa, void *data,
2055 unsigned long len)
2056{
2057 gfn_t gfn = gpa >> PAGE_SHIFT;
2058 struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn);
2059 int offset = offset_in_page(gpa);
2060
2061 return __kvm_read_guest_atomic(slot, gfn, data, offset, len);
2062}
2063EXPORT_SYMBOL_GPL(kvm_read_guest_atomic);
2064
2065int kvm_vcpu_read_guest_atomic(struct kvm_vcpu *vcpu, gpa_t gpa,
2066 void *data, unsigned long len)
2067{
2068 gfn_t gfn = gpa >> PAGE_SHIFT;
2069 struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
2070 int offset = offset_in_page(gpa);
2071
2072 return __kvm_read_guest_atomic(slot, gfn, data, offset, len);
2073}
2074EXPORT_SYMBOL_GPL(kvm_vcpu_read_guest_atomic);
2075
2076static int __kvm_write_guest_page(struct kvm_memory_slot *memslot, gfn_t gfn,
2077 const void *data, int offset, int len)
Izik Eidus195aefd2007-10-01 22:14:18 +02002078{
Izik Eiduse0506bc2007-11-11 22:10:22 +02002079 int r;
2080 unsigned long addr;
Izik Eidus195aefd2007-10-01 22:14:18 +02002081
Radim Krčmář251eb842015-04-10 21:47:27 +02002082 addr = gfn_to_hva_memslot(memslot, gfn);
Izik Eiduse0506bc2007-11-11 22:10:22 +02002083 if (kvm_is_error_hva(addr))
Izik Eidus195aefd2007-10-01 22:14:18 +02002084 return -EFAULT;
Xiao Guangrong8b0cedf2011-05-15 23:22:04 +08002085 r = __copy_to_user((void __user *)addr + offset, data, len);
Izik Eiduse0506bc2007-11-11 22:10:22 +02002086 if (r)
2087 return -EFAULT;
Paolo Bonzinibc009e42015-05-26 12:43:41 +02002088 mark_page_dirty_in_slot(memslot, gfn);
Izik Eidus195aefd2007-10-01 22:14:18 +02002089 return 0;
2090}
Paolo Bonzini8e734852015-05-17 13:58:53 +02002091
2092int kvm_write_guest_page(struct kvm *kvm, gfn_t gfn,
2093 const void *data, int offset, int len)
2094{
2095 struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn);
2096
2097 return __kvm_write_guest_page(slot, gfn, data, offset, len);
2098}
Izik Eidus195aefd2007-10-01 22:14:18 +02002099EXPORT_SYMBOL_GPL(kvm_write_guest_page);
2100
Paolo Bonzini8e734852015-05-17 13:58:53 +02002101int kvm_vcpu_write_guest_page(struct kvm_vcpu *vcpu, gfn_t gfn,
2102 const void *data, int offset, int len)
2103{
2104 struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
2105
2106 return __kvm_write_guest_page(slot, gfn, data, offset, len);
2107}
2108EXPORT_SYMBOL_GPL(kvm_vcpu_write_guest_page);
2109
Izik Eidus195aefd2007-10-01 22:14:18 +02002110int kvm_write_guest(struct kvm *kvm, gpa_t gpa, const void *data,
2111 unsigned long len)
2112{
2113 gfn_t gfn = gpa >> PAGE_SHIFT;
2114 int seg;
2115 int offset = offset_in_page(gpa);
2116 int ret;
2117
2118 while ((seg = next_segment(len, offset)) != 0) {
2119 ret = kvm_write_guest_page(kvm, gfn, data, offset, seg);
2120 if (ret < 0)
2121 return ret;
2122 offset = 0;
2123 len -= seg;
2124 data += seg;
2125 ++gfn;
2126 }
2127 return 0;
2128}
Wincy Vanff651cb2014-12-11 08:52:58 +03002129EXPORT_SYMBOL_GPL(kvm_write_guest);
Izik Eidus195aefd2007-10-01 22:14:18 +02002130
Paolo Bonzini8e734852015-05-17 13:58:53 +02002131int kvm_vcpu_write_guest(struct kvm_vcpu *vcpu, gpa_t gpa, const void *data,
2132 unsigned long len)
2133{
2134 gfn_t gfn = gpa >> PAGE_SHIFT;
2135 int seg;
2136 int offset = offset_in_page(gpa);
2137 int ret;
2138
2139 while ((seg = next_segment(len, offset)) != 0) {
2140 ret = kvm_vcpu_write_guest_page(vcpu, gfn, data, offset, seg);
2141 if (ret < 0)
2142 return ret;
2143 offset = 0;
2144 len -= seg;
2145 data += seg;
2146 ++gfn;
2147 }
2148 return 0;
2149}
2150EXPORT_SYMBOL_GPL(kvm_vcpu_write_guest);
2151
Paolo Bonzini5a2d4362017-02-03 20:32:28 -08002152static int __kvm_gfn_to_hva_cache_init(struct kvm_memslots *slots,
2153 struct gfn_to_hva_cache *ghc,
2154 gpa_t gpa, unsigned long len)
Gleb Natapov49c77542010-10-18 15:22:23 +02002155{
Gleb Natapov49c77542010-10-18 15:22:23 +02002156 int offset = offset_in_page(gpa);
Andrew Honig8f964522013-03-29 09:35:21 -07002157 gfn_t start_gfn = gpa >> PAGE_SHIFT;
2158 gfn_t end_gfn = (gpa + len - 1) >> PAGE_SHIFT;
2159 gfn_t nr_pages_needed = end_gfn - start_gfn + 1;
2160 gfn_t nr_pages_avail;
Jim Mattsonf1b9dd52018-12-17 13:53:33 -08002161 int r = start_gfn <= end_gfn ? 0 : -EINVAL;
Gleb Natapov49c77542010-10-18 15:22:23 +02002162
2163 ghc->gpa = gpa;
2164 ghc->generation = slots->generation;
Andrew Honig8f964522013-03-29 09:35:21 -07002165 ghc->len = len;
Jim Mattsonf1b9dd52018-12-17 13:53:33 -08002166 ghc->hva = KVM_HVA_ERR_BAD;
2167
2168 /*
2169 * If the requested region crosses two memslots, we still
2170 * verify that the entire region is valid here.
2171 */
2172 while (!r && start_gfn <= end_gfn) {
2173 ghc->memslot = __gfn_to_memslot(slots, start_gfn);
2174 ghc->hva = gfn_to_hva_many(ghc->memslot, start_gfn,
2175 &nr_pages_avail);
2176 if (kvm_is_error_hva(ghc->hva))
2177 r = -EFAULT;
2178 start_gfn += nr_pages_avail;
Andrew Honig8f964522013-03-29 09:35:21 -07002179 }
Jim Mattsonf1b9dd52018-12-17 13:53:33 -08002180
2181 /* Use the slow path for cross page reads and writes. */
2182 if (!r && nr_pages_needed == 1)
2183 ghc->hva += offset;
2184 else
2185 ghc->memslot = NULL;
2186
2187 return r;
Gleb Natapov49c77542010-10-18 15:22:23 +02002188}
Paolo Bonzini5a2d4362017-02-03 20:32:28 -08002189
Paolo Bonzini4e335d92017-05-02 16:20:18 +02002190int kvm_gfn_to_hva_cache_init(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
Paolo Bonzini5a2d4362017-02-03 20:32:28 -08002191 gpa_t gpa, unsigned long len)
2192{
Paolo Bonzini4e335d92017-05-02 16:20:18 +02002193 struct kvm_memslots *slots = kvm_memslots(kvm);
Paolo Bonzini5a2d4362017-02-03 20:32:28 -08002194 return __kvm_gfn_to_hva_cache_init(slots, ghc, gpa, len);
2195}
Paolo Bonzini4e335d92017-05-02 16:20:18 +02002196EXPORT_SYMBOL_GPL(kvm_gfn_to_hva_cache_init);
Gleb Natapov49c77542010-10-18 15:22:23 +02002197
Paolo Bonzini4e335d92017-05-02 16:20:18 +02002198int kvm_write_guest_offset_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
Jim Mattson7a86dab2018-12-14 14:34:43 -08002199 void *data, unsigned int offset,
2200 unsigned long len)
Gleb Natapov49c77542010-10-18 15:22:23 +02002201{
Paolo Bonzini4e335d92017-05-02 16:20:18 +02002202 struct kvm_memslots *slots = kvm_memslots(kvm);
Gleb Natapov49c77542010-10-18 15:22:23 +02002203 int r;
Pan Xinhui4ec6e862016-11-02 05:08:34 -04002204 gpa_t gpa = ghc->gpa + offset;
Gleb Natapov49c77542010-10-18 15:22:23 +02002205
Pan Xinhui4ec6e862016-11-02 05:08:34 -04002206 BUG_ON(len + offset > ghc->len);
Andrew Honig8f964522013-03-29 09:35:21 -07002207
Gleb Natapov49c77542010-10-18 15:22:23 +02002208 if (slots->generation != ghc->generation)
Paolo Bonzini5a2d4362017-02-03 20:32:28 -08002209 __kvm_gfn_to_hva_cache_init(slots, ghc, ghc->gpa, ghc->len);
Andrew Honig8f964522013-03-29 09:35:21 -07002210
2211 if (unlikely(!ghc->memslot))
Paolo Bonzini4e335d92017-05-02 16:20:18 +02002212 return kvm_write_guest(kvm, gpa, data, len);
Gleb Natapov49c77542010-10-18 15:22:23 +02002213
2214 if (kvm_is_error_hva(ghc->hva))
2215 return -EFAULT;
2216
Pan Xinhui4ec6e862016-11-02 05:08:34 -04002217 r = __copy_to_user((void __user *)ghc->hva + offset, data, len);
Gleb Natapov49c77542010-10-18 15:22:23 +02002218 if (r)
2219 return -EFAULT;
Pan Xinhui4ec6e862016-11-02 05:08:34 -04002220 mark_page_dirty_in_slot(ghc->memslot, gpa >> PAGE_SHIFT);
Gleb Natapov49c77542010-10-18 15:22:23 +02002221
2222 return 0;
2223}
Paolo Bonzini4e335d92017-05-02 16:20:18 +02002224EXPORT_SYMBOL_GPL(kvm_write_guest_offset_cached);
Pan Xinhui4ec6e862016-11-02 05:08:34 -04002225
Paolo Bonzini4e335d92017-05-02 16:20:18 +02002226int kvm_write_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
2227 void *data, unsigned long len)
Pan Xinhui4ec6e862016-11-02 05:08:34 -04002228{
Paolo Bonzini4e335d92017-05-02 16:20:18 +02002229 return kvm_write_guest_offset_cached(kvm, ghc, data, 0, len);
Pan Xinhui4ec6e862016-11-02 05:08:34 -04002230}
Paolo Bonzini4e335d92017-05-02 16:20:18 +02002231EXPORT_SYMBOL_GPL(kvm_write_guest_cached);
Gleb Natapov49c77542010-10-18 15:22:23 +02002232
Paolo Bonzini4e335d92017-05-02 16:20:18 +02002233int kvm_read_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
2234 void *data, unsigned long len)
Gleb Natapove03b6442011-07-11 15:28:11 -04002235{
Paolo Bonzini4e335d92017-05-02 16:20:18 +02002236 struct kvm_memslots *slots = kvm_memslots(kvm);
Gleb Natapove03b6442011-07-11 15:28:11 -04002237 int r;
2238
Andrew Honig8f964522013-03-29 09:35:21 -07002239 BUG_ON(len > ghc->len);
2240
Gleb Natapove03b6442011-07-11 15:28:11 -04002241 if (slots->generation != ghc->generation)
Paolo Bonzini5a2d4362017-02-03 20:32:28 -08002242 __kvm_gfn_to_hva_cache_init(slots, ghc, ghc->gpa, ghc->len);
Andrew Honig8f964522013-03-29 09:35:21 -07002243
2244 if (unlikely(!ghc->memslot))
Paolo Bonzini4e335d92017-05-02 16:20:18 +02002245 return kvm_read_guest(kvm, ghc->gpa, data, len);
Gleb Natapove03b6442011-07-11 15:28:11 -04002246
2247 if (kvm_is_error_hva(ghc->hva))
2248 return -EFAULT;
2249
2250 r = __copy_from_user(data, (void __user *)ghc->hva, len);
2251 if (r)
2252 return -EFAULT;
2253
2254 return 0;
2255}
Paolo Bonzini4e335d92017-05-02 16:20:18 +02002256EXPORT_SYMBOL_GPL(kvm_read_guest_cached);
Gleb Natapove03b6442011-07-11 15:28:11 -04002257
Izik Eidus195aefd2007-10-01 22:14:18 +02002258int kvm_clear_guest_page(struct kvm *kvm, gfn_t gfn, int offset, int len)
2259{
Heiko Carstens8a3caa62013-11-18 10:35:55 +01002260 const void *zero_page = (const void *) __va(page_to_phys(ZERO_PAGE(0)));
2261
2262 return kvm_write_guest_page(kvm, gfn, zero_page, offset, len);
Izik Eidus195aefd2007-10-01 22:14:18 +02002263}
2264EXPORT_SYMBOL_GPL(kvm_clear_guest_page);
2265
2266int kvm_clear_guest(struct kvm *kvm, gpa_t gpa, unsigned long len)
2267{
2268 gfn_t gfn = gpa >> PAGE_SHIFT;
2269 int seg;
2270 int offset = offset_in_page(gpa);
2271 int ret;
2272
Kevin Mulveybfda0e82015-02-20 08:21:36 -05002273 while ((seg = next_segment(len, offset)) != 0) {
Izik Eidus195aefd2007-10-01 22:14:18 +02002274 ret = kvm_clear_guest_page(kvm, gfn, offset, seg);
2275 if (ret < 0)
2276 return ret;
2277 offset = 0;
2278 len -= seg;
2279 ++gfn;
2280 }
2281 return 0;
2282}
2283EXPORT_SYMBOL_GPL(kvm_clear_guest);
2284
Paolo Bonzinibc009e42015-05-26 12:43:41 +02002285static void mark_page_dirty_in_slot(struct kvm_memory_slot *memslot,
Stephen Hemminger79408762013-12-29 12:12:29 -08002286 gfn_t gfn)
Avi Kivity6aa8b732006-12-10 02:21:36 -08002287{
Rusty Russell7e9d6192007-07-31 20:41:14 +10002288 if (memslot && memslot->dirty_bitmap) {
2289 unsigned long rel_gfn = gfn - memslot->base_gfn;
Avi Kivity6aa8b732006-12-10 02:21:36 -08002290
Takuya Yoshikawab74ca3b2012-10-04 17:13:12 -07002291 set_bit_le(rel_gfn, memslot->dirty_bitmap);
Avi Kivity6aa8b732006-12-10 02:21:36 -08002292 }
2293}
2294
Gleb Natapov49c77542010-10-18 15:22:23 +02002295void mark_page_dirty(struct kvm *kvm, gfn_t gfn)
2296{
2297 struct kvm_memory_slot *memslot;
2298
2299 memslot = gfn_to_memslot(kvm, gfn);
Paolo Bonzinibc009e42015-05-26 12:43:41 +02002300 mark_page_dirty_in_slot(memslot, gfn);
Gleb Natapov49c77542010-10-18 15:22:23 +02002301}
Aneesh Kumar K.V2ba9f0d2013-10-07 22:17:59 +05302302EXPORT_SYMBOL_GPL(mark_page_dirty);
Gleb Natapov49c77542010-10-18 15:22:23 +02002303
Paolo Bonzini8e734852015-05-17 13:58:53 +02002304void kvm_vcpu_mark_page_dirty(struct kvm_vcpu *vcpu, gfn_t gfn)
2305{
2306 struct kvm_memory_slot *memslot;
2307
2308 memslot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
2309 mark_page_dirty_in_slot(memslot, gfn);
2310}
2311EXPORT_SYMBOL_GPL(kvm_vcpu_mark_page_dirty);
2312
Jan H. Schönherr20b70352017-11-24 22:39:01 +01002313void kvm_sigset_activate(struct kvm_vcpu *vcpu)
2314{
2315 if (!vcpu->sigset_active)
2316 return;
2317
2318 /*
2319 * This does a lockless modification of ->real_blocked, which is fine
2320 * because, only current can change ->real_blocked and all readers of
2321 * ->real_blocked don't care as long ->real_blocked is always a subset
2322 * of ->blocked.
2323 */
2324 sigprocmask(SIG_SETMASK, &vcpu->sigset, &current->real_blocked);
2325}
2326
2327void kvm_sigset_deactivate(struct kvm_vcpu *vcpu)
2328{
2329 if (!vcpu->sigset_active)
2330 return;
2331
2332 sigprocmask(SIG_SETMASK, &current->real_blocked, NULL);
2333 sigemptyset(&current->real_blocked);
2334}
2335
Wanpeng Liaca6ff22015-09-03 22:07:38 +08002336static void grow_halt_poll_ns(struct kvm_vcpu *vcpu)
2337{
Nir Weinerdee339b2019-01-27 12:17:16 +02002338 unsigned int old, val, grow, grow_start;
Wanpeng Liaca6ff22015-09-03 22:07:38 +08002339
Wanpeng Li2cbd7822015-09-03 22:07:39 +08002340 old = val = vcpu->halt_poll_ns;
Nir Weinerdee339b2019-01-27 12:17:16 +02002341 grow_start = READ_ONCE(halt_poll_ns_grow_start);
Christian Borntraeger6b6de682016-02-09 13:47:55 +01002342 grow = READ_ONCE(halt_poll_ns_grow);
Nir Weiner7fa08e72019-01-27 12:17:14 +02002343 if (!grow)
2344 goto out;
2345
Nir Weinerdee339b2019-01-27 12:17:16 +02002346 val *= grow;
2347 if (val < grow_start)
2348 val = grow_start;
Wanpeng Liaca6ff22015-09-03 22:07:38 +08002349
David Matlack313f6362016-03-08 16:19:44 -08002350 if (val > halt_poll_ns)
2351 val = halt_poll_ns;
2352
Wanpeng Liaca6ff22015-09-03 22:07:38 +08002353 vcpu->halt_poll_ns = val;
Nir Weiner7fa08e72019-01-27 12:17:14 +02002354out:
Wanpeng Li2cbd7822015-09-03 22:07:39 +08002355 trace_kvm_halt_poll_ns_grow(vcpu->vcpu_id, val, old);
Wanpeng Liaca6ff22015-09-03 22:07:38 +08002356}
2357
2358static void shrink_halt_poll_ns(struct kvm_vcpu *vcpu)
2359{
Christian Borntraeger6b6de682016-02-09 13:47:55 +01002360 unsigned int old, val, shrink;
Wanpeng Liaca6ff22015-09-03 22:07:38 +08002361
Wanpeng Li2cbd7822015-09-03 22:07:39 +08002362 old = val = vcpu->halt_poll_ns;
Christian Borntraeger6b6de682016-02-09 13:47:55 +01002363 shrink = READ_ONCE(halt_poll_ns_shrink);
2364 if (shrink == 0)
Wanpeng Liaca6ff22015-09-03 22:07:38 +08002365 val = 0;
2366 else
Christian Borntraeger6b6de682016-02-09 13:47:55 +01002367 val /= shrink;
Wanpeng Liaca6ff22015-09-03 22:07:38 +08002368
2369 vcpu->halt_poll_ns = val;
Wanpeng Li2cbd7822015-09-03 22:07:39 +08002370 trace_kvm_halt_poll_ns_shrink(vcpu->vcpu_id, val, old);
Wanpeng Liaca6ff22015-09-03 22:07:38 +08002371}
2372
Paolo Bonzinif7819512015-02-04 18:20:58 +01002373static int kvm_vcpu_check_block(struct kvm_vcpu *vcpu)
2374{
Junaid Shahid50c28f22018-06-27 14:59:11 -07002375 int ret = -EINTR;
2376 int idx = srcu_read_lock(&vcpu->kvm->srcu);
2377
Paolo Bonzinif7819512015-02-04 18:20:58 +01002378 if (kvm_arch_vcpu_runnable(vcpu)) {
2379 kvm_make_request(KVM_REQ_UNHALT, vcpu);
Junaid Shahid50c28f22018-06-27 14:59:11 -07002380 goto out;
Paolo Bonzinif7819512015-02-04 18:20:58 +01002381 }
2382 if (kvm_cpu_has_pending_timer(vcpu))
Junaid Shahid50c28f22018-06-27 14:59:11 -07002383 goto out;
Paolo Bonzinif7819512015-02-04 18:20:58 +01002384 if (signal_pending(current))
Junaid Shahid50c28f22018-06-27 14:59:11 -07002385 goto out;
Paolo Bonzinif7819512015-02-04 18:20:58 +01002386
Junaid Shahid50c28f22018-06-27 14:59:11 -07002387 ret = 0;
2388out:
2389 srcu_read_unlock(&vcpu->kvm->srcu, idx);
2390 return ret;
Paolo Bonzinif7819512015-02-04 18:20:58 +01002391}
2392
Eddie Dongb6958ce2007-07-18 12:15:21 +03002393/*
2394 * The vCPU has executed a HLT instruction with in-kernel mode enabled.
2395 */
Hollis Blanchard8776e512007-10-31 17:24:24 -05002396void kvm_vcpu_block(struct kvm_vcpu *vcpu)
Eddie Dongb6958ce2007-07-18 12:15:21 +03002397{
Paolo Bonzinif7819512015-02-04 18:20:58 +01002398 ktime_t start, cur;
Marcelo Tosatti85773702016-02-19 09:46:39 +01002399 DECLARE_SWAITQUEUE(wait);
Paolo Bonzinif7819512015-02-04 18:20:58 +01002400 bool waited = false;
Wanpeng Liaca6ff22015-09-03 22:07:38 +08002401 u64 block_ns;
Paolo Bonzinif7819512015-02-04 18:20:58 +01002402
Marc Zyngier07ab0f82019-08-02 11:37:09 +01002403 kvm_arch_vcpu_blocking(vcpu);
2404
Paolo Bonzinif7819512015-02-04 18:20:58 +01002405 start = cur = ktime_get();
Christian Borntraegercdd6ad32019-03-05 05:30:01 -05002406 if (vcpu->halt_poll_ns && !kvm_arch_no_poll(vcpu)) {
Wanpeng Li19020f82015-09-03 22:07:37 +08002407 ktime_t stop = ktime_add_ns(ktime_get(), vcpu->halt_poll_ns);
Xiubo Lif95ef0cd2015-02-26 14:58:23 +08002408
Paolo Bonzini62bea5b2015-09-15 18:27:57 +02002409 ++vcpu->stat.halt_attempted_poll;
Paolo Bonzinif7819512015-02-04 18:20:58 +01002410 do {
2411 /*
2412 * This sets KVM_REQ_UNHALT if an interrupt
2413 * arrives.
2414 */
2415 if (kvm_vcpu_check_block(vcpu) < 0) {
2416 ++vcpu->stat.halt_successful_poll;
Christian Borntraeger3491caf2016-05-13 12:16:35 +02002417 if (!vcpu_valid_wakeup(vcpu))
2418 ++vcpu->stat.halt_poll_invalid;
Paolo Bonzinif7819512015-02-04 18:20:58 +01002419 goto out;
2420 }
2421 cur = ktime_get();
2422 } while (single_task_running() && ktime_before(cur, stop));
2423 }
Eddie Dongb6958ce2007-07-18 12:15:21 +03002424
Marcelo Tosattie5c239c2008-05-08 19:47:01 -03002425 for (;;) {
Peter Zijlstrab3dae102018-06-12 10:34:52 +02002426 prepare_to_swait_exclusive(&vcpu->wq, &wait, TASK_INTERRUPTIBLE);
Eddie Dongb6958ce2007-07-18 12:15:21 +03002427
Paolo Bonzinif7819512015-02-04 18:20:58 +01002428 if (kvm_vcpu_check_block(vcpu) < 0)
Marcelo Tosattie5c239c2008-05-08 19:47:01 -03002429 break;
2430
Paolo Bonzinif7819512015-02-04 18:20:58 +01002431 waited = true;
Eddie Dongb6958ce2007-07-18 12:15:21 +03002432 schedule();
Eddie Dongb6958ce2007-07-18 12:15:21 +03002433 }
2434
Marcelo Tosatti85773702016-02-19 09:46:39 +01002435 finish_swait(&vcpu->wq, &wait);
Paolo Bonzinif7819512015-02-04 18:20:58 +01002436 cur = ktime_get();
Paolo Bonzinif7819512015-02-04 18:20:58 +01002437out:
Marc Zyngier07ab0f82019-08-02 11:37:09 +01002438 kvm_arch_vcpu_unblocking(vcpu);
Wanpeng Liaca6ff22015-09-03 22:07:38 +08002439 block_ns = ktime_to_ns(cur) - ktime_to_ns(start);
2440
Wanpeng Li44551b22019-09-29 09:06:56 +08002441 if (!kvm_arch_no_poll(vcpu)) {
2442 if (!vcpu_valid_wakeup(vcpu)) {
Wanpeng Liaca6ff22015-09-03 22:07:38 +08002443 shrink_halt_poll_ns(vcpu);
Wanpeng Li44551b22019-09-29 09:06:56 +08002444 } else if (halt_poll_ns) {
2445 if (block_ns <= vcpu->halt_poll_ns)
2446 ;
2447 /* we had a long block, shrink polling */
2448 else if (vcpu->halt_poll_ns && block_ns > halt_poll_ns)
2449 shrink_halt_poll_ns(vcpu);
2450 /* we had a short halt and our poll time is too small */
2451 else if (vcpu->halt_poll_ns < halt_poll_ns &&
2452 block_ns < halt_poll_ns)
2453 grow_halt_poll_ns(vcpu);
2454 } else {
2455 vcpu->halt_poll_ns = 0;
2456 }
2457 }
Wanpeng Liaca6ff22015-09-03 22:07:38 +08002458
Christian Borntraeger3491caf2016-05-13 12:16:35 +02002459 trace_kvm_vcpu_wakeup(block_ns, waited, vcpu_valid_wakeup(vcpu));
2460 kvm_arch_vcpu_block_finish(vcpu);
Eddie Dongb6958ce2007-07-18 12:15:21 +03002461}
Aneesh Kumar K.V2ba9f0d2013-10-07 22:17:59 +05302462EXPORT_SYMBOL_GPL(kvm_vcpu_block);
Eddie Dongb6958ce2007-07-18 12:15:21 +03002463
Radim Krčmář178f02f2017-04-26 22:32:26 +02002464bool kvm_vcpu_wake_up(struct kvm_vcpu *vcpu)
Christoffer Dallb6d33832012-03-08 16:44:24 -05002465{
Marcelo Tosatti85773702016-02-19 09:46:39 +01002466 struct swait_queue_head *wqp;
Christoffer Dallb6d33832012-03-08 16:44:24 -05002467
2468 wqp = kvm_arch_vcpu_wq(vcpu);
Davidlohr Bueso5e0018b2017-09-13 13:08:22 -07002469 if (swq_has_sleeper(wqp)) {
Peter Zijlstrab3dae102018-06-12 10:34:52 +02002470 swake_up_one(wqp);
Wanpeng Lid73eb572019-07-18 19:39:06 +08002471 WRITE_ONCE(vcpu->ready, true);
Christoffer Dallb6d33832012-03-08 16:44:24 -05002472 ++vcpu->stat.halt_wakeup;
Radim Krčmář178f02f2017-04-26 22:32:26 +02002473 return true;
Christoffer Dallb6d33832012-03-08 16:44:24 -05002474 }
2475
Radim Krčmář178f02f2017-04-26 22:32:26 +02002476 return false;
Radim Krčmářdd1a4cc2016-05-04 14:09:44 -05002477}
2478EXPORT_SYMBOL_GPL(kvm_vcpu_wake_up);
2479
Paolo Bonzini0266c892017-05-04 15:14:13 +02002480#ifndef CONFIG_S390
Radim Krčmářdd1a4cc2016-05-04 14:09:44 -05002481/*
2482 * Kick a sleeping VCPU, or a guest VCPU in guest mode, into host kernel mode.
2483 */
2484void kvm_vcpu_kick(struct kvm_vcpu *vcpu)
2485{
2486 int me;
2487 int cpu = vcpu->cpu;
2488
Radim Krčmář178f02f2017-04-26 22:32:26 +02002489 if (kvm_vcpu_wake_up(vcpu))
2490 return;
2491
Christoffer Dallb6d33832012-03-08 16:44:24 -05002492 me = get_cpu();
2493 if (cpu != me && (unsigned)cpu < nr_cpu_ids && cpu_online(cpu))
2494 if (kvm_arch_vcpu_should_kick(vcpu))
2495 smp_send_reschedule(cpu);
2496 put_cpu();
2497}
Yang Zhanga20ed542013-04-11 19:25:15 +08002498EXPORT_SYMBOL_GPL(kvm_vcpu_kick);
Paolo Bonzini0266c892017-05-04 15:14:13 +02002499#endif /* !CONFIG_S390 */
Christoffer Dallb6d33832012-03-08 16:44:24 -05002500
Dan Carpenterfa933842014-05-23 13:20:42 +03002501int kvm_vcpu_yield_to(struct kvm_vcpu *target)
Konstantin Weitz41628d32012-04-25 15:30:38 +02002502{
2503 struct pid *pid;
2504 struct task_struct *task = NULL;
Dan Carpenterfa933842014-05-23 13:20:42 +03002505 int ret = 0;
Konstantin Weitz41628d32012-04-25 15:30:38 +02002506
2507 rcu_read_lock();
2508 pid = rcu_dereference(target->pid);
2509 if (pid)
Sam Bobroff27fbe64b2014-09-19 09:40:41 +10002510 task = get_pid_task(pid, PIDTYPE_PID);
Konstantin Weitz41628d32012-04-25 15:30:38 +02002511 rcu_read_unlock();
2512 if (!task)
Raghavendra K Tc45c5282013-01-22 13:09:24 +05302513 return ret;
Raghavendra K Tc45c5282013-01-22 13:09:24 +05302514 ret = yield_to(task, 1);
Konstantin Weitz41628d32012-04-25 15:30:38 +02002515 put_task_struct(task);
Raghavendra K Tc45c5282013-01-22 13:09:24 +05302516
2517 return ret;
Konstantin Weitz41628d32012-04-25 15:30:38 +02002518}
2519EXPORT_SYMBOL_GPL(kvm_vcpu_yield_to);
2520
Raghavendra K T06e48c52012-07-19 15:17:52 +05302521/*
2522 * Helper that checks whether a VCPU is eligible for directed yield.
2523 * Most eligible candidate to yield is decided by following heuristics:
2524 *
2525 * (a) VCPU which has not done pl-exit or cpu relax intercepted recently
2526 * (preempted lock holder), indicated by @in_spin_loop.
2527 * Set at the beiginning and cleared at the end of interception/PLE handler.
2528 *
2529 * (b) VCPU which has done pl-exit/ cpu relax intercepted but did not get
2530 * chance last time (mostly it has become eligible now since we have probably
2531 * yielded to lockholder in last iteration. This is done by toggling
2532 * @dy_eligible each time a VCPU checked for eligibility.)
2533 *
2534 * Yielding to a recently pl-exited/cpu relax intercepted VCPU before yielding
2535 * to preempted lock-holder could result in wrong VCPU selection and CPU
2536 * burning. Giving priority for a potential lock-holder increases lock
2537 * progress.
2538 *
2539 * Since algorithm is based on heuristics, accessing another VCPU data without
2540 * locking does not harm. It may result in trying to yield to same VCPU, fail
2541 * and continue with next VCPU and so on.
2542 */
Stephen Hemminger79408762013-12-29 12:12:29 -08002543static bool kvm_vcpu_eligible_for_directed_yield(struct kvm_vcpu *vcpu)
Raghavendra K T06e48c52012-07-19 15:17:52 +05302544{
Scott Wood4a55dd72014-01-09 18:43:16 -06002545#ifdef CONFIG_HAVE_KVM_CPU_RELAX_INTERCEPT
Raghavendra K T06e48c52012-07-19 15:17:52 +05302546 bool eligible;
2547
2548 eligible = !vcpu->spin_loop.in_spin_loop ||
Christian Borntraeger34656112014-09-04 21:13:31 +02002549 vcpu->spin_loop.dy_eligible;
Raghavendra K T06e48c52012-07-19 15:17:52 +05302550
2551 if (vcpu->spin_loop.in_spin_loop)
2552 kvm_vcpu_set_dy_eligible(vcpu, !vcpu->spin_loop.dy_eligible);
2553
2554 return eligible;
Scott Wood4a55dd72014-01-09 18:43:16 -06002555#else
2556 return true;
Raghavendra K T06e48c52012-07-19 15:17:52 +05302557#endif
Scott Wood4a55dd72014-01-09 18:43:16 -06002558}
Raghavendra K Tc45c5282013-01-22 13:09:24 +05302559
Wanpeng Li17e433b2019-08-05 10:03:19 +08002560/*
2561 * Unlike kvm_arch_vcpu_runnable, this function is called outside
2562 * a vcpu_load/vcpu_put pair. However, for most architectures
2563 * kvm_arch_vcpu_runnable does not require vcpu_load.
2564 */
2565bool __weak kvm_arch_dy_runnable(struct kvm_vcpu *vcpu)
2566{
2567 return kvm_arch_vcpu_runnable(vcpu);
2568}
2569
2570static bool vcpu_dy_runnable(struct kvm_vcpu *vcpu)
2571{
2572 if (kvm_arch_dy_runnable(vcpu))
2573 return true;
2574
2575#ifdef CONFIG_KVM_ASYNC_PF
2576 if (!list_empty_careful(&vcpu->async_pf.done))
2577 return true;
2578#endif
2579
2580 return false;
2581}
2582
Longpeng(Mike)199b5762017-08-08 12:05:32 +08002583void kvm_vcpu_on_spin(struct kvm_vcpu *me, bool yield_to_kernel_mode)
Zhai, Edwind255f4f2009-10-09 18:03:20 +08002584{
Rik van Riel217ece62011-02-01 09:53:28 -05002585 struct kvm *kvm = me->kvm;
2586 struct kvm_vcpu *vcpu;
2587 int last_boosted_vcpu = me->kvm->last_boosted_vcpu;
2588 int yielded = 0;
Raghavendra K Tc45c5282013-01-22 13:09:24 +05302589 int try = 3;
Rik van Riel217ece62011-02-01 09:53:28 -05002590 int pass;
2591 int i;
Zhai, Edwind255f4f2009-10-09 18:03:20 +08002592
Raghavendra K T4c088492012-07-18 19:07:46 +05302593 kvm_vcpu_set_in_spin_loop(me, true);
Rik van Riel217ece62011-02-01 09:53:28 -05002594 /*
2595 * We boost the priority of a VCPU that is runnable but not
2596 * currently running, because it got preempted by something
2597 * else and called schedule in __vcpu_run. Hopefully that
2598 * VCPU is holding the lock that we need and will release it.
2599 * We approximate round-robin by starting at the last boosted VCPU.
2600 */
Raghavendra K Tc45c5282013-01-22 13:09:24 +05302601 for (pass = 0; pass < 2 && !yielded && try; pass++) {
Rik van Riel217ece62011-02-01 09:53:28 -05002602 kvm_for_each_vcpu(i, vcpu, kvm) {
Rik van Riel5cfc2aa2012-06-19 16:51:04 -04002603 if (!pass && i <= last_boosted_vcpu) {
Rik van Riel217ece62011-02-01 09:53:28 -05002604 i = last_boosted_vcpu;
2605 continue;
2606 } else if (pass && i > last_boosted_vcpu)
2607 break;
Wanpeng Lid73eb572019-07-18 19:39:06 +08002608 if (!READ_ONCE(vcpu->ready))
Raghavendra K T7bc7ae252013-03-04 23:32:27 +05302609 continue;
Rik van Riel217ece62011-02-01 09:53:28 -05002610 if (vcpu == me)
2611 continue;
Wanpeng Li17e433b2019-08-05 10:03:19 +08002612 if (swait_active(&vcpu->wq) && !vcpu_dy_runnable(vcpu))
Rik van Riel217ece62011-02-01 09:53:28 -05002613 continue;
Wanpeng Li046ddee2019-08-01 11:30:14 +08002614 if (READ_ONCE(vcpu->preempted) && yield_to_kernel_mode &&
2615 !kvm_arch_vcpu_in_kernel(vcpu))
Longpeng(Mike)199b5762017-08-08 12:05:32 +08002616 continue;
Raghavendra K T06e48c52012-07-19 15:17:52 +05302617 if (!kvm_vcpu_eligible_for_directed_yield(vcpu))
2618 continue;
Raghavendra K Tc45c5282013-01-22 13:09:24 +05302619
2620 yielded = kvm_vcpu_yield_to(vcpu);
2621 if (yielded > 0) {
Rik van Riel217ece62011-02-01 09:53:28 -05002622 kvm->last_boosted_vcpu = i;
Rik van Riel217ece62011-02-01 09:53:28 -05002623 break;
Raghavendra K Tc45c5282013-01-22 13:09:24 +05302624 } else if (yielded < 0) {
2625 try--;
2626 if (!try)
2627 break;
Rik van Riel217ece62011-02-01 09:53:28 -05002628 }
Rik van Riel217ece62011-02-01 09:53:28 -05002629 }
2630 }
Raghavendra K T4c088492012-07-18 19:07:46 +05302631 kvm_vcpu_set_in_spin_loop(me, false);
Raghavendra K T06e48c52012-07-19 15:17:52 +05302632
2633 /* Ensure vcpu is not eligible during next spinloop */
2634 kvm_vcpu_set_dy_eligible(me, false);
Zhai, Edwind255f4f2009-10-09 18:03:20 +08002635}
2636EXPORT_SYMBOL_GPL(kvm_vcpu_on_spin);
2637
Souptick Joarder1499fa82018-04-19 00:49:58 +05302638static vm_fault_t kvm_vcpu_fault(struct vm_fault *vmf)
Avi Kivity9a2bb7f2007-02-22 12:58:31 +02002639{
Dave Jiang11bac802017-02-24 14:56:41 -08002640 struct kvm_vcpu *vcpu = vmf->vma->vm_file->private_data;
Avi Kivity9a2bb7f2007-02-22 12:58:31 +02002641 struct page *page;
2642
npiggin@suse.dee4a533a2007-12-05 18:15:52 +11002643 if (vmf->pgoff == 0)
Avi Kivity039576c2007-03-20 12:46:50 +02002644 page = virt_to_page(vcpu->run);
Avi Kivity09566762008-01-23 18:14:23 +02002645#ifdef CONFIG_X86
npiggin@suse.dee4a533a2007-12-05 18:15:52 +11002646 else if (vmf->pgoff == KVM_PIO_PAGE_OFFSET)
Zhang Xiantaoad312c72007-12-13 23:50:52 +08002647 page = virt_to_page(vcpu->arch.pio_data);
Avi Kivity09566762008-01-23 18:14:23 +02002648#endif
Paolo Bonzini4b4357e2017-03-31 13:53:23 +02002649#ifdef CONFIG_KVM_MMIO
Laurent Vivier5f94c172008-05-30 16:05:54 +02002650 else if (vmf->pgoff == KVM_COALESCED_MMIO_PAGE_OFFSET)
2651 page = virt_to_page(vcpu->kvm->coalesced_mmio_ring);
2652#endif
Avi Kivity039576c2007-03-20 12:46:50 +02002653 else
Carsten Otte5b1c1492012-01-04 10:25:23 +01002654 return kvm_arch_vcpu_fault(vcpu, vmf);
Avi Kivity9a2bb7f2007-02-22 12:58:31 +02002655 get_page(page);
npiggin@suse.dee4a533a2007-12-05 18:15:52 +11002656 vmf->page = page;
2657 return 0;
Avi Kivity9a2bb7f2007-02-22 12:58:31 +02002658}
2659
Alexey Dobriyanf0f37e2f2009-09-27 22:29:37 +04002660static const struct vm_operations_struct kvm_vcpu_vm_ops = {
npiggin@suse.dee4a533a2007-12-05 18:15:52 +11002661 .fault = kvm_vcpu_fault,
Avi Kivity9a2bb7f2007-02-22 12:58:31 +02002662};
2663
2664static int kvm_vcpu_mmap(struct file *file, struct vm_area_struct *vma)
2665{
2666 vma->vm_ops = &kvm_vcpu_vm_ops;
2667 return 0;
2668}
2669
Avi Kivitybccf2152007-02-21 18:04:26 +02002670static int kvm_vcpu_release(struct inode *inode, struct file *filp)
2671{
2672 struct kvm_vcpu *vcpu = filp->private_data;
2673
Luiz Capitulino45b59392016-09-16 10:27:35 -04002674 debugfs_remove_recursive(vcpu->debugfs_dentry);
Al Viro66c0b392008-04-19 20:33:56 +01002675 kvm_put_kvm(vcpu->kvm);
Avi Kivitybccf2152007-02-21 18:04:26 +02002676 return 0;
2677}
2678
Christian Borntraeger3d3aab12008-12-02 11:17:32 +01002679static struct file_operations kvm_vcpu_fops = {
Avi Kivitybccf2152007-02-21 18:04:26 +02002680 .release = kvm_vcpu_release,
2681 .unlocked_ioctl = kvm_vcpu_ioctl,
Avi Kivity9a2bb7f2007-02-22 12:58:31 +02002682 .mmap = kvm_vcpu_mmap,
Arnd Bergmann6038f372010-08-15 18:52:59 +02002683 .llseek = noop_llseek,
Marc Zyngier7ddfd3e2018-06-17 10:16:21 +01002684 KVM_COMPAT(kvm_vcpu_compat_ioctl),
Avi Kivitybccf2152007-02-21 18:04:26 +02002685};
2686
2687/*
2688 * Allocates an inode for the vcpu.
2689 */
2690static int create_vcpu_fd(struct kvm_vcpu *vcpu)
2691{
Masatake YAMATOe46b4692018-01-20 04:04:22 +09002692 char name[8 + 1 + ITOA_MAX_LEN + 1];
2693
2694 snprintf(name, sizeof(name), "kvm-vcpu:%d", vcpu->vcpu_id);
2695 return anon_inode_getfd(name, &kvm_vcpu_fops, vcpu, O_RDWR | O_CLOEXEC);
Avi Kivitybccf2152007-02-21 18:04:26 +02002696}
2697
Greg KH3e7093d2019-07-31 20:56:20 +02002698static void kvm_create_vcpu_debugfs(struct kvm_vcpu *vcpu)
Luiz Capitulino45b59392016-09-16 10:27:35 -04002699{
Paolo Bonzini741cbba2019-08-03 08:14:25 +02002700#ifdef __KVM_HAVE_ARCH_VCPU_DEBUGFS
Luiz Capitulino45b59392016-09-16 10:27:35 -04002701 char dir_name[ITOA_MAX_LEN * 2];
Luiz Capitulino45b59392016-09-16 10:27:35 -04002702
Luiz Capitulino45b59392016-09-16 10:27:35 -04002703 if (!debugfs_initialized())
Greg KH3e7093d2019-07-31 20:56:20 +02002704 return;
Luiz Capitulino45b59392016-09-16 10:27:35 -04002705
2706 snprintf(dir_name, sizeof(dir_name), "vcpu%d", vcpu->vcpu_id);
2707 vcpu->debugfs_dentry = debugfs_create_dir(dir_name,
Greg KH3e7093d2019-07-31 20:56:20 +02002708 vcpu->kvm->debugfs_dentry);
Luiz Capitulino45b59392016-09-16 10:27:35 -04002709
Greg KH3e7093d2019-07-31 20:56:20 +02002710 kvm_arch_create_vcpu_debugfs(vcpu);
Paolo Bonzini741cbba2019-08-03 08:14:25 +02002711#endif
Luiz Capitulino45b59392016-09-16 10:27:35 -04002712}
2713
Avi Kivityc5ea7662007-02-20 18:41:05 +02002714/*
2715 * Creates some virtual cpus. Good luck creating more than one.
2716 */
Gleb Natapov73880c82009-06-09 15:56:28 +03002717static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, u32 id)
Avi Kivityc5ea7662007-02-20 18:41:05 +02002718{
2719 int r;
David Hildenbrande09fefd2015-11-05 09:03:50 +01002720 struct kvm_vcpu *vcpu;
Avi Kivityc5ea7662007-02-20 18:41:05 +02002721
Greg Kurz0b1b1df2016-05-09 18:13:37 +02002722 if (id >= KVM_MAX_VCPU_ID)
Andy Honig338c7db2013-11-18 16:09:22 -08002723 return -EINVAL;
2724
Paolo Bonzini6c7caeb2016-06-13 14:48:25 +02002725 mutex_lock(&kvm->lock);
2726 if (kvm->created_vcpus == KVM_MAX_VCPUS) {
2727 mutex_unlock(&kvm->lock);
2728 return -EINVAL;
2729 }
2730
2731 kvm->created_vcpus++;
2732 mutex_unlock(&kvm->lock);
2733
Gleb Natapov73880c82009-06-09 15:56:28 +03002734 vcpu = kvm_arch_vcpu_create(kvm, id);
Paolo Bonzini6c7caeb2016-06-13 14:48:25 +02002735 if (IS_ERR(vcpu)) {
2736 r = PTR_ERR(vcpu);
2737 goto vcpu_decrement;
2738 }
Avi Kivityc5ea7662007-02-20 18:41:05 +02002739
Avi Kivity15ad7142007-07-11 18:17:21 +03002740 preempt_notifier_init(&vcpu->preempt_notifier, &kvm_preempt_ops);
2741
Avi Kivity26e52152007-11-20 15:30:24 +02002742 r = kvm_arch_vcpu_setup(vcpu);
2743 if (r)
Jan Kiszkad7805922011-05-23 10:33:05 +02002744 goto vcpu_destroy;
Avi Kivity26e52152007-11-20 15:30:24 +02002745
Greg KH3e7093d2019-07-31 20:56:20 +02002746 kvm_create_vcpu_debugfs(vcpu);
Luiz Capitulino45b59392016-09-16 10:27:35 -04002747
Shaohua Li11ec2802007-07-23 14:51:37 +08002748 mutex_lock(&kvm->lock);
David Hildenbrande09fefd2015-11-05 09:03:50 +01002749 if (kvm_get_vcpu_by_id(kvm, id)) {
2750 r = -EEXIST;
2751 goto unlock_vcpu_destroy;
2752 }
Gleb Natapov73880c82009-06-09 15:56:28 +03002753
Radim Krčmář8750e722019-11-07 07:53:42 -05002754 vcpu->vcpu_idx = atomic_read(&kvm->online_vcpus);
2755 BUG_ON(kvm->vcpus[vcpu->vcpu_idx]);
Rusty Russellfb3f0f52007-07-27 17:16:56 +10002756
2757 /* Now it's all set up, let userspace reach it */
Al Viro66c0b392008-04-19 20:33:56 +01002758 kvm_get_kvm(kvm);
Avi Kivitybccf2152007-02-21 18:04:26 +02002759 r = create_vcpu_fd(vcpu);
Gleb Natapov73880c82009-06-09 15:56:28 +03002760 if (r < 0) {
Sean Christopherson149487b2019-10-21 15:58:42 -07002761 kvm_put_kvm_no_destroy(kvm);
Jan Kiszkad7805922011-05-23 10:33:05 +02002762 goto unlock_vcpu_destroy;
Gleb Natapov73880c82009-06-09 15:56:28 +03002763 }
2764
Radim Krčmář8750e722019-11-07 07:53:42 -05002765 kvm->vcpus[vcpu->vcpu_idx] = vcpu;
Paolo Bonzinidd489242015-07-29 11:32:20 +02002766
2767 /*
2768 * Pairs with smp_rmb() in kvm_get_vcpu. Write kvm->vcpus
2769 * before kvm->online_vcpu's incremented value.
2770 */
Gleb Natapov73880c82009-06-09 15:56:28 +03002771 smp_wmb();
2772 atomic_inc(&kvm->online_vcpus);
2773
Gleb Natapov73880c82009-06-09 15:56:28 +03002774 mutex_unlock(&kvm->lock);
Marcelo Tosatti42897d82012-11-27 23:29:02 -02002775 kvm_arch_vcpu_postcreate(vcpu);
Avi Kivitybccf2152007-02-21 18:04:26 +02002776 return r;
Avi Kivityc5ea7662007-02-20 18:41:05 +02002777
Jan Kiszkad7805922011-05-23 10:33:05 +02002778unlock_vcpu_destroy:
Glauber Costa7d8fece2008-09-17 23:16:59 -03002779 mutex_unlock(&kvm->lock);
Luiz Capitulino45b59392016-09-16 10:27:35 -04002780 debugfs_remove_recursive(vcpu->debugfs_dentry);
Jan Kiszkad7805922011-05-23 10:33:05 +02002781vcpu_destroy:
Hollis Blanchardd40ccc62007-11-19 14:04:43 -06002782 kvm_arch_vcpu_destroy(vcpu);
Paolo Bonzini6c7caeb2016-06-13 14:48:25 +02002783vcpu_decrement:
2784 mutex_lock(&kvm->lock);
2785 kvm->created_vcpus--;
2786 mutex_unlock(&kvm->lock);
Avi Kivityc5ea7662007-02-20 18:41:05 +02002787 return r;
2788}
2789
Avi Kivity1961d272007-03-05 19:46:05 +02002790static int kvm_vcpu_ioctl_set_sigmask(struct kvm_vcpu *vcpu, sigset_t *sigset)
2791{
2792 if (sigset) {
2793 sigdelsetmask(sigset, sigmask(SIGKILL)|sigmask(SIGSTOP));
2794 vcpu->sigset_active = 1;
2795 vcpu->sigset = *sigset;
2796 } else
2797 vcpu->sigset_active = 0;
2798 return 0;
2799}
2800
Avi Kivitybccf2152007-02-21 18:04:26 +02002801static long kvm_vcpu_ioctl(struct file *filp,
2802 unsigned int ioctl, unsigned long arg)
Avi Kivity6aa8b732006-12-10 02:21:36 -08002803{
Avi Kivitybccf2152007-02-21 18:04:26 +02002804 struct kvm_vcpu *vcpu = filp->private_data;
Al Viro2f3669872007-02-09 16:38:35 +00002805 void __user *argp = (void __user *)arg;
Carsten Otte313a3dc2007-10-11 19:16:52 +02002806 int r;
Dave Hansenfa3795a2008-08-11 10:01:46 -07002807 struct kvm_fpu *fpu = NULL;
2808 struct kvm_sregs *kvm_sregs = NULL;
Avi Kivity6aa8b732006-12-10 02:21:36 -08002809
Avi Kivity6d4e4c42007-11-21 16:41:05 +02002810 if (vcpu->kvm->mm != current->mm)
2811 return -EIO;
Avi Kivity2122ff52010-05-13 11:25:04 +03002812
David Matlack2ea75be2014-09-19 16:03:25 -07002813 if (unlikely(_IOC_TYPE(ioctl) != KVMIO))
2814 return -EINVAL;
2815
Avi Kivity2122ff52010-05-13 11:25:04 +03002816 /*
Paolo Bonzini5cb09442017-12-12 17:41:34 +01002817 * Some architectures have vcpu ioctls that are asynchronous to vcpu
2818 * execution; mutex_lock() would break them.
Avi Kivity2122ff52010-05-13 11:25:04 +03002819 */
Paolo Bonzini5cb09442017-12-12 17:41:34 +01002820 r = kvm_arch_vcpu_async_ioctl(filp, ioctl, arg);
2821 if (r != -ENOIOCTLCMD)
Michael S. Tsirkin9fc77442012-09-16 11:50:30 +03002822 return r;
Avi Kivity2122ff52010-05-13 11:25:04 +03002823
Christoffer Dallec7660c2017-12-04 21:35:23 +01002824 if (mutex_lock_killable(&vcpu->mutex))
2825 return -EINTR;
Avi Kivity6aa8b732006-12-10 02:21:36 -08002826 switch (ioctl) {
Christian Borntraeger0e4524a2017-07-06 14:44:28 +02002827 case KVM_RUN: {
2828 struct pid *oldpid;
Avi Kivityf0fe5102007-03-07 13:11:17 +02002829 r = -EINVAL;
2830 if (arg)
2831 goto out;
Christian Borntraeger0e4524a2017-07-06 14:44:28 +02002832 oldpid = rcu_access_pointer(vcpu->pid);
Eric W. Biederman71dbc8a2017-07-16 21:39:32 -05002833 if (unlikely(oldpid != task_pid(current))) {
Christian Borntraeger7a72f7a2014-08-05 16:44:14 +02002834 /* The thread running this VCPU changed. */
Christoffer Dallbd2a6392018-02-23 17:23:57 +01002835 struct pid *newpid;
Xiubo Lif95ef0cd2015-02-26 14:58:23 +08002836
Christoffer Dallbd2a6392018-02-23 17:23:57 +01002837 r = kvm_arch_vcpu_run_pid_change(vcpu);
2838 if (r)
2839 break;
2840
2841 newpid = get_task_pid(current, PIDTYPE_PID);
Christian Borntraeger7a72f7a2014-08-05 16:44:14 +02002842 rcu_assign_pointer(vcpu->pid, newpid);
2843 if (oldpid)
2844 synchronize_rcu();
2845 put_pid(oldpid);
2846 }
Hollis Blanchardb6c7a5d2007-11-01 14:16:10 -05002847 r = kvm_arch_vcpu_ioctl_run(vcpu, vcpu->run);
Gleb Natapov64be5002010-10-24 16:49:08 +02002848 trace_kvm_userspace_exit(vcpu->run->exit_reason, r);
Avi Kivity6aa8b732006-12-10 02:21:36 -08002849 break;
Christian Borntraeger0e4524a2017-07-06 14:44:28 +02002850 }
Avi Kivity6aa8b732006-12-10 02:21:36 -08002851 case KVM_GET_REGS: {
Xiantao Zhang3e4bb3a2008-02-25 18:52:20 +08002852 struct kvm_regs *kvm_regs;
Avi Kivity6aa8b732006-12-10 02:21:36 -08002853
Xiantao Zhang3e4bb3a2008-02-25 18:52:20 +08002854 r = -ENOMEM;
Ben Gardonb12ce362019-02-11 11:02:49 -08002855 kvm_regs = kzalloc(sizeof(struct kvm_regs), GFP_KERNEL_ACCOUNT);
Xiantao Zhang3e4bb3a2008-02-25 18:52:20 +08002856 if (!kvm_regs)
2857 goto out;
2858 r = kvm_arch_vcpu_ioctl_get_regs(vcpu, kvm_regs);
Avi Kivity6aa8b732006-12-10 02:21:36 -08002859 if (r)
Xiantao Zhang3e4bb3a2008-02-25 18:52:20 +08002860 goto out_free1;
Avi Kivity6aa8b732006-12-10 02:21:36 -08002861 r = -EFAULT;
Xiantao Zhang3e4bb3a2008-02-25 18:52:20 +08002862 if (copy_to_user(argp, kvm_regs, sizeof(struct kvm_regs)))
2863 goto out_free1;
Avi Kivity6aa8b732006-12-10 02:21:36 -08002864 r = 0;
Xiantao Zhang3e4bb3a2008-02-25 18:52:20 +08002865out_free1:
2866 kfree(kvm_regs);
Avi Kivity6aa8b732006-12-10 02:21:36 -08002867 break;
2868 }
2869 case KVM_SET_REGS: {
Xiantao Zhang3e4bb3a2008-02-25 18:52:20 +08002870 struct kvm_regs *kvm_regs;
Avi Kivity6aa8b732006-12-10 02:21:36 -08002871
Xiantao Zhang3e4bb3a2008-02-25 18:52:20 +08002872 r = -ENOMEM;
Sasha Levinff5c2c02011-12-04 19:36:29 +02002873 kvm_regs = memdup_user(argp, sizeof(*kvm_regs));
2874 if (IS_ERR(kvm_regs)) {
2875 r = PTR_ERR(kvm_regs);
Xiantao Zhang3e4bb3a2008-02-25 18:52:20 +08002876 goto out;
Sasha Levinff5c2c02011-12-04 19:36:29 +02002877 }
Xiantao Zhang3e4bb3a2008-02-25 18:52:20 +08002878 r = kvm_arch_vcpu_ioctl_set_regs(vcpu, kvm_regs);
Xiantao Zhang3e4bb3a2008-02-25 18:52:20 +08002879 kfree(kvm_regs);
Avi Kivity6aa8b732006-12-10 02:21:36 -08002880 break;
2881 }
2882 case KVM_GET_SREGS: {
Ben Gardonb12ce362019-02-11 11:02:49 -08002883 kvm_sregs = kzalloc(sizeof(struct kvm_sregs),
2884 GFP_KERNEL_ACCOUNT);
Dave Hansenfa3795a2008-08-11 10:01:46 -07002885 r = -ENOMEM;
2886 if (!kvm_sregs)
2887 goto out;
2888 r = kvm_arch_vcpu_ioctl_get_sregs(vcpu, kvm_sregs);
Avi Kivity6aa8b732006-12-10 02:21:36 -08002889 if (r)
2890 goto out;
2891 r = -EFAULT;
Dave Hansenfa3795a2008-08-11 10:01:46 -07002892 if (copy_to_user(argp, kvm_sregs, sizeof(struct kvm_sregs)))
Avi Kivity6aa8b732006-12-10 02:21:36 -08002893 goto out;
2894 r = 0;
2895 break;
2896 }
2897 case KVM_SET_SREGS: {
Sasha Levinff5c2c02011-12-04 19:36:29 +02002898 kvm_sregs = memdup_user(argp, sizeof(*kvm_sregs));
2899 if (IS_ERR(kvm_sregs)) {
2900 r = PTR_ERR(kvm_sregs);
Guo Chao18595412012-11-02 18:33:21 +08002901 kvm_sregs = NULL;
Avi Kivity6aa8b732006-12-10 02:21:36 -08002902 goto out;
Sasha Levinff5c2c02011-12-04 19:36:29 +02002903 }
Dave Hansenfa3795a2008-08-11 10:01:46 -07002904 r = kvm_arch_vcpu_ioctl_set_sregs(vcpu, kvm_sregs);
Avi Kivity6aa8b732006-12-10 02:21:36 -08002905 break;
2906 }
Marcelo Tosatti62d9f0d2008-04-11 13:24:45 -03002907 case KVM_GET_MP_STATE: {
2908 struct kvm_mp_state mp_state;
2909
2910 r = kvm_arch_vcpu_ioctl_get_mpstate(vcpu, &mp_state);
2911 if (r)
2912 goto out;
2913 r = -EFAULT;
Xiubo Li893bdbf2015-02-26 14:58:19 +08002914 if (copy_to_user(argp, &mp_state, sizeof(mp_state)))
Marcelo Tosatti62d9f0d2008-04-11 13:24:45 -03002915 goto out;
2916 r = 0;
2917 break;
2918 }
2919 case KVM_SET_MP_STATE: {
2920 struct kvm_mp_state mp_state;
2921
2922 r = -EFAULT;
Xiubo Li893bdbf2015-02-26 14:58:19 +08002923 if (copy_from_user(&mp_state, argp, sizeof(mp_state)))
Marcelo Tosatti62d9f0d2008-04-11 13:24:45 -03002924 goto out;
2925 r = kvm_arch_vcpu_ioctl_set_mpstate(vcpu, &mp_state);
Marcelo Tosatti62d9f0d2008-04-11 13:24:45 -03002926 break;
2927 }
Avi Kivity6aa8b732006-12-10 02:21:36 -08002928 case KVM_TRANSLATE: {
2929 struct kvm_translation tr;
2930
2931 r = -EFAULT;
Xiubo Li893bdbf2015-02-26 14:58:19 +08002932 if (copy_from_user(&tr, argp, sizeof(tr)))
Avi Kivity6aa8b732006-12-10 02:21:36 -08002933 goto out;
Zhang Xiantao8b00679132007-11-16 13:05:55 +08002934 r = kvm_arch_vcpu_ioctl_translate(vcpu, &tr);
Avi Kivity6aa8b732006-12-10 02:21:36 -08002935 if (r)
2936 goto out;
2937 r = -EFAULT;
Xiubo Li893bdbf2015-02-26 14:58:19 +08002938 if (copy_to_user(argp, &tr, sizeof(tr)))
Avi Kivity6aa8b732006-12-10 02:21:36 -08002939 goto out;
2940 r = 0;
2941 break;
2942 }
Jan Kiszkad0bfb942008-12-15 13:52:10 +01002943 case KVM_SET_GUEST_DEBUG: {
2944 struct kvm_guest_debug dbg;
Avi Kivity6aa8b732006-12-10 02:21:36 -08002945
2946 r = -EFAULT;
Xiubo Li893bdbf2015-02-26 14:58:19 +08002947 if (copy_from_user(&dbg, argp, sizeof(dbg)))
Avi Kivity6aa8b732006-12-10 02:21:36 -08002948 goto out;
Jan Kiszkad0bfb942008-12-15 13:52:10 +01002949 r = kvm_arch_vcpu_ioctl_set_guest_debug(vcpu, &dbg);
Avi Kivity6aa8b732006-12-10 02:21:36 -08002950 break;
2951 }
Avi Kivity1961d272007-03-05 19:46:05 +02002952 case KVM_SET_SIGNAL_MASK: {
2953 struct kvm_signal_mask __user *sigmask_arg = argp;
2954 struct kvm_signal_mask kvm_sigmask;
2955 sigset_t sigset, *p;
2956
2957 p = NULL;
2958 if (argp) {
2959 r = -EFAULT;
2960 if (copy_from_user(&kvm_sigmask, argp,
Xiubo Li893bdbf2015-02-26 14:58:19 +08002961 sizeof(kvm_sigmask)))
Avi Kivity1961d272007-03-05 19:46:05 +02002962 goto out;
2963 r = -EINVAL;
Xiubo Li893bdbf2015-02-26 14:58:19 +08002964 if (kvm_sigmask.len != sizeof(sigset))
Avi Kivity1961d272007-03-05 19:46:05 +02002965 goto out;
2966 r = -EFAULT;
2967 if (copy_from_user(&sigset, sigmask_arg->sigset,
Xiubo Li893bdbf2015-02-26 14:58:19 +08002968 sizeof(sigset)))
Avi Kivity1961d272007-03-05 19:46:05 +02002969 goto out;
2970 p = &sigset;
2971 }
Andi Kleen376d41f2010-06-10 13:10:47 +02002972 r = kvm_vcpu_ioctl_set_sigmask(vcpu, p);
Avi Kivity1961d272007-03-05 19:46:05 +02002973 break;
2974 }
Avi Kivityb8836732007-04-01 16:34:31 +03002975 case KVM_GET_FPU: {
Ben Gardonb12ce362019-02-11 11:02:49 -08002976 fpu = kzalloc(sizeof(struct kvm_fpu), GFP_KERNEL_ACCOUNT);
Dave Hansenfa3795a2008-08-11 10:01:46 -07002977 r = -ENOMEM;
2978 if (!fpu)
2979 goto out;
2980 r = kvm_arch_vcpu_ioctl_get_fpu(vcpu, fpu);
Avi Kivityb8836732007-04-01 16:34:31 +03002981 if (r)
2982 goto out;
2983 r = -EFAULT;
Dave Hansenfa3795a2008-08-11 10:01:46 -07002984 if (copy_to_user(argp, fpu, sizeof(struct kvm_fpu)))
Avi Kivityb8836732007-04-01 16:34:31 +03002985 goto out;
2986 r = 0;
2987 break;
2988 }
2989 case KVM_SET_FPU: {
Sasha Levinff5c2c02011-12-04 19:36:29 +02002990 fpu = memdup_user(argp, sizeof(*fpu));
2991 if (IS_ERR(fpu)) {
2992 r = PTR_ERR(fpu);
Guo Chao18595412012-11-02 18:33:21 +08002993 fpu = NULL;
Avi Kivityb8836732007-04-01 16:34:31 +03002994 goto out;
Sasha Levinff5c2c02011-12-04 19:36:29 +02002995 }
Dave Hansenfa3795a2008-08-11 10:01:46 -07002996 r = kvm_arch_vcpu_ioctl_set_fpu(vcpu, fpu);
Avi Kivityb8836732007-04-01 16:34:31 +03002997 break;
2998 }
Avi Kivitybccf2152007-02-21 18:04:26 +02002999 default:
Carsten Otte313a3dc2007-10-11 19:16:52 +02003000 r = kvm_arch_vcpu_ioctl(filp, ioctl, arg);
Avi Kivitybccf2152007-02-21 18:04:26 +02003001 }
3002out:
Christoffer Dallec7660c2017-12-04 21:35:23 +01003003 mutex_unlock(&vcpu->mutex);
Dave Hansenfa3795a2008-08-11 10:01:46 -07003004 kfree(fpu);
3005 kfree(kvm_sregs);
Avi Kivitybccf2152007-02-21 18:04:26 +02003006 return r;
3007}
3008
Christian Borntraegerde8e5d72015-02-03 09:35:15 +01003009#ifdef CONFIG_KVM_COMPAT
Alexander Graf1dda6062011-06-08 02:45:37 +02003010static long kvm_vcpu_compat_ioctl(struct file *filp,
3011 unsigned int ioctl, unsigned long arg)
3012{
3013 struct kvm_vcpu *vcpu = filp->private_data;
3014 void __user *argp = compat_ptr(arg);
3015 int r;
3016
3017 if (vcpu->kvm->mm != current->mm)
3018 return -EIO;
3019
3020 switch (ioctl) {
3021 case KVM_SET_SIGNAL_MASK: {
3022 struct kvm_signal_mask __user *sigmask_arg = argp;
3023 struct kvm_signal_mask kvm_sigmask;
Alexander Graf1dda6062011-06-08 02:45:37 +02003024 sigset_t sigset;
3025
3026 if (argp) {
3027 r = -EFAULT;
3028 if (copy_from_user(&kvm_sigmask, argp,
Xiubo Li893bdbf2015-02-26 14:58:19 +08003029 sizeof(kvm_sigmask)))
Alexander Graf1dda6062011-06-08 02:45:37 +02003030 goto out;
3031 r = -EINVAL;
Al Viro3968cf62017-09-03 21:45:17 -04003032 if (kvm_sigmask.len != sizeof(compat_sigset_t))
Alexander Graf1dda6062011-06-08 02:45:37 +02003033 goto out;
3034 r = -EFAULT;
Al Viro3968cf62017-09-03 21:45:17 -04003035 if (get_compat_sigset(&sigset, (void *)sigmask_arg->sigset))
Alexander Graf1dda6062011-06-08 02:45:37 +02003036 goto out;
Alan Cox760a9a32012-08-22 14:34:11 +01003037 r = kvm_vcpu_ioctl_set_sigmask(vcpu, &sigset);
3038 } else
3039 r = kvm_vcpu_ioctl_set_sigmask(vcpu, NULL);
Alexander Graf1dda6062011-06-08 02:45:37 +02003040 break;
3041 }
3042 default:
3043 r = kvm_vcpu_ioctl(filp, ioctl, arg);
3044 }
3045
3046out:
3047 return r;
3048}
3049#endif
3050
Cédric Le Goatera1cd3f02019-04-18 12:39:36 +02003051static int kvm_device_mmap(struct file *filp, struct vm_area_struct *vma)
3052{
3053 struct kvm_device *dev = filp->private_data;
3054
3055 if (dev->ops->mmap)
3056 return dev->ops->mmap(dev, vma);
3057
3058 return -ENODEV;
3059}
3060
Scott Wood852b6d52013-04-12 14:08:42 +00003061static int kvm_device_ioctl_attr(struct kvm_device *dev,
3062 int (*accessor)(struct kvm_device *dev,
3063 struct kvm_device_attr *attr),
3064 unsigned long arg)
3065{
3066 struct kvm_device_attr attr;
3067
3068 if (!accessor)
3069 return -EPERM;
3070
3071 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
3072 return -EFAULT;
3073
3074 return accessor(dev, &attr);
3075}
3076
3077static long kvm_device_ioctl(struct file *filp, unsigned int ioctl,
3078 unsigned long arg)
3079{
3080 struct kvm_device *dev = filp->private_data;
3081
Sean Christophersonddba9182019-02-15 12:48:39 -08003082 if (dev->kvm->mm != current->mm)
3083 return -EIO;
3084
Scott Wood852b6d52013-04-12 14:08:42 +00003085 switch (ioctl) {
3086 case KVM_SET_DEVICE_ATTR:
3087 return kvm_device_ioctl_attr(dev, dev->ops->set_attr, arg);
3088 case KVM_GET_DEVICE_ATTR:
3089 return kvm_device_ioctl_attr(dev, dev->ops->get_attr, arg);
3090 case KVM_HAS_DEVICE_ATTR:
3091 return kvm_device_ioctl_attr(dev, dev->ops->has_attr, arg);
3092 default:
3093 if (dev->ops->ioctl)
3094 return dev->ops->ioctl(dev, ioctl, arg);
3095
3096 return -ENOTTY;
3097 }
3098}
3099
Scott Wood852b6d52013-04-12 14:08:42 +00003100static int kvm_device_release(struct inode *inode, struct file *filp)
3101{
3102 struct kvm_device *dev = filp->private_data;
3103 struct kvm *kvm = dev->kvm;
3104
Cédric Le Goater2bde9b32019-04-18 12:39:41 +02003105 if (dev->ops->release) {
3106 mutex_lock(&kvm->lock);
3107 list_del(&dev->vm_node);
3108 dev->ops->release(dev);
3109 mutex_unlock(&kvm->lock);
3110 }
3111
Scott Wood852b6d52013-04-12 14:08:42 +00003112 kvm_put_kvm(kvm);
3113 return 0;
3114}
3115
3116static const struct file_operations kvm_device_fops = {
3117 .unlocked_ioctl = kvm_device_ioctl,
3118 .release = kvm_device_release,
Marc Zyngier7ddfd3e2018-06-17 10:16:21 +01003119 KVM_COMPAT(kvm_device_ioctl),
Cédric Le Goatera1cd3f02019-04-18 12:39:36 +02003120 .mmap = kvm_device_mmap,
Scott Wood852b6d52013-04-12 14:08:42 +00003121};
3122
3123struct kvm_device *kvm_device_from_filp(struct file *filp)
3124{
3125 if (filp->f_op != &kvm_device_fops)
3126 return NULL;
3127
3128 return filp->private_data;
3129}
3130
Steven Price8538cb22019-10-21 16:28:19 +01003131static const struct kvm_device_ops *kvm_device_ops_table[KVM_DEV_TYPE_MAX] = {
Will Deacond60eacb2014-09-02 10:27:33 +01003132#ifdef CONFIG_KVM_MPIC
3133 [KVM_DEV_TYPE_FSL_MPIC_20] = &kvm_mpic_ops,
3134 [KVM_DEV_TYPE_FSL_MPIC_42] = &kvm_mpic_ops,
3135#endif
Will Deacond60eacb2014-09-02 10:27:33 +01003136};
3137
Steven Price8538cb22019-10-21 16:28:19 +01003138int kvm_register_device_ops(const struct kvm_device_ops *ops, u32 type)
Will Deacond60eacb2014-09-02 10:27:33 +01003139{
3140 if (type >= ARRAY_SIZE(kvm_device_ops_table))
3141 return -ENOSPC;
3142
3143 if (kvm_device_ops_table[type] != NULL)
3144 return -EEXIST;
3145
3146 kvm_device_ops_table[type] = ops;
3147 return 0;
3148}
3149
Wanpeng Li571ee1b2014-10-09 18:30:08 +08003150void kvm_unregister_device_ops(u32 type)
3151{
3152 if (kvm_device_ops_table[type] != NULL)
3153 kvm_device_ops_table[type] = NULL;
3154}
3155
Scott Wood852b6d52013-04-12 14:08:42 +00003156static int kvm_ioctl_create_device(struct kvm *kvm,
3157 struct kvm_create_device *cd)
3158{
Steven Price8538cb22019-10-21 16:28:19 +01003159 const struct kvm_device_ops *ops = NULL;
Scott Wood852b6d52013-04-12 14:08:42 +00003160 struct kvm_device *dev;
3161 bool test = cd->flags & KVM_CREATE_DEVICE_TEST;
Paolo Bonzini1d487e92019-04-11 11:16:47 +02003162 int type;
Scott Wood852b6d52013-04-12 14:08:42 +00003163 int ret;
3164
Will Deacond60eacb2014-09-02 10:27:33 +01003165 if (cd->type >= ARRAY_SIZE(kvm_device_ops_table))
Scott Wood852b6d52013-04-12 14:08:42 +00003166 return -ENODEV;
Will Deacond60eacb2014-09-02 10:27:33 +01003167
Paolo Bonzini1d487e92019-04-11 11:16:47 +02003168 type = array_index_nospec(cd->type, ARRAY_SIZE(kvm_device_ops_table));
3169 ops = kvm_device_ops_table[type];
Will Deacond60eacb2014-09-02 10:27:33 +01003170 if (ops == NULL)
3171 return -ENODEV;
Scott Wood852b6d52013-04-12 14:08:42 +00003172
3173 if (test)
3174 return 0;
3175
Ben Gardonb12ce362019-02-11 11:02:49 -08003176 dev = kzalloc(sizeof(*dev), GFP_KERNEL_ACCOUNT);
Scott Wood852b6d52013-04-12 14:08:42 +00003177 if (!dev)
3178 return -ENOMEM;
3179
3180 dev->ops = ops;
3181 dev->kvm = kvm;
Scott Wood852b6d52013-04-12 14:08:42 +00003182
Christoffer Dalla28ebea2016-08-09 19:13:01 +02003183 mutex_lock(&kvm->lock);
Paolo Bonzini1d487e92019-04-11 11:16:47 +02003184 ret = ops->create(dev, type);
Scott Wood852b6d52013-04-12 14:08:42 +00003185 if (ret < 0) {
Christoffer Dalla28ebea2016-08-09 19:13:01 +02003186 mutex_unlock(&kvm->lock);
Scott Wood852b6d52013-04-12 14:08:42 +00003187 kfree(dev);
3188 return ret;
3189 }
Christoffer Dalla28ebea2016-08-09 19:13:01 +02003190 list_add(&dev->vm_node, &kvm->devices);
3191 mutex_unlock(&kvm->lock);
Scott Wood852b6d52013-04-12 14:08:42 +00003192
Christoffer Dall023e9fd2016-08-09 19:13:00 +02003193 if (ops->init)
3194 ops->init(dev);
3195
Jann Horncfa39382019-01-26 01:54:33 +01003196 kvm_get_kvm(kvm);
Yann Droneaud24009b02013-08-24 22:14:07 +02003197 ret = anon_inode_getfd(ops->name, &kvm_device_fops, dev, O_RDWR | O_CLOEXEC);
Scott Wood852b6d52013-04-12 14:08:42 +00003198 if (ret < 0) {
Sean Christopherson149487b2019-10-21 15:58:42 -07003199 kvm_put_kvm_no_destroy(kvm);
Christoffer Dalla28ebea2016-08-09 19:13:01 +02003200 mutex_lock(&kvm->lock);
3201 list_del(&dev->vm_node);
3202 mutex_unlock(&kvm->lock);
Dan Carpentera0f1d212016-11-30 22:21:05 +03003203 ops->destroy(dev);
Scott Wood852b6d52013-04-12 14:08:42 +00003204 return ret;
3205 }
3206
Scott Wood852b6d52013-04-12 14:08:42 +00003207 cd->fd = ret;
3208 return 0;
3209}
3210
Alexander Graf92b591a2014-07-14 18:33:08 +02003211static long kvm_vm_ioctl_check_extension_generic(struct kvm *kvm, long arg)
3212{
3213 switch (arg) {
3214 case KVM_CAP_USER_MEMORY:
3215 case KVM_CAP_DESTROY_MEMORY_REGION_WORKS:
3216 case KVM_CAP_JOIN_MEMORY_REGIONS_WORKS:
Alexander Graf92b591a2014-07-14 18:33:08 +02003217 case KVM_CAP_INTERNAL_ERROR_DATA:
3218#ifdef CONFIG_HAVE_KVM_MSI
3219 case KVM_CAP_SIGNAL_MSI:
3220#endif
Paul Mackerras297e2102014-06-30 20:51:13 +10003221#ifdef CONFIG_HAVE_KVM_IRQFD
Paolo Bonzinidc9be0f2015-03-05 11:54:46 +01003222 case KVM_CAP_IRQFD:
Alexander Graf92b591a2014-07-14 18:33:08 +02003223 case KVM_CAP_IRQFD_RESAMPLE:
3224#endif
Jason Wange9ea5062015-09-15 14:41:59 +08003225 case KVM_CAP_IOEVENTFD_ANY_LENGTH:
Alexander Graf92b591a2014-07-14 18:33:08 +02003226 case KVM_CAP_CHECK_EXTENSION_VM:
Paolo Bonzinie5d83c72017-02-16 10:40:56 +01003227 case KVM_CAP_ENABLE_CAP_VM:
Paolo Bonzini2a31b9d2018-10-23 02:36:47 +02003228#ifdef CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT
Peter Xud7547c52019-05-08 17:15:47 +08003229 case KVM_CAP_MANUAL_DIRTY_LOG_PROTECT2:
Paolo Bonzini2a31b9d2018-10-23 02:36:47 +02003230#endif
Alexander Graf92b591a2014-07-14 18:33:08 +02003231 return 1;
Paolo Bonzini4b4357e2017-03-31 13:53:23 +02003232#ifdef CONFIG_KVM_MMIO
Paolo Bonzini30422552017-03-31 13:53:22 +02003233 case KVM_CAP_COALESCED_MMIO:
3234 return KVM_COALESCED_MMIO_PAGE_OFFSET;
Peng Hao0804c842018-10-14 07:09:55 +08003235 case KVM_CAP_COALESCED_PIO:
3236 return 1;
Paolo Bonzini30422552017-03-31 13:53:22 +02003237#endif
Alexander Graf92b591a2014-07-14 18:33:08 +02003238#ifdef CONFIG_HAVE_KVM_IRQ_ROUTING
3239 case KVM_CAP_IRQ_ROUTING:
3240 return KVM_MAX_IRQ_ROUTES;
3241#endif
Paolo Bonzinif481b062015-05-17 17:30:37 +02003242#if KVM_ADDRESS_SPACE_NUM > 1
3243 case KVM_CAP_MULTI_ADDRESS_SPACE:
3244 return KVM_ADDRESS_SPACE_NUM;
3245#endif
Paolo Bonzinic110ae52019-03-28 17:24:03 +01003246 case KVM_CAP_NR_MEMSLOTS:
3247 return KVM_USER_MEM_SLOTS;
Alexander Graf92b591a2014-07-14 18:33:08 +02003248 default:
3249 break;
3250 }
3251 return kvm_vm_ioctl_check_extension(kvm, arg);
3252}
3253
Paolo Bonzinie5d83c72017-02-16 10:40:56 +01003254int __attribute__((weak)) kvm_vm_ioctl_enable_cap(struct kvm *kvm,
3255 struct kvm_enable_cap *cap)
3256{
3257 return -EINVAL;
3258}
3259
3260static int kvm_vm_ioctl_enable_cap_generic(struct kvm *kvm,
3261 struct kvm_enable_cap *cap)
3262{
3263 switch (cap->cap) {
Paolo Bonzini2a31b9d2018-10-23 02:36:47 +02003264#ifdef CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT
Peter Xud7547c52019-05-08 17:15:47 +08003265 case KVM_CAP_MANUAL_DIRTY_LOG_PROTECT2:
Paolo Bonzini2a31b9d2018-10-23 02:36:47 +02003266 if (cap->flags || (cap->args[0] & ~1))
3267 return -EINVAL;
3268 kvm->manual_dirty_log_protect = cap->args[0];
3269 return 0;
3270#endif
Paolo Bonzinie5d83c72017-02-16 10:40:56 +01003271 default:
3272 return kvm_vm_ioctl_enable_cap(kvm, cap);
3273 }
3274}
3275
Avi Kivitybccf2152007-02-21 18:04:26 +02003276static long kvm_vm_ioctl(struct file *filp,
3277 unsigned int ioctl, unsigned long arg)
3278{
3279 struct kvm *kvm = filp->private_data;
3280 void __user *argp = (void __user *)arg;
Carsten Otte1fe779f2007-10-29 16:08:35 +01003281 int r;
Avi Kivitybccf2152007-02-21 18:04:26 +02003282
Avi Kivity6d4e4c42007-11-21 16:41:05 +02003283 if (kvm->mm != current->mm)
3284 return -EIO;
Avi Kivitybccf2152007-02-21 18:04:26 +02003285 switch (ioctl) {
3286 case KVM_CREATE_VCPU:
3287 r = kvm_vm_ioctl_create_vcpu(kvm, arg);
Avi Kivitybccf2152007-02-21 18:04:26 +02003288 break;
Paolo Bonzinie5d83c72017-02-16 10:40:56 +01003289 case KVM_ENABLE_CAP: {
3290 struct kvm_enable_cap cap;
3291
3292 r = -EFAULT;
3293 if (copy_from_user(&cap, argp, sizeof(cap)))
3294 goto out;
3295 r = kvm_vm_ioctl_enable_cap_generic(kvm, &cap);
3296 break;
3297 }
Izik Eidus6fc138d2007-10-09 19:20:39 +02003298 case KVM_SET_USER_MEMORY_REGION: {
3299 struct kvm_userspace_memory_region kvm_userspace_mem;
3300
3301 r = -EFAULT;
3302 if (copy_from_user(&kvm_userspace_mem, argp,
Xiubo Li893bdbf2015-02-26 14:58:19 +08003303 sizeof(kvm_userspace_mem)))
Izik Eidus6fc138d2007-10-09 19:20:39 +02003304 goto out;
3305
Takuya Yoshikawa47ae31e2013-02-27 19:43:00 +09003306 r = kvm_vm_ioctl_set_memory_region(kvm, &kvm_userspace_mem);
Avi Kivity6aa8b732006-12-10 02:21:36 -08003307 break;
3308 }
3309 case KVM_GET_DIRTY_LOG: {
3310 struct kvm_dirty_log log;
3311
3312 r = -EFAULT;
Xiubo Li893bdbf2015-02-26 14:58:19 +08003313 if (copy_from_user(&log, argp, sizeof(log)))
Avi Kivity6aa8b732006-12-10 02:21:36 -08003314 goto out;
Avi Kivity2c6f5df2007-02-20 18:27:58 +02003315 r = kvm_vm_ioctl_get_dirty_log(kvm, &log);
Avi Kivity6aa8b732006-12-10 02:21:36 -08003316 break;
3317 }
Paolo Bonzini2a31b9d2018-10-23 02:36:47 +02003318#ifdef CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT
3319 case KVM_CLEAR_DIRTY_LOG: {
3320 struct kvm_clear_dirty_log log;
3321
3322 r = -EFAULT;
3323 if (copy_from_user(&log, argp, sizeof(log)))
3324 goto out;
3325 r = kvm_vm_ioctl_clear_dirty_log(kvm, &log);
3326 break;
3327 }
3328#endif
Paolo Bonzini4b4357e2017-03-31 13:53:23 +02003329#ifdef CONFIG_KVM_MMIO
Laurent Vivier5f94c172008-05-30 16:05:54 +02003330 case KVM_REGISTER_COALESCED_MMIO: {
3331 struct kvm_coalesced_mmio_zone zone;
Xiubo Lif95ef0cd2015-02-26 14:58:23 +08003332
Laurent Vivier5f94c172008-05-30 16:05:54 +02003333 r = -EFAULT;
Xiubo Li893bdbf2015-02-26 14:58:19 +08003334 if (copy_from_user(&zone, argp, sizeof(zone)))
Laurent Vivier5f94c172008-05-30 16:05:54 +02003335 goto out;
Laurent Vivier5f94c172008-05-30 16:05:54 +02003336 r = kvm_vm_ioctl_register_coalesced_mmio(kvm, &zone);
Laurent Vivier5f94c172008-05-30 16:05:54 +02003337 break;
3338 }
3339 case KVM_UNREGISTER_COALESCED_MMIO: {
3340 struct kvm_coalesced_mmio_zone zone;
Xiubo Lif95ef0cd2015-02-26 14:58:23 +08003341
Laurent Vivier5f94c172008-05-30 16:05:54 +02003342 r = -EFAULT;
Xiubo Li893bdbf2015-02-26 14:58:19 +08003343 if (copy_from_user(&zone, argp, sizeof(zone)))
Laurent Vivier5f94c172008-05-30 16:05:54 +02003344 goto out;
Laurent Vivier5f94c172008-05-30 16:05:54 +02003345 r = kvm_vm_ioctl_unregister_coalesced_mmio(kvm, &zone);
Laurent Vivier5f94c172008-05-30 16:05:54 +02003346 break;
3347 }
3348#endif
Gregory Haskins721eecbf2009-05-20 10:30:49 -04003349 case KVM_IRQFD: {
3350 struct kvm_irqfd data;
3351
3352 r = -EFAULT;
Xiubo Li893bdbf2015-02-26 14:58:19 +08003353 if (copy_from_user(&data, argp, sizeof(data)))
Gregory Haskins721eecbf2009-05-20 10:30:49 -04003354 goto out;
Alex Williamsond4db2932012-06-29 09:56:08 -06003355 r = kvm_irqfd(kvm, &data);
Gregory Haskins721eecbf2009-05-20 10:30:49 -04003356 break;
3357 }
Gregory Haskinsd34e6b12009-07-07 17:08:49 -04003358 case KVM_IOEVENTFD: {
3359 struct kvm_ioeventfd data;
3360
3361 r = -EFAULT;
Xiubo Li893bdbf2015-02-26 14:58:19 +08003362 if (copy_from_user(&data, argp, sizeof(data)))
Gregory Haskinsd34e6b12009-07-07 17:08:49 -04003363 goto out;
3364 r = kvm_ioeventfd(kvm, &data);
3365 break;
3366 }
Jan Kiszka07975ad2012-03-29 21:14:12 +02003367#ifdef CONFIG_HAVE_KVM_MSI
3368 case KVM_SIGNAL_MSI: {
3369 struct kvm_msi msi;
3370
3371 r = -EFAULT;
Xiubo Li893bdbf2015-02-26 14:58:19 +08003372 if (copy_from_user(&msi, argp, sizeof(msi)))
Jan Kiszka07975ad2012-03-29 21:14:12 +02003373 goto out;
3374 r = kvm_send_userspace_msi(kvm, &msi);
3375 break;
3376 }
3377#endif
Christoffer Dall23d43cf2012-07-24 08:51:20 -04003378#ifdef __KVM_HAVE_IRQ_LINE
3379 case KVM_IRQ_LINE_STATUS:
3380 case KVM_IRQ_LINE: {
3381 struct kvm_irq_level irq_event;
3382
3383 r = -EFAULT;
Xiubo Li893bdbf2015-02-26 14:58:19 +08003384 if (copy_from_user(&irq_event, argp, sizeof(irq_event)))
Christoffer Dall23d43cf2012-07-24 08:51:20 -04003385 goto out;
3386
Yang Zhangaa2fbe62013-04-11 19:21:40 +08003387 r = kvm_vm_ioctl_irq_line(kvm, &irq_event,
3388 ioctl == KVM_IRQ_LINE_STATUS);
Christoffer Dall23d43cf2012-07-24 08:51:20 -04003389 if (r)
3390 goto out;
3391
3392 r = -EFAULT;
3393 if (ioctl == KVM_IRQ_LINE_STATUS) {
Xiubo Li893bdbf2015-02-26 14:58:19 +08003394 if (copy_to_user(argp, &irq_event, sizeof(irq_event)))
Christoffer Dall23d43cf2012-07-24 08:51:20 -04003395 goto out;
3396 }
3397
3398 r = 0;
3399 break;
3400 }
3401#endif
Alexander Grafaa8d5942013-04-15 21:12:53 +02003402#ifdef CONFIG_HAVE_KVM_IRQ_ROUTING
3403 case KVM_SET_GSI_ROUTING: {
3404 struct kvm_irq_routing routing;
3405 struct kvm_irq_routing __user *urouting;
Paolo Bonzinif8c1b852016-06-01 14:09:22 +02003406 struct kvm_irq_routing_entry *entries = NULL;
Alexander Grafaa8d5942013-04-15 21:12:53 +02003407
3408 r = -EFAULT;
3409 if (copy_from_user(&routing, argp, sizeof(routing)))
3410 goto out;
3411 r = -EINVAL;
David Hildenbrand5c0aea02017-04-28 17:06:20 +02003412 if (!kvm_arch_can_set_irq_routing(kvm))
3413 goto out;
Xiubo Licaf1ff22016-06-15 18:00:33 +08003414 if (routing.nr > KVM_MAX_IRQ_ROUTES)
Alexander Grafaa8d5942013-04-15 21:12:53 +02003415 goto out;
3416 if (routing.flags)
3417 goto out;
Paolo Bonzinif8c1b852016-06-01 14:09:22 +02003418 if (routing.nr) {
3419 r = -ENOMEM;
Kees Cook42bc47b2018-06-12 14:27:11 -07003420 entries = vmalloc(array_size(sizeof(*entries),
3421 routing.nr));
Paolo Bonzinif8c1b852016-06-01 14:09:22 +02003422 if (!entries)
3423 goto out;
3424 r = -EFAULT;
3425 urouting = argp;
3426 if (copy_from_user(entries, urouting->entries,
3427 routing.nr * sizeof(*entries)))
3428 goto out_free_irq_routing;
3429 }
Alexander Grafaa8d5942013-04-15 21:12:53 +02003430 r = kvm_set_irq_routing(kvm, entries, routing.nr,
3431 routing.flags);
Xiubo Lia642a172015-02-26 14:58:20 +08003432out_free_irq_routing:
Alexander Grafaa8d5942013-04-15 21:12:53 +02003433 vfree(entries);
3434 break;
3435 }
3436#endif /* CONFIG_HAVE_KVM_IRQ_ROUTING */
Scott Wood852b6d52013-04-12 14:08:42 +00003437 case KVM_CREATE_DEVICE: {
3438 struct kvm_create_device cd;
3439
3440 r = -EFAULT;
3441 if (copy_from_user(&cd, argp, sizeof(cd)))
3442 goto out;
3443
3444 r = kvm_ioctl_create_device(kvm, &cd);
3445 if (r)
3446 goto out;
3447
3448 r = -EFAULT;
3449 if (copy_to_user(argp, &cd, sizeof(cd)))
3450 goto out;
3451
3452 r = 0;
3453 break;
3454 }
Alexander Graf92b591a2014-07-14 18:33:08 +02003455 case KVM_CHECK_EXTENSION:
3456 r = kvm_vm_ioctl_check_extension_generic(kvm, arg);
3457 break;
Avi Kivityf17abe92007-02-21 19:28:04 +02003458 default:
Carsten Otte1fe779f2007-10-29 16:08:35 +01003459 r = kvm_arch_vm_ioctl(filp, ioctl, arg);
Avi Kivityf17abe92007-02-21 19:28:04 +02003460 }
3461out:
3462 return r;
3463}
3464
Christian Borntraegerde8e5d72015-02-03 09:35:15 +01003465#ifdef CONFIG_KVM_COMPAT
Arnd Bergmann6ff58942009-10-22 14:19:27 +02003466struct compat_kvm_dirty_log {
3467 __u32 slot;
3468 __u32 padding1;
3469 union {
3470 compat_uptr_t dirty_bitmap; /* one bit per page */
3471 __u64 padding2;
3472 };
3473};
3474
3475static long kvm_vm_compat_ioctl(struct file *filp,
3476 unsigned int ioctl, unsigned long arg)
3477{
3478 struct kvm *kvm = filp->private_data;
3479 int r;
3480
3481 if (kvm->mm != current->mm)
3482 return -EIO;
3483 switch (ioctl) {
3484 case KVM_GET_DIRTY_LOG: {
3485 struct compat_kvm_dirty_log compat_log;
3486 struct kvm_dirty_log log;
3487
Arnd Bergmann6ff58942009-10-22 14:19:27 +02003488 if (copy_from_user(&compat_log, (void __user *)arg,
3489 sizeof(compat_log)))
Markus Elfringf6a3b162017-01-22 11:30:21 +01003490 return -EFAULT;
Arnd Bergmann6ff58942009-10-22 14:19:27 +02003491 log.slot = compat_log.slot;
3492 log.padding1 = compat_log.padding1;
3493 log.padding2 = compat_log.padding2;
3494 log.dirty_bitmap = compat_ptr(compat_log.dirty_bitmap);
3495
3496 r = kvm_vm_ioctl_get_dirty_log(kvm, &log);
Arnd Bergmann6ff58942009-10-22 14:19:27 +02003497 break;
3498 }
3499 default:
3500 r = kvm_vm_ioctl(filp, ioctl, arg);
3501 }
Arnd Bergmann6ff58942009-10-22 14:19:27 +02003502 return r;
3503}
3504#endif
3505
Christian Borntraeger3d3aab12008-12-02 11:17:32 +01003506static struct file_operations kvm_vm_fops = {
Avi Kivityf17abe92007-02-21 19:28:04 +02003507 .release = kvm_vm_release,
3508 .unlocked_ioctl = kvm_vm_ioctl,
Arnd Bergmann6038f372010-08-15 18:52:59 +02003509 .llseek = noop_llseek,
Marc Zyngier7ddfd3e2018-06-17 10:16:21 +01003510 KVM_COMPAT(kvm_vm_compat_ioctl),
Avi Kivityf17abe92007-02-21 19:28:04 +02003511};
3512
Carsten Ottee08b9632012-01-04 10:25:20 +01003513static int kvm_dev_ioctl_create_vm(unsigned long type)
Avi Kivityf17abe92007-02-21 19:28:04 +02003514{
Heiko Carstensaac87632010-10-27 17:22:10 +02003515 int r;
Avi Kivityf17abe92007-02-21 19:28:04 +02003516 struct kvm *kvm;
Al Viro506cfba2016-07-14 18:54:17 +02003517 struct file *file;
Avi Kivityf17abe92007-02-21 19:28:04 +02003518
Carsten Ottee08b9632012-01-04 10:25:20 +01003519 kvm = kvm_create_vm(type);
Avi Kivityd6d28162007-06-28 08:38:16 -04003520 if (IS_ERR(kvm))
3521 return PTR_ERR(kvm);
Paolo Bonzini4b4357e2017-03-31 13:53:23 +02003522#ifdef CONFIG_KVM_MMIO
Takuya Yoshikawa6ce5a092010-03-15 22:13:30 +09003523 r = kvm_coalesced_mmio_init(kvm);
Markus Elfring785883352017-11-21 13:40:17 +01003524 if (r < 0)
3525 goto put_kvm;
Takuya Yoshikawa6ce5a092010-03-15 22:13:30 +09003526#endif
Al Viro506cfba2016-07-14 18:54:17 +02003527 r = get_unused_fd_flags(O_CLOEXEC);
Markus Elfring785883352017-11-21 13:40:17 +01003528 if (r < 0)
3529 goto put_kvm;
3530
Al Viro506cfba2016-07-14 18:54:17 +02003531 file = anon_inode_getfile("kvm-vm", &kvm_vm_fops, kvm, O_RDWR);
3532 if (IS_ERR(file)) {
3533 put_unused_fd(r);
Markus Elfring785883352017-11-21 13:40:17 +01003534 r = PTR_ERR(file);
3535 goto put_kvm;
Al Viro506cfba2016-07-14 18:54:17 +02003536 }
Janosch Frank536a6f82016-05-18 13:26:23 +02003537
Paolo Bonzini525df862017-06-27 15:45:09 +02003538 /*
3539 * Don't call kvm_put_kvm anymore at this point; file->f_op is
3540 * already set, with ->release() being kvm_vm_release(). In error
3541 * cases it will be called by the final fput(file) and will take
3542 * care of doing kvm_put_kvm(kvm).
3543 */
Janosch Frank536a6f82016-05-18 13:26:23 +02003544 if (kvm_create_vm_debugfs(kvm, r) < 0) {
Al Viro506cfba2016-07-14 18:54:17 +02003545 put_unused_fd(r);
3546 fput(file);
Janosch Frank536a6f82016-05-18 13:26:23 +02003547 return -ENOMEM;
3548 }
Claudio Imbrenda286de8f2017-07-12 17:56:44 +02003549 kvm_uevent_notify_change(KVM_EVENT_CREATE_VM, kvm);
Avi Kivityf17abe92007-02-21 19:28:04 +02003550
Al Viro506cfba2016-07-14 18:54:17 +02003551 fd_install(r, file);
Heiko Carstensaac87632010-10-27 17:22:10 +02003552 return r;
Markus Elfring785883352017-11-21 13:40:17 +01003553
3554put_kvm:
3555 kvm_put_kvm(kvm);
3556 return r;
Avi Kivityf17abe92007-02-21 19:28:04 +02003557}
3558
3559static long kvm_dev_ioctl(struct file *filp,
3560 unsigned int ioctl, unsigned long arg)
3561{
Avi Kivity07c45a32007-03-07 13:05:38 +02003562 long r = -EINVAL;
Avi Kivityf17abe92007-02-21 19:28:04 +02003563
3564 switch (ioctl) {
3565 case KVM_GET_API_VERSION:
Avi Kivityf0fe5102007-03-07 13:11:17 +02003566 if (arg)
3567 goto out;
Avi Kivityf17abe92007-02-21 19:28:04 +02003568 r = KVM_API_VERSION;
3569 break;
3570 case KVM_CREATE_VM:
Carsten Ottee08b9632012-01-04 10:25:20 +01003571 r = kvm_dev_ioctl_create_vm(arg);
Avi Kivityf17abe92007-02-21 19:28:04 +02003572 break;
Zhang Xiantao018d00d2007-11-15 23:07:47 +08003573 case KVM_CHECK_EXTENSION:
Alexander Graf784aa3d2014-07-14 18:27:35 +02003574 r = kvm_vm_ioctl_check_extension_generic(NULL, arg);
Avi Kivity5d308f42007-03-01 17:56:20 +02003575 break;
Avi Kivity07c45a32007-03-07 13:05:38 +02003576 case KVM_GET_VCPU_MMAP_SIZE:
Avi Kivity07c45a32007-03-07 13:05:38 +02003577 if (arg)
3578 goto out;
Avi Kivityadb1ff42008-01-24 15:13:08 +02003579 r = PAGE_SIZE; /* struct kvm_run */
3580#ifdef CONFIG_X86
3581 r += PAGE_SIZE; /* pio data page */
3582#endif
Paolo Bonzini4b4357e2017-03-31 13:53:23 +02003583#ifdef CONFIG_KVM_MMIO
Laurent Vivier5f94c172008-05-30 16:05:54 +02003584 r += PAGE_SIZE; /* coalesced mmio ring page */
3585#endif
Avi Kivity07c45a32007-03-07 13:05:38 +02003586 break;
Feng(Eric) Liud4c9ff22008-04-10 08:47:53 -04003587 case KVM_TRACE_ENABLE:
3588 case KVM_TRACE_PAUSE:
3589 case KVM_TRACE_DISABLE:
Marcelo Tosatti2023a292009-06-18 11:47:28 -03003590 r = -EOPNOTSUPP;
Feng(Eric) Liud4c9ff22008-04-10 08:47:53 -04003591 break;
Avi Kivity6aa8b732006-12-10 02:21:36 -08003592 default:
Carsten Otte043405e2007-10-10 17:16:19 +02003593 return kvm_arch_dev_ioctl(filp, ioctl, arg);
Avi Kivity6aa8b732006-12-10 02:21:36 -08003594 }
3595out:
3596 return r;
3597}
3598
Avi Kivity6aa8b732006-12-10 02:21:36 -08003599static struct file_operations kvm_chardev_ops = {
Avi Kivity6aa8b732006-12-10 02:21:36 -08003600 .unlocked_ioctl = kvm_dev_ioctl,
Arnd Bergmann6038f372010-08-15 18:52:59 +02003601 .llseek = noop_llseek,
Marc Zyngier7ddfd3e2018-06-17 10:16:21 +01003602 KVM_COMPAT(kvm_dev_ioctl),
Avi Kivity6aa8b732006-12-10 02:21:36 -08003603};
3604
3605static struct miscdevice kvm_dev = {
Avi Kivitybbe44322007-03-04 13:27:36 +02003606 KVM_MINOR,
Avi Kivity6aa8b732006-12-10 02:21:36 -08003607 "kvm",
3608 &kvm_chardev_ops,
3609};
3610
Takuya Yoshikawa75b71272010-11-16 17:37:41 +09003611static void hardware_enable_nolock(void *junk)
Avi Kivity1b6c0162007-05-24 13:03:52 +03003612{
3613 int cpu = raw_smp_processor_id();
Alexander Graf10474ae2009-09-15 11:37:46 +02003614 int r;
Avi Kivity1b6c0162007-05-24 13:03:52 +03003615
Rusty Russell7f59f492008-12-07 21:25:45 +10303616 if (cpumask_test_cpu(cpu, cpus_hardware_enabled))
Avi Kivity1b6c0162007-05-24 13:03:52 +03003617 return;
Alexander Graf10474ae2009-09-15 11:37:46 +02003618
Rusty Russell7f59f492008-12-07 21:25:45 +10303619 cpumask_set_cpu(cpu, cpus_hardware_enabled);
Alexander Graf10474ae2009-09-15 11:37:46 +02003620
Radim Krčmář13a34e02014-08-28 15:13:03 +02003621 r = kvm_arch_hardware_enable();
Alexander Graf10474ae2009-09-15 11:37:46 +02003622
3623 if (r) {
3624 cpumask_clear_cpu(cpu, cpus_hardware_enabled);
3625 atomic_inc(&hardware_enable_failed);
Xiubo Li1170adc2015-02-26 14:58:26 +08003626 pr_info("kvm: enabling virtualization on CPU%d failed\n", cpu);
Alexander Graf10474ae2009-09-15 11:37:46 +02003627 }
Avi Kivity1b6c0162007-05-24 13:03:52 +03003628}
3629
Thomas Gleixner8c18b2d2016-07-13 17:16:37 +00003630static int kvm_starting_cpu(unsigned int cpu)
Takuya Yoshikawa75b71272010-11-16 17:37:41 +09003631{
Paolo Bonzini4a937f92013-09-10 12:58:35 +02003632 raw_spin_lock(&kvm_count_lock);
Paolo Bonzini4fa92fb2013-09-10 12:57:17 +02003633 if (kvm_usage_count)
3634 hardware_enable_nolock(NULL);
Paolo Bonzini4a937f92013-09-10 12:58:35 +02003635 raw_spin_unlock(&kvm_count_lock);
Thomas Gleixner8c18b2d2016-07-13 17:16:37 +00003636 return 0;
Takuya Yoshikawa75b71272010-11-16 17:37:41 +09003637}
3638
3639static void hardware_disable_nolock(void *junk)
Avi Kivity1b6c0162007-05-24 13:03:52 +03003640{
3641 int cpu = raw_smp_processor_id();
3642
Rusty Russell7f59f492008-12-07 21:25:45 +10303643 if (!cpumask_test_cpu(cpu, cpus_hardware_enabled))
Avi Kivity1b6c0162007-05-24 13:03:52 +03003644 return;
Rusty Russell7f59f492008-12-07 21:25:45 +10303645 cpumask_clear_cpu(cpu, cpus_hardware_enabled);
Radim Krčmář13a34e02014-08-28 15:13:03 +02003646 kvm_arch_hardware_disable();
Avi Kivity1b6c0162007-05-24 13:03:52 +03003647}
3648
Thomas Gleixner8c18b2d2016-07-13 17:16:37 +00003649static int kvm_dying_cpu(unsigned int cpu)
Takuya Yoshikawa75b71272010-11-16 17:37:41 +09003650{
Paolo Bonzini4a937f92013-09-10 12:58:35 +02003651 raw_spin_lock(&kvm_count_lock);
Paolo Bonzini4fa92fb2013-09-10 12:57:17 +02003652 if (kvm_usage_count)
3653 hardware_disable_nolock(NULL);
Paolo Bonzini4a937f92013-09-10 12:58:35 +02003654 raw_spin_unlock(&kvm_count_lock);
Thomas Gleixner8c18b2d2016-07-13 17:16:37 +00003655 return 0;
Takuya Yoshikawa75b71272010-11-16 17:37:41 +09003656}
3657
Alexander Graf10474ae2009-09-15 11:37:46 +02003658static void hardware_disable_all_nolock(void)
3659{
3660 BUG_ON(!kvm_usage_count);
3661
3662 kvm_usage_count--;
3663 if (!kvm_usage_count)
Takuya Yoshikawa75b71272010-11-16 17:37:41 +09003664 on_each_cpu(hardware_disable_nolock, NULL, 1);
Alexander Graf10474ae2009-09-15 11:37:46 +02003665}
3666
3667static void hardware_disable_all(void)
3668{
Paolo Bonzini4a937f92013-09-10 12:58:35 +02003669 raw_spin_lock(&kvm_count_lock);
Alexander Graf10474ae2009-09-15 11:37:46 +02003670 hardware_disable_all_nolock();
Paolo Bonzini4a937f92013-09-10 12:58:35 +02003671 raw_spin_unlock(&kvm_count_lock);
Alexander Graf10474ae2009-09-15 11:37:46 +02003672}
3673
3674static int hardware_enable_all(void)
3675{
3676 int r = 0;
3677
Paolo Bonzini4a937f92013-09-10 12:58:35 +02003678 raw_spin_lock(&kvm_count_lock);
Alexander Graf10474ae2009-09-15 11:37:46 +02003679
3680 kvm_usage_count++;
3681 if (kvm_usage_count == 1) {
3682 atomic_set(&hardware_enable_failed, 0);
Takuya Yoshikawa75b71272010-11-16 17:37:41 +09003683 on_each_cpu(hardware_enable_nolock, NULL, 1);
Alexander Graf10474ae2009-09-15 11:37:46 +02003684
3685 if (atomic_read(&hardware_enable_failed)) {
3686 hardware_disable_all_nolock();
3687 r = -EBUSY;
3688 }
3689 }
3690
Paolo Bonzini4a937f92013-09-10 12:58:35 +02003691 raw_spin_unlock(&kvm_count_lock);
Alexander Graf10474ae2009-09-15 11:37:46 +02003692
3693 return r;
3694}
3695
Rusty Russell9a2b85c2007-07-17 23:17:55 +10003696static int kvm_reboot(struct notifier_block *notifier, unsigned long val,
Mike Dayd77c26f2007-10-08 09:02:08 -04003697 void *v)
Rusty Russell9a2b85c2007-07-17 23:17:55 +10003698{
Sheng Yang8e1c1812009-04-29 11:09:04 +08003699 /*
3700 * Some (well, at least mine) BIOSes hang on reboot if
3701 * in vmx root mode.
3702 *
3703 * And Intel TXT required VMX off for all cpu when system shutdown.
3704 */
Xiubo Li1170adc2015-02-26 14:58:26 +08003705 pr_info("kvm: exiting hardware virtualization\n");
Sheng Yang8e1c1812009-04-29 11:09:04 +08003706 kvm_rebooting = true;
Takuya Yoshikawa75b71272010-11-16 17:37:41 +09003707 on_each_cpu(hardware_disable_nolock, NULL, 1);
Rusty Russell9a2b85c2007-07-17 23:17:55 +10003708 return NOTIFY_OK;
3709}
3710
3711static struct notifier_block kvm_reboot_notifier = {
3712 .notifier_call = kvm_reboot,
3713 .priority = 0,
3714};
3715
Marcelo Tosattie93f8a02009-12-23 14:35:24 -02003716static void kvm_io_bus_destroy(struct kvm_io_bus *bus)
Gregory Haskins2eeb2e92007-05-31 14:08:53 -04003717{
3718 int i;
3719
3720 for (i = 0; i < bus->dev_count; i++) {
Sasha Levin743eeb02011-07-27 16:00:48 +03003721 struct kvm_io_device *pos = bus->range[i].dev;
Gregory Haskins2eeb2e92007-05-31 14:08:53 -04003722
3723 kvm_iodevice_destructor(pos);
3724 }
Marcelo Tosattie93f8a02009-12-23 14:35:24 -02003725 kfree(bus);
Gregory Haskins2eeb2e92007-05-31 14:08:53 -04003726}
3727
Paolo Bonzinic21fbff2013-08-27 15:41:41 +02003728static inline int kvm_io_bus_cmp(const struct kvm_io_range *r1,
Xiubo Li20e87b72015-02-26 14:58:25 +08003729 const struct kvm_io_range *r2)
Sasha Levin743eeb02011-07-27 16:00:48 +03003730{
Jason Wang8f4216c72015-09-15 14:41:57 +08003731 gpa_t addr1 = r1->addr;
3732 gpa_t addr2 = r2->addr;
3733
3734 if (addr1 < addr2)
Sasha Levin743eeb02011-07-27 16:00:48 +03003735 return -1;
Jason Wang8f4216c72015-09-15 14:41:57 +08003736
3737 /* If r2->len == 0, match the exact address. If r2->len != 0,
3738 * accept any overlapping write. Any order is acceptable for
3739 * overlapping ranges, because kvm_io_bus_get_first_dev ensures
3740 * we process all of them.
3741 */
3742 if (r2->len) {
3743 addr1 += r1->len;
3744 addr2 += r2->len;
3745 }
3746
3747 if (addr1 > addr2)
Sasha Levin743eeb02011-07-27 16:00:48 +03003748 return 1;
Jason Wang8f4216c72015-09-15 14:41:57 +08003749
Sasha Levin743eeb02011-07-27 16:00:48 +03003750 return 0;
3751}
3752
Paolo Bonzinia343c9b2013-07-16 13:03:29 +02003753static int kvm_io_bus_sort_cmp(const void *p1, const void *p2)
3754{
Paolo Bonzinic21fbff2013-08-27 15:41:41 +02003755 return kvm_io_bus_cmp(p1, p2);
Paolo Bonzinia343c9b2013-07-16 13:03:29 +02003756}
3757
Geoff Levand39369f72013-04-05 19:20:30 +00003758static int kvm_io_bus_get_first_dev(struct kvm_io_bus *bus,
Sasha Levin743eeb02011-07-27 16:00:48 +03003759 gpa_t addr, int len)
3760{
3761 struct kvm_io_range *range, key;
3762 int off;
3763
3764 key = (struct kvm_io_range) {
3765 .addr = addr,
3766 .len = len,
3767 };
3768
3769 range = bsearch(&key, bus->range, bus->dev_count,
3770 sizeof(struct kvm_io_range), kvm_io_bus_sort_cmp);
3771 if (range == NULL)
3772 return -ENOENT;
3773
3774 off = range - bus->range;
3775
Paolo Bonzinic21fbff2013-08-27 15:41:41 +02003776 while (off > 0 && kvm_io_bus_cmp(&key, &bus->range[off-1]) == 0)
Sasha Levin743eeb02011-07-27 16:00:48 +03003777 off--;
3778
3779 return off;
3780}
3781
Nikolay Nikolaeve32edf42015-03-26 14:39:28 +00003782static int __kvm_io_bus_write(struct kvm_vcpu *vcpu, struct kvm_io_bus *bus,
Cornelia Huck126a5af2013-07-03 16:30:53 +02003783 struct kvm_io_range *range, const void *val)
3784{
3785 int idx;
3786
3787 idx = kvm_io_bus_get_first_dev(bus, range->addr, range->len);
3788 if (idx < 0)
3789 return -EOPNOTSUPP;
3790
3791 while (idx < bus->dev_count &&
Paolo Bonzinic21fbff2013-08-27 15:41:41 +02003792 kvm_io_bus_cmp(range, &bus->range[idx]) == 0) {
Nikolay Nikolaeve32edf42015-03-26 14:39:28 +00003793 if (!kvm_iodevice_write(vcpu, bus->range[idx].dev, range->addr,
Cornelia Huck126a5af2013-07-03 16:30:53 +02003794 range->len, val))
3795 return idx;
3796 idx++;
3797 }
3798
3799 return -EOPNOTSUPP;
3800}
3801
Michael S. Tsirkinbda90202009-06-29 22:24:32 +03003802/* kvm_io_bus_write - called under kvm->slots_lock */
Nikolay Nikolaeve32edf42015-03-26 14:39:28 +00003803int kvm_io_bus_write(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, gpa_t addr,
Michael S. Tsirkinbda90202009-06-29 22:24:32 +03003804 int len, const void *val)
Gregory Haskins2eeb2e92007-05-31 14:08:53 -04003805{
Cornelia Huck126a5af2013-07-03 16:30:53 +02003806 struct kvm_io_bus *bus;
3807 struct kvm_io_range range;
3808 int r;
3809
3810 range = (struct kvm_io_range) {
3811 .addr = addr,
3812 .len = len,
3813 };
3814
Nikolay Nikolaeve32edf42015-03-26 14:39:28 +00003815 bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu);
David Hildenbrand90db1042017-03-23 18:24:19 +01003816 if (!bus)
3817 return -ENOMEM;
Nikolay Nikolaeve32edf42015-03-26 14:39:28 +00003818 r = __kvm_io_bus_write(vcpu, bus, &range, val);
Cornelia Huck126a5af2013-07-03 16:30:53 +02003819 return r < 0 ? r : 0;
3820}
Leo Yana2420102019-02-22 16:10:09 +08003821EXPORT_SYMBOL_GPL(kvm_io_bus_write);
Cornelia Huck126a5af2013-07-03 16:30:53 +02003822
3823/* kvm_io_bus_write_cookie - called under kvm->slots_lock */
Nikolay Nikolaeve32edf42015-03-26 14:39:28 +00003824int kvm_io_bus_write_cookie(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx,
3825 gpa_t addr, int len, const void *val, long cookie)
Cornelia Huck126a5af2013-07-03 16:30:53 +02003826{
Lai Jiangshan90d83dc2010-04-19 17:41:23 +08003827 struct kvm_io_bus *bus;
Sasha Levin743eeb02011-07-27 16:00:48 +03003828 struct kvm_io_range range;
3829
3830 range = (struct kvm_io_range) {
3831 .addr = addr,
3832 .len = len,
3833 };
Lai Jiangshan90d83dc2010-04-19 17:41:23 +08003834
Nikolay Nikolaeve32edf42015-03-26 14:39:28 +00003835 bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu);
David Hildenbrand90db1042017-03-23 18:24:19 +01003836 if (!bus)
3837 return -ENOMEM;
Cornelia Huck126a5af2013-07-03 16:30:53 +02003838
3839 /* First try the device referenced by cookie. */
3840 if ((cookie >= 0) && (cookie < bus->dev_count) &&
Paolo Bonzinic21fbff2013-08-27 15:41:41 +02003841 (kvm_io_bus_cmp(&range, &bus->range[cookie]) == 0))
Nikolay Nikolaeve32edf42015-03-26 14:39:28 +00003842 if (!kvm_iodevice_write(vcpu, bus->range[cookie].dev, addr, len,
Cornelia Huck126a5af2013-07-03 16:30:53 +02003843 val))
3844 return cookie;
3845
3846 /*
3847 * cookie contained garbage; fall back to search and return the
3848 * correct cookie value.
3849 */
Nikolay Nikolaeve32edf42015-03-26 14:39:28 +00003850 return __kvm_io_bus_write(vcpu, bus, &range, val);
Cornelia Huck126a5af2013-07-03 16:30:53 +02003851}
3852
Nikolay Nikolaeve32edf42015-03-26 14:39:28 +00003853static int __kvm_io_bus_read(struct kvm_vcpu *vcpu, struct kvm_io_bus *bus,
3854 struct kvm_io_range *range, void *val)
Cornelia Huck126a5af2013-07-03 16:30:53 +02003855{
3856 int idx;
3857
3858 idx = kvm_io_bus_get_first_dev(bus, range->addr, range->len);
Sasha Levin743eeb02011-07-27 16:00:48 +03003859 if (idx < 0)
3860 return -EOPNOTSUPP;
3861
3862 while (idx < bus->dev_count &&
Paolo Bonzinic21fbff2013-08-27 15:41:41 +02003863 kvm_io_bus_cmp(range, &bus->range[idx]) == 0) {
Nikolay Nikolaeve32edf42015-03-26 14:39:28 +00003864 if (!kvm_iodevice_read(vcpu, bus->range[idx].dev, range->addr,
Cornelia Huck126a5af2013-07-03 16:30:53 +02003865 range->len, val))
3866 return idx;
Sasha Levin743eeb02011-07-27 16:00:48 +03003867 idx++;
3868 }
3869
Michael S. Tsirkinbda90202009-06-29 22:24:32 +03003870 return -EOPNOTSUPP;
3871}
Gregory Haskins2eeb2e92007-05-31 14:08:53 -04003872
Michael S. Tsirkinbda90202009-06-29 22:24:32 +03003873/* kvm_io_bus_read - called under kvm->slots_lock */
Nikolay Nikolaeve32edf42015-03-26 14:39:28 +00003874int kvm_io_bus_read(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, gpa_t addr,
Marcelo Tosattie93f8a02009-12-23 14:35:24 -02003875 int len, void *val)
Michael S. Tsirkinbda90202009-06-29 22:24:32 +03003876{
Cornelia Huck126a5af2013-07-03 16:30:53 +02003877 struct kvm_io_bus *bus;
3878 struct kvm_io_range range;
3879 int r;
3880
3881 range = (struct kvm_io_range) {
3882 .addr = addr,
3883 .len = len,
3884 };
3885
Nikolay Nikolaeve32edf42015-03-26 14:39:28 +00003886 bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu);
David Hildenbrand90db1042017-03-23 18:24:19 +01003887 if (!bus)
3888 return -ENOMEM;
Nikolay Nikolaeve32edf42015-03-26 14:39:28 +00003889 r = __kvm_io_bus_read(vcpu, bus, &range, val);
Cornelia Huck126a5af2013-07-03 16:30:53 +02003890 return r < 0 ? r : 0;
3891}
3892
Marcelo Tosatti79fac952009-12-23 14:35:26 -02003893/* Caller must hold slots_lock. */
Sasha Levin743eeb02011-07-27 16:00:48 +03003894int kvm_io_bus_register_dev(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr,
3895 int len, struct kvm_io_device *dev)
Michael S. Tsirkin6c474692009-06-29 22:24:26 +03003896{
Gal Hammerd4c67a72018-01-16 15:34:41 +02003897 int i;
Marcelo Tosattie93f8a02009-12-23 14:35:24 -02003898 struct kvm_io_bus *new_bus, *bus;
Gal Hammerd4c67a72018-01-16 15:34:41 +02003899 struct kvm_io_range range;
Gregory Haskins090b7af2009-07-07 17:08:44 -04003900
Christian Borntraeger4a12f952017-07-07 10:51:38 +02003901 bus = kvm_get_bus(kvm, bus_idx);
David Hildenbrand90db1042017-03-23 18:24:19 +01003902 if (!bus)
3903 return -ENOMEM;
3904
Amos Kong6ea34c92013-05-25 06:44:15 +08003905 /* exclude ioeventfd which is limited by maximum fd */
3906 if (bus->dev_count - bus->ioeventfd_count > NR_IOBUS_DEVS - 1)
Gregory Haskins090b7af2009-07-07 17:08:44 -04003907 return -ENOSPC;
3908
Gustavo A. R. Silva90952cd2019-01-30 17:07:47 +01003909 new_bus = kmalloc(struct_size(bus, range, bus->dev_count + 1),
Ben Gardonb12ce362019-02-11 11:02:49 -08003910 GFP_KERNEL_ACCOUNT);
Marcelo Tosattie93f8a02009-12-23 14:35:24 -02003911 if (!new_bus)
3912 return -ENOMEM;
Gal Hammerd4c67a72018-01-16 15:34:41 +02003913
3914 range = (struct kvm_io_range) {
3915 .addr = addr,
3916 .len = len,
3917 .dev = dev,
3918 };
3919
3920 for (i = 0; i < bus->dev_count; i++)
3921 if (kvm_io_bus_cmp(&bus->range[i], &range) > 0)
3922 break;
3923
3924 memcpy(new_bus, bus, sizeof(*bus) + i * sizeof(struct kvm_io_range));
3925 new_bus->dev_count++;
3926 new_bus->range[i] = range;
3927 memcpy(new_bus->range + i + 1, bus->range + i,
3928 (bus->dev_count - i) * sizeof(struct kvm_io_range));
Marcelo Tosattie93f8a02009-12-23 14:35:24 -02003929 rcu_assign_pointer(kvm->buses[bus_idx], new_bus);
3930 synchronize_srcu_expedited(&kvm->srcu);
3931 kfree(bus);
Gregory Haskins090b7af2009-07-07 17:08:44 -04003932
3933 return 0;
3934}
3935
Marcelo Tosatti79fac952009-12-23 14:35:26 -02003936/* Caller must hold slots_lock. */
David Hildenbrand90db1042017-03-23 18:24:19 +01003937void kvm_io_bus_unregister_dev(struct kvm *kvm, enum kvm_bus bus_idx,
3938 struct kvm_io_device *dev)
Gregory Haskins090b7af2009-07-07 17:08:44 -04003939{
David Hildenbrand90db1042017-03-23 18:24:19 +01003940 int i;
Marcelo Tosattie93f8a02009-12-23 14:35:24 -02003941 struct kvm_io_bus *new_bus, *bus;
Michael S. Tsirkin6c474692009-06-29 22:24:26 +03003942
Christian Borntraeger4a12f952017-07-07 10:51:38 +02003943 bus = kvm_get_bus(kvm, bus_idx);
Peter Xudf630b82017-03-15 16:01:17 +08003944 if (!bus)
David Hildenbrand90db1042017-03-23 18:24:19 +01003945 return;
Peter Xudf630b82017-03-15 16:01:17 +08003946
Amos Konga13007162012-03-09 12:17:32 +08003947 for (i = 0; i < bus->dev_count; i++)
3948 if (bus->range[i].dev == dev) {
Gregory Haskins090b7af2009-07-07 17:08:44 -04003949 break;
3950 }
Marcelo Tosattie93f8a02009-12-23 14:35:24 -02003951
David Hildenbrand90db1042017-03-23 18:24:19 +01003952 if (i == bus->dev_count)
3953 return;
Amos Konga13007162012-03-09 12:17:32 +08003954
Gustavo A. R. Silva90952cd2019-01-30 17:07:47 +01003955 new_bus = kmalloc(struct_size(bus, range, bus->dev_count - 1),
Ben Gardonb12ce362019-02-11 11:02:49 -08003956 GFP_KERNEL_ACCOUNT);
David Hildenbrand90db1042017-03-23 18:24:19 +01003957 if (!new_bus) {
3958 pr_err("kvm: failed to shrink bus, removing it completely\n");
3959 goto broken;
3960 }
Amos Konga13007162012-03-09 12:17:32 +08003961
3962 memcpy(new_bus, bus, sizeof(*bus) + i * sizeof(struct kvm_io_range));
3963 new_bus->dev_count--;
3964 memcpy(new_bus->range + i, bus->range + i + 1,
3965 (new_bus->dev_count - i) * sizeof(struct kvm_io_range));
Marcelo Tosattie93f8a02009-12-23 14:35:24 -02003966
David Hildenbrand90db1042017-03-23 18:24:19 +01003967broken:
Marcelo Tosattie93f8a02009-12-23 14:35:24 -02003968 rcu_assign_pointer(kvm->buses[bus_idx], new_bus);
3969 synchronize_srcu_expedited(&kvm->srcu);
3970 kfree(bus);
David Hildenbrand90db1042017-03-23 18:24:19 +01003971 return;
Gregory Haskins2eeb2e92007-05-31 14:08:53 -04003972}
3973
Andre Przywara8a39d002016-07-15 12:43:26 +01003974struct kvm_io_device *kvm_io_bus_get_dev(struct kvm *kvm, enum kvm_bus bus_idx,
3975 gpa_t addr)
3976{
3977 struct kvm_io_bus *bus;
3978 int dev_idx, srcu_idx;
3979 struct kvm_io_device *iodev = NULL;
3980
3981 srcu_idx = srcu_read_lock(&kvm->srcu);
3982
3983 bus = srcu_dereference(kvm->buses[bus_idx], &kvm->srcu);
David Hildenbrand90db1042017-03-23 18:24:19 +01003984 if (!bus)
3985 goto out_unlock;
Andre Przywara8a39d002016-07-15 12:43:26 +01003986
3987 dev_idx = kvm_io_bus_get_first_dev(bus, addr, 1);
3988 if (dev_idx < 0)
3989 goto out_unlock;
3990
3991 iodev = bus->range[dev_idx].dev;
3992
3993out_unlock:
3994 srcu_read_unlock(&kvm->srcu, srcu_idx);
3995
3996 return iodev;
3997}
3998EXPORT_SYMBOL_GPL(kvm_io_bus_get_dev);
3999
Janosch Frank536a6f82016-05-18 13:26:23 +02004000static int kvm_debugfs_open(struct inode *inode, struct file *file,
4001 int (*get)(void *, u64 *), int (*set)(void *, u64),
4002 const char *fmt)
4003{
4004 struct kvm_stat_data *stat_data = (struct kvm_stat_data *)
4005 inode->i_private;
4006
4007 /* The debugfs files are a reference to the kvm struct which
4008 * is still valid when kvm_destroy_vm is called.
4009 * To avoid the race between open and the removal of the debugfs
4010 * directory we test against the users count.
4011 */
Elena Reshetovae3736c32017-02-20 13:06:21 +02004012 if (!refcount_inc_not_zero(&stat_data->kvm->users_count))
Janosch Frank536a6f82016-05-18 13:26:23 +02004013 return -ENOENT;
4014
Paolo Bonzini833b45d2019-09-30 18:48:44 +02004015 if (simple_attr_open(inode, file, get,
4016 stat_data->mode & S_IWUGO ? set : NULL,
4017 fmt)) {
Janosch Frank536a6f82016-05-18 13:26:23 +02004018 kvm_put_kvm(stat_data->kvm);
4019 return -ENOMEM;
4020 }
4021
4022 return 0;
4023}
4024
4025static int kvm_debugfs_release(struct inode *inode, struct file *file)
4026{
4027 struct kvm_stat_data *stat_data = (struct kvm_stat_data *)
4028 inode->i_private;
4029
4030 simple_attr_release(inode, file);
4031 kvm_put_kvm(stat_data->kvm);
4032
4033 return 0;
4034}
4035
4036static int vm_stat_get_per_vm(void *data, u64 *val)
4037{
4038 struct kvm_stat_data *stat_data = (struct kvm_stat_data *)data;
4039
Suraj Jitindar Singh8a7e75d2016-08-02 14:03:22 +10004040 *val = *(ulong *)((void *)stat_data->kvm + stat_data->offset);
Janosch Frank536a6f82016-05-18 13:26:23 +02004041
4042 return 0;
4043}
4044
Suraj Jitindar Singhce35ef22016-10-19 13:49:47 +11004045static int vm_stat_clear_per_vm(void *data, u64 val)
4046{
4047 struct kvm_stat_data *stat_data = (struct kvm_stat_data *)data;
4048
4049 if (val)
4050 return -EINVAL;
4051
4052 *(ulong *)((void *)stat_data->kvm + stat_data->offset) = 0;
4053
4054 return 0;
4055}
4056
Janosch Frank536a6f82016-05-18 13:26:23 +02004057static int vm_stat_get_per_vm_open(struct inode *inode, struct file *file)
4058{
4059 __simple_attr_check_format("%llu\n", 0ull);
4060 return kvm_debugfs_open(inode, file, vm_stat_get_per_vm,
Suraj Jitindar Singhce35ef22016-10-19 13:49:47 +11004061 vm_stat_clear_per_vm, "%llu\n");
Janosch Frank536a6f82016-05-18 13:26:23 +02004062}
4063
4064static const struct file_operations vm_stat_get_per_vm_fops = {
4065 .owner = THIS_MODULE,
4066 .open = vm_stat_get_per_vm_open,
4067 .release = kvm_debugfs_release,
4068 .read = simple_attr_read,
4069 .write = simple_attr_write,
Geliang Tang3bed8882017-05-06 23:37:19 +08004070 .llseek = no_llseek,
Janosch Frank536a6f82016-05-18 13:26:23 +02004071};
4072
4073static int vcpu_stat_get_per_vm(void *data, u64 *val)
4074{
4075 int i;
4076 struct kvm_stat_data *stat_data = (struct kvm_stat_data *)data;
4077 struct kvm_vcpu *vcpu;
4078
4079 *val = 0;
4080
4081 kvm_for_each_vcpu(i, vcpu, stat_data->kvm)
Suraj Jitindar Singh8a7e75d2016-08-02 14:03:22 +10004082 *val += *(u64 *)((void *)vcpu + stat_data->offset);
Janosch Frank536a6f82016-05-18 13:26:23 +02004083
4084 return 0;
4085}
4086
Suraj Jitindar Singhce35ef22016-10-19 13:49:47 +11004087static int vcpu_stat_clear_per_vm(void *data, u64 val)
4088{
4089 int i;
4090 struct kvm_stat_data *stat_data = (struct kvm_stat_data *)data;
4091 struct kvm_vcpu *vcpu;
4092
4093 if (val)
4094 return -EINVAL;
4095
4096 kvm_for_each_vcpu(i, vcpu, stat_data->kvm)
4097 *(u64 *)((void *)vcpu + stat_data->offset) = 0;
4098
4099 return 0;
4100}
4101
Janosch Frank536a6f82016-05-18 13:26:23 +02004102static int vcpu_stat_get_per_vm_open(struct inode *inode, struct file *file)
4103{
4104 __simple_attr_check_format("%llu\n", 0ull);
4105 return kvm_debugfs_open(inode, file, vcpu_stat_get_per_vm,
Suraj Jitindar Singhce35ef22016-10-19 13:49:47 +11004106 vcpu_stat_clear_per_vm, "%llu\n");
Janosch Frank536a6f82016-05-18 13:26:23 +02004107}
4108
4109static const struct file_operations vcpu_stat_get_per_vm_fops = {
4110 .owner = THIS_MODULE,
4111 .open = vcpu_stat_get_per_vm_open,
4112 .release = kvm_debugfs_release,
4113 .read = simple_attr_read,
4114 .write = simple_attr_write,
Geliang Tang3bed8882017-05-06 23:37:19 +08004115 .llseek = no_llseek,
Janosch Frank536a6f82016-05-18 13:26:23 +02004116};
4117
4118static const struct file_operations *stat_fops_per_vm[] = {
4119 [KVM_STAT_VCPU] = &vcpu_stat_get_per_vm_fops,
4120 [KVM_STAT_VM] = &vm_stat_get_per_vm_fops,
4121};
4122
Christoph Hellwig8b88b092008-02-08 04:20:26 -08004123static int vm_stat_get(void *_offset, u64 *val)
Avi Kivityba1389b2007-11-18 16:24:12 +02004124{
4125 unsigned offset = (long)_offset;
Avi Kivityba1389b2007-11-18 16:24:12 +02004126 struct kvm *kvm;
Janosch Frank536a6f82016-05-18 13:26:23 +02004127 struct kvm_stat_data stat_tmp = {.offset = offset};
4128 u64 tmp_val;
Avi Kivityba1389b2007-11-18 16:24:12 +02004129
Christoph Hellwig8b88b092008-02-08 04:20:26 -08004130 *val = 0;
Junaid Shahid0d9ce162019-01-03 17:14:28 -08004131 mutex_lock(&kvm_lock);
Janosch Frank536a6f82016-05-18 13:26:23 +02004132 list_for_each_entry(kvm, &vm_list, vm_list) {
4133 stat_tmp.kvm = kvm;
4134 vm_stat_get_per_vm((void *)&stat_tmp, &tmp_val);
4135 *val += tmp_val;
4136 }
Junaid Shahid0d9ce162019-01-03 17:14:28 -08004137 mutex_unlock(&kvm_lock);
Christoph Hellwig8b88b092008-02-08 04:20:26 -08004138 return 0;
Avi Kivityba1389b2007-11-18 16:24:12 +02004139}
4140
Suraj Jitindar Singhce35ef22016-10-19 13:49:47 +11004141static int vm_stat_clear(void *_offset, u64 val)
4142{
4143 unsigned offset = (long)_offset;
4144 struct kvm *kvm;
4145 struct kvm_stat_data stat_tmp = {.offset = offset};
4146
4147 if (val)
4148 return -EINVAL;
4149
Junaid Shahid0d9ce162019-01-03 17:14:28 -08004150 mutex_lock(&kvm_lock);
Suraj Jitindar Singhce35ef22016-10-19 13:49:47 +11004151 list_for_each_entry(kvm, &vm_list, vm_list) {
4152 stat_tmp.kvm = kvm;
4153 vm_stat_clear_per_vm((void *)&stat_tmp, 0);
4154 }
Junaid Shahid0d9ce162019-01-03 17:14:28 -08004155 mutex_unlock(&kvm_lock);
Suraj Jitindar Singhce35ef22016-10-19 13:49:47 +11004156
4157 return 0;
4158}
4159
4160DEFINE_SIMPLE_ATTRIBUTE(vm_stat_fops, vm_stat_get, vm_stat_clear, "%llu\n");
Avi Kivityba1389b2007-11-18 16:24:12 +02004161
Christoph Hellwig8b88b092008-02-08 04:20:26 -08004162static int vcpu_stat_get(void *_offset, u64 *val)
Avi Kivity1165f5f2007-04-19 17:27:43 +03004163{
4164 unsigned offset = (long)_offset;
Avi Kivity1165f5f2007-04-19 17:27:43 +03004165 struct kvm *kvm;
Janosch Frank536a6f82016-05-18 13:26:23 +02004166 struct kvm_stat_data stat_tmp = {.offset = offset};
4167 u64 tmp_val;
Avi Kivity1165f5f2007-04-19 17:27:43 +03004168
Christoph Hellwig8b88b092008-02-08 04:20:26 -08004169 *val = 0;
Junaid Shahid0d9ce162019-01-03 17:14:28 -08004170 mutex_lock(&kvm_lock);
Janosch Frank536a6f82016-05-18 13:26:23 +02004171 list_for_each_entry(kvm, &vm_list, vm_list) {
4172 stat_tmp.kvm = kvm;
4173 vcpu_stat_get_per_vm((void *)&stat_tmp, &tmp_val);
4174 *val += tmp_val;
4175 }
Junaid Shahid0d9ce162019-01-03 17:14:28 -08004176 mutex_unlock(&kvm_lock);
Christoph Hellwig8b88b092008-02-08 04:20:26 -08004177 return 0;
Avi Kivity1165f5f2007-04-19 17:27:43 +03004178}
4179
Suraj Jitindar Singhce35ef22016-10-19 13:49:47 +11004180static int vcpu_stat_clear(void *_offset, u64 val)
4181{
4182 unsigned offset = (long)_offset;
4183 struct kvm *kvm;
4184 struct kvm_stat_data stat_tmp = {.offset = offset};
4185
4186 if (val)
4187 return -EINVAL;
4188
Junaid Shahid0d9ce162019-01-03 17:14:28 -08004189 mutex_lock(&kvm_lock);
Suraj Jitindar Singhce35ef22016-10-19 13:49:47 +11004190 list_for_each_entry(kvm, &vm_list, vm_list) {
4191 stat_tmp.kvm = kvm;
4192 vcpu_stat_clear_per_vm((void *)&stat_tmp, 0);
4193 }
Junaid Shahid0d9ce162019-01-03 17:14:28 -08004194 mutex_unlock(&kvm_lock);
Suraj Jitindar Singhce35ef22016-10-19 13:49:47 +11004195
4196 return 0;
4197}
4198
4199DEFINE_SIMPLE_ATTRIBUTE(vcpu_stat_fops, vcpu_stat_get, vcpu_stat_clear,
4200 "%llu\n");
Avi Kivityba1389b2007-11-18 16:24:12 +02004201
Alexey Dobriyan828c0952009-10-01 15:43:56 -07004202static const struct file_operations *stat_fops[] = {
Avi Kivityba1389b2007-11-18 16:24:12 +02004203 [KVM_STAT_VCPU] = &vcpu_stat_fops,
4204 [KVM_STAT_VM] = &vm_stat_fops,
4205};
Avi Kivity1165f5f2007-04-19 17:27:43 +03004206
Claudio Imbrenda286de8f2017-07-12 17:56:44 +02004207static void kvm_uevent_notify_change(unsigned int type, struct kvm *kvm)
4208{
4209 struct kobj_uevent_env *env;
Claudio Imbrenda286de8f2017-07-12 17:56:44 +02004210 unsigned long long created, active;
4211
4212 if (!kvm_dev.this_device || !kvm)
4213 return;
4214
Junaid Shahid0d9ce162019-01-03 17:14:28 -08004215 mutex_lock(&kvm_lock);
Claudio Imbrenda286de8f2017-07-12 17:56:44 +02004216 if (type == KVM_EVENT_CREATE_VM) {
4217 kvm_createvm_count++;
4218 kvm_active_vms++;
4219 } else if (type == KVM_EVENT_DESTROY_VM) {
4220 kvm_active_vms--;
4221 }
4222 created = kvm_createvm_count;
4223 active = kvm_active_vms;
Junaid Shahid0d9ce162019-01-03 17:14:28 -08004224 mutex_unlock(&kvm_lock);
Claudio Imbrenda286de8f2017-07-12 17:56:44 +02004225
Ben Gardonb12ce362019-02-11 11:02:49 -08004226 env = kzalloc(sizeof(*env), GFP_KERNEL_ACCOUNT);
Claudio Imbrenda286de8f2017-07-12 17:56:44 +02004227 if (!env)
4228 return;
4229
4230 add_uevent_var(env, "CREATED=%llu", created);
4231 add_uevent_var(env, "COUNT=%llu", active);
4232
Claudio Imbrendafdeaf7e2017-07-24 13:40:03 +02004233 if (type == KVM_EVENT_CREATE_VM) {
Claudio Imbrenda286de8f2017-07-12 17:56:44 +02004234 add_uevent_var(env, "EVENT=create");
Claudio Imbrendafdeaf7e2017-07-24 13:40:03 +02004235 kvm->userspace_pid = task_pid_nr(current);
4236 } else if (type == KVM_EVENT_DESTROY_VM) {
Claudio Imbrenda286de8f2017-07-12 17:56:44 +02004237 add_uevent_var(env, "EVENT=destroy");
Claudio Imbrendafdeaf7e2017-07-24 13:40:03 +02004238 }
4239 add_uevent_var(env, "PID=%d", kvm->userspace_pid);
Claudio Imbrenda286de8f2017-07-12 17:56:44 +02004240
Greg Kroah-Hartman8ed05792019-02-28 16:34:37 +01004241 if (!IS_ERR_OR_NULL(kvm->debugfs_dentry)) {
Ben Gardonb12ce362019-02-11 11:02:49 -08004242 char *tmp, *p = kmalloc(PATH_MAX, GFP_KERNEL_ACCOUNT);
Claudio Imbrenda286de8f2017-07-12 17:56:44 +02004243
Claudio Imbrendafdeaf7e2017-07-24 13:40:03 +02004244 if (p) {
4245 tmp = dentry_path_raw(kvm->debugfs_dentry, p, PATH_MAX);
4246 if (!IS_ERR(tmp))
4247 add_uevent_var(env, "STATS_PATH=%s", tmp);
4248 kfree(p);
Claudio Imbrenda286de8f2017-07-12 17:56:44 +02004249 }
4250 }
4251 /* no need for checks, since we are adding at most only 5 keys */
4252 env->envp[env->envp_idx++] = NULL;
4253 kobject_uevent_env(&kvm_dev.this_device->kobj, KOBJ_CHANGE, env->envp);
4254 kfree(env);
Claudio Imbrenda286de8f2017-07-12 17:56:44 +02004255}
4256
Greg Kroah-Hartman929f45e2018-05-29 18:22:04 +02004257static void kvm_init_debug(void)
Avi Kivity6aa8b732006-12-10 02:21:36 -08004258{
4259 struct kvm_stats_debugfs_item *p;
4260
Hollis Blanchard76f7c872008-04-15 16:05:42 -05004261 kvm_debugfs_dir = debugfs_create_dir("kvm", NULL);
Hamo4f69b682011-12-15 14:23:16 +08004262
Janosch Frank536a6f82016-05-18 13:26:23 +02004263 kvm_debugfs_num_entries = 0;
4264 for (p = debugfs_entries; p->name; ++p, kvm_debugfs_num_entries++) {
Paolo Bonzini833b45d2019-09-30 18:48:44 +02004265 int mode = p->mode ? p->mode : 0644;
4266 debugfs_create_file(p->name, mode, kvm_debugfs_dir,
Greg Kroah-Hartman929f45e2018-05-29 18:22:04 +02004267 (void *)(long)p->offset,
4268 stat_fops[p->kind]);
Hamo4f69b682011-12-15 14:23:16 +08004269 }
Avi Kivity6aa8b732006-12-10 02:21:36 -08004270}
4271
Rafael J. Wysockifb3600c2011-03-23 22:16:23 +01004272static int kvm_suspend(void)
Avi Kivity59ae6c62007-02-12 00:54:48 -08004273{
Alexander Graf10474ae2009-09-15 11:37:46 +02004274 if (kvm_usage_count)
Takuya Yoshikawa75b71272010-11-16 17:37:41 +09004275 hardware_disable_nolock(NULL);
Avi Kivity59ae6c62007-02-12 00:54:48 -08004276 return 0;
4277}
4278
Rafael J. Wysockifb3600c2011-03-23 22:16:23 +01004279static void kvm_resume(void)
Avi Kivity59ae6c62007-02-12 00:54:48 -08004280{
Zachary Amsdenca84d1a2010-08-19 22:07:28 -10004281 if (kvm_usage_count) {
Wanpeng Li2eb06c32019-05-17 16:49:49 +08004282#ifdef CONFIG_LOCKDEP
4283 WARN_ON(lockdep_is_held(&kvm_count_lock));
4284#endif
Takuya Yoshikawa75b71272010-11-16 17:37:41 +09004285 hardware_enable_nolock(NULL);
Zachary Amsdenca84d1a2010-08-19 22:07:28 -10004286 }
Avi Kivity59ae6c62007-02-12 00:54:48 -08004287}
4288
Rafael J. Wysockifb3600c2011-03-23 22:16:23 +01004289static struct syscore_ops kvm_syscore_ops = {
Avi Kivity59ae6c62007-02-12 00:54:48 -08004290 .suspend = kvm_suspend,
4291 .resume = kvm_resume,
4292};
4293
Avi Kivity15ad7142007-07-11 18:17:21 +03004294static inline
4295struct kvm_vcpu *preempt_notifier_to_vcpu(struct preempt_notifier *pn)
4296{
4297 return container_of(pn, struct kvm_vcpu, preempt_notifier);
4298}
4299
4300static void kvm_sched_in(struct preempt_notifier *pn, int cpu)
4301{
4302 struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn);
Xiubo Lif95ef0cd2015-02-26 14:58:23 +08004303
Wanpeng Li046ddee2019-08-01 11:30:14 +08004304 WRITE_ONCE(vcpu->preempted, false);
Wanpeng Lid73eb572019-07-18 19:39:06 +08004305 WRITE_ONCE(vcpu->ready, false);
Avi Kivity15ad7142007-07-11 18:17:21 +03004306
Radim Krčmáře790d9e2014-08-21 18:08:05 +02004307 kvm_arch_sched_in(vcpu, cpu);
4308
Zhang Xiantaoe9b11c12007-11-14 20:38:21 +08004309 kvm_arch_vcpu_load(vcpu, cpu);
Avi Kivity15ad7142007-07-11 18:17:21 +03004310}
4311
4312static void kvm_sched_out(struct preempt_notifier *pn,
4313 struct task_struct *next)
4314{
4315 struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn);
4316
Wanpeng Lid73eb572019-07-18 19:39:06 +08004317 if (current->state == TASK_RUNNING) {
Wanpeng Li046ddee2019-08-01 11:30:14 +08004318 WRITE_ONCE(vcpu->preempted, true);
Wanpeng Lid73eb572019-07-18 19:39:06 +08004319 WRITE_ONCE(vcpu->ready, true);
4320 }
Zhang Xiantaoe9b11c12007-11-14 20:38:21 +08004321 kvm_arch_vcpu_put(vcpu);
Avi Kivity15ad7142007-07-11 18:17:21 +03004322}
4323
Sean Christophersonf257d6d2019-04-19 22:18:17 -07004324static void check_processor_compat(void *rtn)
4325{
4326 *(int *)rtn = kvm_arch_check_processor_compat();
4327}
4328
Avi Kivity0ee75be2010-04-28 15:39:01 +03004329int kvm_init(void *opaque, unsigned vcpu_size, unsigned vcpu_align,
Rusty Russellc16f8622007-07-30 21:12:19 +10004330 struct module *module)
Avi Kivity6aa8b732006-12-10 02:21:36 -08004331{
4332 int r;
Yang, Sheng002c7f72007-07-31 14:23:01 +03004333 int cpu;
Avi Kivity6aa8b732006-12-10 02:21:36 -08004334
Zhang Xiantaof8c16bb2007-11-14 20:40:21 +08004335 r = kvm_arch_init(opaque);
4336 if (r)
Zhang Xiantaod23087842007-11-29 15:35:39 +08004337 goto out_fail;
Zhang Xiantaocb498ea2007-11-14 20:39:31 +08004338
Asias He7dac16c2013-05-08 10:57:29 +08004339 /*
4340 * kvm_arch_init makes sure there's at most one caller
4341 * for architectures that support multiple implementations,
4342 * like intel and amd on x86.
Paolo Bonzini36343f62016-10-26 13:35:56 +02004343 * kvm_arch_init must be called before kvm_irqfd_init to avoid creating
4344 * conflicts in case kvm is already setup for another implementation.
Asias He7dac16c2013-05-08 10:57:29 +08004345 */
Paolo Bonzini36343f62016-10-26 13:35:56 +02004346 r = kvm_irqfd_init();
4347 if (r)
4348 goto out_irqfd;
Asias He7dac16c2013-05-08 10:57:29 +08004349
Avi Kivity8437a6172009-06-06 14:52:35 -07004350 if (!zalloc_cpumask_var(&cpus_hardware_enabled, GFP_KERNEL)) {
Rusty Russell7f59f492008-12-07 21:25:45 +10304351 r = -ENOMEM;
4352 goto out_free_0;
4353 }
4354
Zhang Xiantaoe9b11c12007-11-14 20:38:21 +08004355 r = kvm_arch_hardware_setup();
Avi Kivity6aa8b732006-12-10 02:21:36 -08004356 if (r < 0)
Miaohe Linfaf0be22019-11-23 10:45:50 +08004357 goto out_free_1;
Avi Kivity6aa8b732006-12-10 02:21:36 -08004358
Yang, Sheng002c7f72007-07-31 14:23:01 +03004359 for_each_online_cpu(cpu) {
Sean Christophersonf257d6d2019-04-19 22:18:17 -07004360 smp_call_function_single(cpu, check_processor_compat, &r, 1);
Yang, Sheng002c7f72007-07-31 14:23:01 +03004361 if (r < 0)
Miaohe Linfaf0be22019-11-23 10:45:50 +08004362 goto out_free_2;
Yang, Sheng002c7f72007-07-31 14:23:01 +03004363 }
4364
Thomas Gleixner73c1b412016-12-21 20:19:54 +01004365 r = cpuhp_setup_state_nocalls(CPUHP_AP_KVM_STARTING, "kvm/cpu:starting",
Thomas Gleixner8c18b2d2016-07-13 17:16:37 +00004366 kvm_starting_cpu, kvm_dying_cpu);
Avi Kivity774c47f2007-02-12 00:54:47 -08004367 if (r)
Zhang Xiantaod23087842007-11-29 15:35:39 +08004368 goto out_free_2;
Avi Kivity6aa8b732006-12-10 02:21:36 -08004369 register_reboot_notifier(&kvm_reboot_notifier);
4370
Rusty Russellc16f8622007-07-30 21:12:19 +10004371 /* A kmem cache lets us meet the alignment requirements of fx_save. */
Avi Kivity0ee75be2010-04-28 15:39:01 +03004372 if (!vcpu_align)
4373 vcpu_align = __alignof__(struct kvm_vcpu);
Paolo Bonzini46515732017-10-26 15:45:46 +02004374 kvm_vcpu_cache =
4375 kmem_cache_create_usercopy("kvm_vcpu", vcpu_size, vcpu_align,
4376 SLAB_ACCOUNT,
4377 offsetof(struct kvm_vcpu, arch),
4378 sizeof_field(struct kvm_vcpu, arch),
4379 NULL);
Rusty Russellc16f8622007-07-30 21:12:19 +10004380 if (!kvm_vcpu_cache) {
4381 r = -ENOMEM;
Rafael J. Wysockifb3600c2011-03-23 22:16:23 +01004382 goto out_free_3;
Rusty Russellc16f8622007-07-30 21:12:19 +10004383 }
4384
Gleb Natapovaf585b92010-10-14 11:22:46 +02004385 r = kvm_async_pf_init();
4386 if (r)
4387 goto out_free;
4388
Avi Kivity6aa8b732006-12-10 02:21:36 -08004389 kvm_chardev_ops.owner = module;
Christian Borntraeger3d3aab12008-12-02 11:17:32 +01004390 kvm_vm_fops.owner = module;
4391 kvm_vcpu_fops.owner = module;
Avi Kivity6aa8b732006-12-10 02:21:36 -08004392
4393 r = misc_register(&kvm_dev);
4394 if (r) {
Xiubo Li1170adc2015-02-26 14:58:26 +08004395 pr_err("kvm: misc device register failed\n");
Gleb Natapovaf585b92010-10-14 11:22:46 +02004396 goto out_unreg;
Avi Kivity6aa8b732006-12-10 02:21:36 -08004397 }
4398
Rafael J. Wysockifb3600c2011-03-23 22:16:23 +01004399 register_syscore_ops(&kvm_syscore_ops);
4400
Avi Kivity15ad7142007-07-11 18:17:21 +03004401 kvm_preempt_ops.sched_in = kvm_sched_in;
4402 kvm_preempt_ops.sched_out = kvm_sched_out;
4403
Greg Kroah-Hartman929f45e2018-05-29 18:22:04 +02004404 kvm_init_debug();
Darrick J. Wong0ea4ed82009-10-14 16:21:00 -07004405
Paolo Bonzini3c3c29f2014-09-24 13:02:46 +02004406 r = kvm_vfio_ops_init();
4407 WARN_ON(r);
4408
Avi Kivityc7addb92007-09-16 18:58:32 +02004409 return 0;
Avi Kivity6aa8b732006-12-10 02:21:36 -08004410
Gleb Natapovaf585b92010-10-14 11:22:46 +02004411out_unreg:
4412 kvm_async_pf_deinit();
Avi Kivity6aa8b732006-12-10 02:21:36 -08004413out_free:
Rusty Russellc16f8622007-07-30 21:12:19 +10004414 kmem_cache_destroy(kvm_vcpu_cache);
Zhang Xiantaod23087842007-11-29 15:35:39 +08004415out_free_3:
Avi Kivity6aa8b732006-12-10 02:21:36 -08004416 unregister_reboot_notifier(&kvm_reboot_notifier);
Thomas Gleixner8c18b2d2016-07-13 17:16:37 +00004417 cpuhp_remove_state_nocalls(CPUHP_AP_KVM_STARTING);
Zhang Xiantaod23087842007-11-29 15:35:39 +08004418out_free_2:
Zhang Xiantaoe9b11c12007-11-14 20:38:21 +08004419 kvm_arch_hardware_unsetup();
Miaohe Linfaf0be22019-11-23 10:45:50 +08004420out_free_1:
Rusty Russell7f59f492008-12-07 21:25:45 +10304421 free_cpumask_var(cpus_hardware_enabled);
Zhang Xiantaod23087842007-11-29 15:35:39 +08004422out_free_0:
Cornelia Hucka0f155e2013-02-28 12:33:18 +01004423 kvm_irqfd_exit();
Paolo Bonzini36343f62016-10-26 13:35:56 +02004424out_irqfd:
Asias He7dac16c2013-05-08 10:57:29 +08004425 kvm_arch_exit();
4426out_fail:
Avi Kivity6aa8b732006-12-10 02:21:36 -08004427 return r;
4428}
Zhang Xiantaocb498ea2007-11-14 20:39:31 +08004429EXPORT_SYMBOL_GPL(kvm_init);
Avi Kivity6aa8b732006-12-10 02:21:36 -08004430
Zhang Xiantaocb498ea2007-11-14 20:39:31 +08004431void kvm_exit(void)
Avi Kivity6aa8b732006-12-10 02:21:36 -08004432{
Janosch Frank4bd33b52015-10-14 12:37:35 +02004433 debugfs_remove_recursive(kvm_debugfs_dir);
Avi Kivity6aa8b732006-12-10 02:21:36 -08004434 misc_deregister(&kvm_dev);
Rusty Russellc16f8622007-07-30 21:12:19 +10004435 kmem_cache_destroy(kvm_vcpu_cache);
Gleb Natapovaf585b92010-10-14 11:22:46 +02004436 kvm_async_pf_deinit();
Rafael J. Wysockifb3600c2011-03-23 22:16:23 +01004437 unregister_syscore_ops(&kvm_syscore_ops);
Avi Kivity6aa8b732006-12-10 02:21:36 -08004438 unregister_reboot_notifier(&kvm_reboot_notifier);
Thomas Gleixner8c18b2d2016-07-13 17:16:37 +00004439 cpuhp_remove_state_nocalls(CPUHP_AP_KVM_STARTING);
Takuya Yoshikawa75b71272010-11-16 17:37:41 +09004440 on_each_cpu(hardware_disable_nolock, NULL, 1);
Zhang Xiantaoe9b11c12007-11-14 20:38:21 +08004441 kvm_arch_hardware_unsetup();
Zhang Xiantaof8c16bb2007-11-14 20:40:21 +08004442 kvm_arch_exit();
Cornelia Hucka0f155e2013-02-28 12:33:18 +01004443 kvm_irqfd_exit();
Rusty Russell7f59f492008-12-07 21:25:45 +10304444 free_cpumask_var(cpus_hardware_enabled);
Wanpeng Li571ee1b2014-10-09 18:30:08 +08004445 kvm_vfio_ops_exit();
Avi Kivity6aa8b732006-12-10 02:21:36 -08004446}
Zhang Xiantaocb498ea2007-11-14 20:39:31 +08004447EXPORT_SYMBOL_GPL(kvm_exit);
Junaid Shahidc57c8042019-11-04 12:22:02 +01004448
4449struct kvm_vm_worker_thread_context {
4450 struct kvm *kvm;
4451 struct task_struct *parent;
4452 struct completion init_done;
4453 kvm_vm_thread_fn_t thread_fn;
4454 uintptr_t data;
4455 int err;
4456};
4457
4458static int kvm_vm_worker_thread(void *context)
4459{
4460 /*
4461 * The init_context is allocated on the stack of the parent thread, so
4462 * we have to locally copy anything that is needed beyond initialization
4463 */
4464 struct kvm_vm_worker_thread_context *init_context = context;
4465 struct kvm *kvm = init_context->kvm;
4466 kvm_vm_thread_fn_t thread_fn = init_context->thread_fn;
4467 uintptr_t data = init_context->data;
4468 int err;
4469
4470 err = kthread_park(current);
4471 /* kthread_park(current) is never supposed to return an error */
4472 WARN_ON(err != 0);
4473 if (err)
4474 goto init_complete;
4475
4476 err = cgroup_attach_task_all(init_context->parent, current);
4477 if (err) {
4478 kvm_err("%s: cgroup_attach_task_all failed with err %d\n",
4479 __func__, err);
4480 goto init_complete;
4481 }
4482
4483 set_user_nice(current, task_nice(init_context->parent));
4484
4485init_complete:
4486 init_context->err = err;
4487 complete(&init_context->init_done);
4488 init_context = NULL;
4489
4490 if (err)
4491 return err;
4492
4493 /* Wait to be woken up by the spawner before proceeding. */
4494 kthread_parkme();
4495
4496 if (!kthread_should_stop())
4497 err = thread_fn(kvm, data);
4498
4499 return err;
4500}
4501
4502int kvm_vm_create_worker_thread(struct kvm *kvm, kvm_vm_thread_fn_t thread_fn,
4503 uintptr_t data, const char *name,
4504 struct task_struct **thread_ptr)
4505{
4506 struct kvm_vm_worker_thread_context init_context = {};
4507 struct task_struct *thread;
4508
4509 *thread_ptr = NULL;
4510 init_context.kvm = kvm;
4511 init_context.parent = current;
4512 init_context.thread_fn = thread_fn;
4513 init_context.data = data;
4514 init_completion(&init_context.init_done);
4515
4516 thread = kthread_run(kvm_vm_worker_thread, &init_context,
4517 "%s-%d", name, task_pid_nr(current));
4518 if (IS_ERR(thread))
4519 return PTR_ERR(thread);
4520
4521 /* kthread_run is never supposed to return NULL */
4522 WARN_ON(thread == NULL);
4523
4524 wait_for_completion(&init_context.init_done);
4525
4526 if (!init_context.err)
4527 *thread_ptr = thread;
4528
4529 return init_context.err;
4530}