blob: 27ac833e5ec7c2f7fd3c58ab2bde385022f8b629 [file] [log] [blame]
Thomas Gleixnercaab2772019-06-03 07:44:50 +02001// SPDX-License-Identifier: GPL-2.0-only
Marc Zyngier74fe55d2017-10-27 15:28:38 +01002/*
3 * Copyright (C) 2017 ARM Ltd.
4 * Author: Marc Zyngier <marc.zyngier@arm.com>
Marc Zyngier74fe55d2017-10-27 15:28:38 +01005 */
6
7#include <linux/interrupt.h>
Marc Zyngierbdb2d2c2017-10-27 15:28:48 +01008#include <linux/irq.h>
Marc Zyngier74fe55d2017-10-27 15:28:38 +01009#include <linux/irqdomain.h>
10#include <linux/kvm_host.h>
Marc Zyngier196b1362017-10-27 15:28:39 +010011#include <linux/irqchip/arm-gic-v3.h>
Marc Zyngier74fe55d2017-10-27 15:28:38 +010012
13#include "vgic.h"
14
Marc Zyngiered8703a2017-10-27 15:28:55 +010015/*
16 * How KVM uses GICv4 (insert rude comments here):
17 *
18 * The vgic-v4 layer acts as a bridge between several entities:
19 * - The GICv4 ITS representation offered by the ITS driver
20 * - VFIO, which is in charge of the PCI endpoint
21 * - The virtual ITS, which is the only thing the guest sees
22 *
23 * The configuration of VLPIs is triggered by a callback from VFIO,
24 * instructing KVM that a PCI device has been configured to deliver
25 * MSIs to a vITS.
26 *
27 * kvm_vgic_v4_set_forwarding() is thus called with the routing entry,
28 * and this is used to find the corresponding vITS data structures
29 * (ITS instance, device, event and irq) using a process that is
30 * extremely similar to the injection of an MSI.
31 *
32 * At this stage, we can link the guest's view of an LPI (uniquely
33 * identified by the routing entry) and the host irq, using the GICv4
34 * driver mapping operation. Should the mapping succeed, we've then
35 * successfully upgraded the guest's LPI to a VLPI. We can then start
36 * with updating GICv4's view of the property table and generating an
37 * INValidation in order to kickstart the delivery of this VLPI to the
38 * guest directly, without software intervention. Well, almost.
39 *
40 * When the PCI endpoint is deconfigured, this operation is reversed
41 * with VFIO calling kvm_vgic_v4_unset_forwarding().
42 *
43 * Once the VLPI has been mapped, it needs to follow any change the
44 * guest performs on its LPI through the vITS. For that, a number of
45 * command handlers have hooks to communicate these changes to the HW:
46 * - Any invalidation triggers a call to its_prop_update_vlpi()
47 * - The INT command results in a irq_set_irqchip_state(), which
48 * generates an INT on the corresponding VLPI.
49 * - The CLEAR command results in a irq_set_irqchip_state(), which
50 * generates an CLEAR on the corresponding VLPI.
51 * - DISCARD translates into an unmap, similar to a call to
52 * kvm_vgic_v4_unset_forwarding().
53 * - MOVI is translated by an update of the existing mapping, changing
54 * the target vcpu, resulting in a VMOVI being generated.
55 * - MOVALL is translated by a string of mapping updates (similar to
56 * the handling of MOVI). MOVALL is horrible.
57 *
58 * Note that a DISCARD/MAPTI sequence emitted from the guest without
59 * reprogramming the PCI endpoint after MAPTI does not result in a
60 * VLPI being mapped, as there is no callback from VFIO (the guest
61 * will get the interrupt via the normal SW injection). Fixing this is
62 * not trivial, and requires some horrible messing with the VFIO
63 * internals. Not fun. Don't do that.
64 *
65 * Then there is the scheduling. Each time a vcpu is about to run on a
66 * physical CPU, KVM must tell the corresponding redistributor about
67 * it. And if we've migrated our vcpu from one CPU to another, we must
68 * tell the ITS (so that the messages reach the right redistributor).
69 * This is done in two steps: first issue a irq_set_affinity() on the
Marc Zyngierae699ad2020-03-04 20:33:20 +000070 * irq corresponding to the vcpu, then call its_make_vpe_resident().
71 * You must be in a non-preemptible context. On exit, a call to
72 * its_make_vpe_non_resident() tells the redistributor that we're done
73 * with the vcpu.
Marc Zyngiered8703a2017-10-27 15:28:55 +010074 *
75 * Finally, the doorbell handling: Each vcpu is allocated an interrupt
76 * which will fire each time a VLPI is made pending whilst the vcpu is
77 * not running. Each time the vcpu gets blocked, the doorbell
78 * interrupt gets enabled. When the vcpu is unblocked (for whatever
79 * reason), the doorbell interrupt is disabled.
80 */
81
Marc Zyngierd3d83f72017-10-27 15:28:53 +010082#define DB_IRQ_FLAGS (IRQ_NOAUTOEN | IRQ_DISABLE_UNLAZY | IRQ_NO_BALANCING)
83
Marc Zyngierbdb2d2c2017-10-27 15:28:48 +010084static irqreturn_t vgic_v4_doorbell_handler(int irq, void *info)
85{
86 struct kvm_vcpu *vcpu = info;
87
Marc Zyngier8e01d9a2019-10-27 14:41:59 +000088 /* We got the message, no need to fire again */
Marc Zyngierae699ad2020-03-04 20:33:20 +000089 if (!kvm_vgic_global_state.has_gicv4_1 &&
90 !irqd_irq_disabled(&irq_to_desc(irq)->irq_data))
Marc Zyngier8e01d9a2019-10-27 14:41:59 +000091 disable_irq_nosync(irq);
92
Marc Zyngierbdb2d2c2017-10-27 15:28:48 +010093 vcpu->arch.vgic_cpu.vgic_v3.its_vpe.pending_last = true;
94 kvm_make_request(KVM_REQ_IRQ_PENDING, vcpu);
95 kvm_vcpu_kick(vcpu);
96
97 return IRQ_HANDLED;
98}
99
Marc Zyngierbacf2c62020-03-04 20:33:26 +0000100static void vgic_v4_sync_sgi_config(struct its_vpe *vpe, struct vgic_irq *irq)
101{
102 vpe->sgi_config[irq->intid].enabled = irq->enabled;
103 vpe->sgi_config[irq->intid].group = irq->group;
104 vpe->sgi_config[irq->intid].priority = irq->priority;
105}
106
107static void vgic_v4_enable_vsgis(struct kvm_vcpu *vcpu)
108{
109 struct its_vpe *vpe = &vcpu->arch.vgic_cpu.vgic_v3.its_vpe;
110 int i;
111
112 /*
113 * With GICv4.1, every virtual SGI can be directly injected. So
114 * let's pretend that they are HW interrupts, tied to a host
115 * IRQ. The SGI code will do its magic.
116 */
117 for (i = 0; i < VGIC_NR_SGIS; i++) {
118 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, i);
119 struct irq_desc *desc;
120 unsigned long flags;
121 int ret;
122
123 raw_spin_lock_irqsave(&irq->irq_lock, flags);
124
125 if (irq->hw)
126 goto unlock;
127
128 irq->hw = true;
129 irq->host_irq = irq_find_mapping(vpe->sgi_domain, i);
130
131 /* Transfer the full irq state to the vPE */
132 vgic_v4_sync_sgi_config(vpe, irq);
133 desc = irq_to_desc(irq->host_irq);
134 ret = irq_domain_activate_irq(irq_desc_get_irq_data(desc),
135 false);
136 if (!WARN_ON(ret)) {
137 /* Transfer pending state */
138 ret = irq_set_irqchip_state(irq->host_irq,
139 IRQCHIP_STATE_PENDING,
140 irq->pending_latch);
141 WARN_ON(ret);
142 irq->pending_latch = false;
143 }
144 unlock:
145 raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
146 vgic_put_irq(vcpu->kvm, irq);
147 }
148}
149
150static void vgic_v4_disable_vsgis(struct kvm_vcpu *vcpu)
151{
152 int i;
153
154 for (i = 0; i < VGIC_NR_SGIS; i++) {
155 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, i);
156 struct irq_desc *desc;
157 unsigned long flags;
158 int ret;
159
160 raw_spin_lock_irqsave(&irq->irq_lock, flags);
161
162 if (!irq->hw)
163 goto unlock;
164
165 irq->hw = false;
166 ret = irq_get_irqchip_state(irq->host_irq,
167 IRQCHIP_STATE_PENDING,
168 &irq->pending_latch);
169 WARN_ON(ret);
170
171 desc = irq_to_desc(irq->host_irq);
172 irq_domain_deactivate_irq(irq_desc_get_irq_data(desc));
173 unlock:
174 raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
175 vgic_put_irq(vcpu->kvm, irq);
176 }
177}
178
179/* Must be called with the kvm lock held */
180void vgic_v4_configure_vsgis(struct kvm *kvm)
181{
182 struct vgic_dist *dist = &kvm->arch.vgic;
183 struct kvm_vcpu *vcpu;
184 int i;
185
186 kvm_arm_halt_guest(kvm);
187
188 kvm_for_each_vcpu(i, vcpu, kvm) {
189 if (dist->nassgireq)
190 vgic_v4_enable_vsgis(vcpu);
191 else
192 vgic_v4_disable_vsgis(vcpu);
193 }
194
195 kvm_arm_resume_guest(kvm);
196}
197
Marc Zyngier74fe55d2017-10-27 15:28:38 +0100198/**
199 * vgic_v4_init - Initialize the GICv4 data structures
200 * @kvm: Pointer to the VM being initialized
201 *
202 * We may be called each time a vITS is created, or when the
203 * vgic is initialized. This relies on kvm->lock to be
204 * held. In both cases, the number of vcpus should now be
205 * fixed.
206 */
207int vgic_v4_init(struct kvm *kvm)
208{
209 struct vgic_dist *dist = &kvm->arch.vgic;
210 struct kvm_vcpu *vcpu;
211 int i, nr_vcpus, ret;
212
Christoffer Dallf8f85dc2018-01-12 11:40:21 +0100213 if (!kvm_vgic_global_state.has_gicv4)
Christoffer Dall3d1ad642017-11-10 09:16:23 +0100214 return 0; /* Nothing to see here... move along. */
215
Marc Zyngier74fe55d2017-10-27 15:28:38 +0100216 if (dist->its_vm.vpes)
217 return 0;
218
219 nr_vcpus = atomic_read(&kvm->online_vcpus);
220
Kees Cook6396bb22018-06-12 14:03:40 -0700221 dist->its_vm.vpes = kcalloc(nr_vcpus, sizeof(*dist->its_vm.vpes),
Marc Zyngier74fe55d2017-10-27 15:28:38 +0100222 GFP_KERNEL);
223 if (!dist->its_vm.vpes)
224 return -ENOMEM;
225
226 dist->its_vm.nr_vpes = nr_vcpus;
227
228 kvm_for_each_vcpu(i, vcpu, kvm)
229 dist->its_vm.vpes[i] = &vcpu->arch.vgic_cpu.vgic_v3.its_vpe;
230
231 ret = its_alloc_vcpu_irqs(&dist->its_vm);
232 if (ret < 0) {
233 kvm_err("VPE IRQ allocation failure\n");
234 kfree(dist->its_vm.vpes);
235 dist->its_vm.nr_vpes = 0;
236 dist->its_vm.vpes = NULL;
237 return ret;
238 }
239
Marc Zyngierbdb2d2c2017-10-27 15:28:48 +0100240 kvm_for_each_vcpu(i, vcpu, kvm) {
241 int irq = dist->its_vm.vpes[i]->irq;
Marc Zyngier9879b79a2020-03-04 20:33:24 +0000242 unsigned long irq_flags = DB_IRQ_FLAGS;
Marc Zyngierbdb2d2c2017-10-27 15:28:48 +0100243
244 /*
245 * Don't automatically enable the doorbell, as we're
246 * flipping it back and forth when the vcpu gets
247 * blocked. Also disable the lazy disabling, as the
248 * doorbell could kick us out of the guest too
249 * early...
Marc Zyngier9879b79a2020-03-04 20:33:24 +0000250 *
251 * On GICv4.1, the doorbell is managed in HW and must
252 * be left enabled.
Marc Zyngierbdb2d2c2017-10-27 15:28:48 +0100253 */
Marc Zyngier9879b79a2020-03-04 20:33:24 +0000254 if (kvm_vgic_global_state.has_gicv4_1)
255 irq_flags &= ~IRQ_NOAUTOEN;
256 irq_set_status_flags(irq, irq_flags);
257
Marc Zyngierbdb2d2c2017-10-27 15:28:48 +0100258 ret = request_irq(irq, vgic_v4_doorbell_handler,
259 0, "vcpu", vcpu);
260 if (ret) {
261 kvm_err("failed to allocate vcpu IRQ%d\n", irq);
262 /*
263 * Trick: adjust the number of vpes so we know
264 * how many to nuke on teardown...
265 */
266 dist->its_vm.nr_vpes = i;
267 break;
268 }
269 }
270
271 if (ret)
272 vgic_v4_teardown(kvm);
273
Marc Zyngier74fe55d2017-10-27 15:28:38 +0100274 return ret;
275}
276
277/**
278 * vgic_v4_teardown - Free the GICv4 data structures
279 * @kvm: Pointer to the VM being destroyed
280 *
281 * Relies on kvm->lock to be held.
282 */
283void vgic_v4_teardown(struct kvm *kvm)
284{
285 struct its_vm *its_vm = &kvm->arch.vgic.its_vm;
Marc Zyngierbdb2d2c2017-10-27 15:28:48 +0100286 int i;
Marc Zyngier74fe55d2017-10-27 15:28:38 +0100287
288 if (!its_vm->vpes)
289 return;
290
Marc Zyngierbdb2d2c2017-10-27 15:28:48 +0100291 for (i = 0; i < its_vm->nr_vpes; i++) {
292 struct kvm_vcpu *vcpu = kvm_get_vcpu(kvm, i);
293 int irq = its_vm->vpes[i]->irq;
294
Marc Zyngierd3d83f72017-10-27 15:28:53 +0100295 irq_clear_status_flags(irq, DB_IRQ_FLAGS);
Marc Zyngierbdb2d2c2017-10-27 15:28:48 +0100296 free_irq(irq, vcpu);
297 }
298
Marc Zyngier74fe55d2017-10-27 15:28:38 +0100299 its_free_vcpu_irqs(its_vm);
300 kfree(its_vm->vpes);
301 its_vm->nr_vpes = 0;
302 its_vm->vpes = NULL;
303}
Marc Zyngier196b1362017-10-27 15:28:39 +0100304
Marc Zyngier8e01d9a2019-10-27 14:41:59 +0000305int vgic_v4_put(struct kvm_vcpu *vcpu, bool need_db)
Marc Zyngier62775792017-10-27 15:28:50 +0100306{
Marc Zyngier8e01d9a2019-10-27 14:41:59 +0000307 struct its_vpe *vpe = &vcpu->arch.vgic_cpu.vgic_v3.its_vpe;
Marc Zyngier8e01d9a2019-10-27 14:41:59 +0000308
309 if (!vgic_supports_direct_msis(vcpu->kvm) || !vpe->resident)
Marc Zyngier62775792017-10-27 15:28:50 +0100310 return 0;
311
Marc Zyngierae699ad2020-03-04 20:33:20 +0000312 return its_make_vpe_non_resident(vpe, need_db);
Marc Zyngier62775792017-10-27 15:28:50 +0100313}
314
Marc Zyngier8e01d9a2019-10-27 14:41:59 +0000315int vgic_v4_load(struct kvm_vcpu *vcpu)
Marc Zyngier62775792017-10-27 15:28:50 +0100316{
Marc Zyngier8e01d9a2019-10-27 14:41:59 +0000317 struct its_vpe *vpe = &vcpu->arch.vgic_cpu.vgic_v3.its_vpe;
Marc Zyngier62775792017-10-27 15:28:50 +0100318 int err;
319
Marc Zyngier8e01d9a2019-10-27 14:41:59 +0000320 if (!vgic_supports_direct_msis(vcpu->kvm) || vpe->resident)
Marc Zyngier62775792017-10-27 15:28:50 +0100321 return 0;
322
323 /*
324 * Before making the VPE resident, make sure the redistributor
325 * corresponding to our current CPU expects us here. See the
326 * doc in drivers/irqchip/irq-gic-v4.c to understand how this
327 * turns into a VMOVP command at the ITS level.
328 */
Marc Zyngier8e01d9a2019-10-27 14:41:59 +0000329 err = irq_set_affinity(vpe->irq, cpumask_of(smp_processor_id()));
Marc Zyngier62775792017-10-27 15:28:50 +0100330 if (err)
331 return err;
332
Marc Zyngierae699ad2020-03-04 20:33:20 +0000333 err = its_make_vpe_resident(vpe, false, vcpu->kvm->arch.vgic.enabled);
Marc Zyngier62775792017-10-27 15:28:50 +0100334 if (err)
335 return err;
336
337 /*
338 * Now that the VPE is resident, let's get rid of a potential
Marc Zyngierae699ad2020-03-04 20:33:20 +0000339 * doorbell interrupt that would still be pending. This is a
340 * GICv4.0 only "feature"...
Marc Zyngier62775792017-10-27 15:28:50 +0100341 */
Marc Zyngierae699ad2020-03-04 20:33:20 +0000342 if (!kvm_vgic_global_state.has_gicv4_1)
343 err = irq_set_irqchip_state(vpe->irq, IRQCHIP_STATE_PENDING, false);
344
345 return err;
Marc Zyngier62775792017-10-27 15:28:50 +0100346}
347
Marc Zyngier196b1362017-10-27 15:28:39 +0100348static struct vgic_its *vgic_get_its(struct kvm *kvm,
349 struct kvm_kernel_irq_routing_entry *irq_entry)
350{
351 struct kvm_msi msi = (struct kvm_msi) {
352 .address_lo = irq_entry->msi.address_lo,
353 .address_hi = irq_entry->msi.address_hi,
354 .data = irq_entry->msi.data,
355 .flags = irq_entry->msi.flags,
356 .devid = irq_entry->msi.devid,
357 };
358
359 return vgic_msi_to_its(kvm, &msi);
360}
361
362int kvm_vgic_v4_set_forwarding(struct kvm *kvm, int virq,
363 struct kvm_kernel_irq_routing_entry *irq_entry)
364{
365 struct vgic_its *its;
366 struct vgic_irq *irq;
367 struct its_vlpi_map map;
368 int ret;
369
370 if (!vgic_supports_direct_msis(kvm))
371 return 0;
372
373 /*
374 * Get the ITS, and escape early on error (not a valid
375 * doorbell for any of our vITSs).
376 */
377 its = vgic_get_its(kvm, irq_entry);
378 if (IS_ERR(its))
379 return 0;
380
381 mutex_lock(&its->its_lock);
382
Zenghui Yubad36e42019-10-29 15:19:18 +0800383 /* Perform the actual DevID/EventID -> LPI translation. */
Marc Zyngier196b1362017-10-27 15:28:39 +0100384 ret = vgic_its_resolve_lpi(kvm, its, irq_entry->msi.devid,
385 irq_entry->msi.data, &irq);
386 if (ret)
387 goto out;
388
389 /*
390 * Emit the mapping request. If it fails, the ITS probably
391 * isn't v4 compatible, so let's silently bail out. Holding
392 * the ITS lock should ensure that nothing can modify the
393 * target vcpu.
394 */
395 map = (struct its_vlpi_map) {
396 .vm = &kvm->arch.vgic.its_vm,
397 .vpe = &irq->target_vcpu->arch.vgic_cpu.vgic_v3.its_vpe,
398 .vintid = irq->intid,
399 .properties = ((irq->priority & 0xfc) |
400 (irq->enabled ? LPI_PROP_ENABLED : 0) |
401 LPI_PROP_GROUP1),
402 .db_enabled = true,
403 };
404
405 ret = its_map_vlpi(virq, &map);
406 if (ret)
407 goto out;
408
409 irq->hw = true;
410 irq->host_irq = virq;
Marc Zyngier5bd90b02019-11-07 16:04:11 +0000411 atomic_inc(&map.vpe->vlpi_count);
Marc Zyngier196b1362017-10-27 15:28:39 +0100412
413out:
414 mutex_unlock(&its->its_lock);
415 return ret;
416}
417
418int kvm_vgic_v4_unset_forwarding(struct kvm *kvm, int virq,
419 struct kvm_kernel_irq_routing_entry *irq_entry)
420{
421 struct vgic_its *its;
422 struct vgic_irq *irq;
423 int ret;
424
425 if (!vgic_supports_direct_msis(kvm))
426 return 0;
427
428 /*
429 * Get the ITS, and escape early on error (not a valid
430 * doorbell for any of our vITSs).
431 */
432 its = vgic_get_its(kvm, irq_entry);
433 if (IS_ERR(its))
434 return 0;
435
436 mutex_lock(&its->its_lock);
437
438 ret = vgic_its_resolve_lpi(kvm, its, irq_entry->msi.devid,
439 irq_entry->msi.data, &irq);
440 if (ret)
441 goto out;
442
443 WARN_ON(!(irq->hw && irq->host_irq == virq));
Marc Zyngiera05d1c02017-11-16 17:58:19 +0000444 if (irq->hw) {
Marc Zyngier5bd90b02019-11-07 16:04:11 +0000445 atomic_dec(&irq->target_vcpu->arch.vgic_cpu.vgic_v3.its_vpe.vlpi_count);
Marc Zyngiera05d1c02017-11-16 17:58:19 +0000446 irq->hw = false;
447 ret = its_unmap_vlpi(virq);
448 }
Marc Zyngier196b1362017-10-27 15:28:39 +0100449
450out:
451 mutex_unlock(&its->its_lock);
452 return ret;
453}