blob: fdec478ba5e70a772daee24c332d5bb1167fda00 [file] [log] [blame]
Thomas Gleixnercaab2772019-06-03 07:44:50 +02001// SPDX-License-Identifier: GPL-2.0-only
Marc Zyngiercc2d3212014-11-24 14:35:11 +00002/*
Marc Zyngierd7276b82016-12-20 15:11:47 +00003 * Copyright (C) 2013-2017 ARM Limited, All Rights Reserved.
Marc Zyngiercc2d3212014-11-24 14:35:11 +00004 * Author: Marc Zyngier <marc.zyngier@arm.com>
Marc Zyngiercc2d3212014-11-24 14:35:11 +00005 */
6
Tomasz Nowicki3f010cf2016-09-12 20:32:25 +02007#include <linux/acpi.h>
Hanjun Guo8d3554b2017-03-07 20:39:59 +08008#include <linux/acpi_iort.h>
Marc Zyngierffedbf02019-11-08 16:57:59 +00009#include <linux/bitfield.h>
Marc Zyngiercc2d3212014-11-24 14:35:11 +000010#include <linux/bitmap.h>
11#include <linux/cpu.h>
Marc Zyngierc6e2ccb2018-06-26 11:21:11 +010012#include <linux/crash_dump.h>
Marc Zyngiercc2d3212014-11-24 14:35:11 +000013#include <linux/delay.h>
Marc Zyngier3fb68fa2018-07-27 16:21:18 +010014#include <linux/efi.h>
Marc Zyngiercc2d3212014-11-24 14:35:11 +000015#include <linux/interrupt.h>
Robin Murphyfa493642022-08-16 18:28:04 +010016#include <linux/iommu.h>
Marc Zyngier96806222020-04-10 11:13:26 +010017#include <linux/iopoll.h>
Tomasz Nowicki3f010cf2016-09-12 20:32:25 +020018#include <linux/irqdomain.h>
Marc Zyngier880cb3c2018-05-27 16:14:15 +010019#include <linux/list.h>
Marc Zyngiercc2d3212014-11-24 14:35:11 +000020#include <linux/log2.h>
Marc Zyngier5e2c9f92018-07-27 16:23:18 +010021#include <linux/memblock.h>
Marc Zyngiercc2d3212014-11-24 14:35:11 +000022#include <linux/mm.h>
23#include <linux/msi.h>
24#include <linux/of.h>
25#include <linux/of_address.h>
26#include <linux/of_irq.h>
27#include <linux/of_pci.h>
28#include <linux/of_platform.h>
29#include <linux/percpu.h>
30#include <linux/slab.h>
Derek Basehoredba0bc72018-02-28 21:48:18 -080031#include <linux/syscore_ops.h>
Marc Zyngiercc2d3212014-11-24 14:35:11 +000032
Joel Porquet41a83e062015-07-07 17:11:46 -040033#include <linux/irqchip.h>
Marc Zyngiercc2d3212014-11-24 14:35:11 +000034#include <linux/irqchip/arm-gic-v3.h>
Marc Zyngierc808eea2016-12-20 09:31:20 +000035#include <linux/irqchip/arm-gic-v4.h>
Marc Zyngiercc2d3212014-11-24 14:35:11 +000036
Marc Zyngiercc2d3212014-11-24 14:35:11 +000037#include <asm/cputype.h>
38#include <asm/exception.h>
39
Robert Richter67510cc2015-09-21 22:58:37 +020040#include "irq-gic-common.h"
Thomas Gleixner48f71d52024-06-23 17:18:36 +020041#include "irq-msi-lib.h"
Robert Richter67510cc2015-09-21 22:58:37 +020042
Robert Richter94100972015-09-21 22:58:38 +020043#define ITS_FLAGS_CMDQ_NEEDS_FLUSHING (1ULL << 0)
44#define ITS_FLAGS_WORKAROUND_CAVIUM_22375 (1ULL << 1)
Ganapatrao Kulkarnifbf8f402016-05-25 15:29:20 +020045#define ITS_FLAGS_WORKAROUND_CAVIUM_23144 (1ULL << 2)
Sebastian Reichela8707f52023-04-18 16:21:08 +020046#define ITS_FLAGS_FORCE_NON_SHAREABLE (1ULL << 3)
Marc Zyngiercc2d3212014-11-24 14:35:11 +000047
Valentin Schneiderc0cdc8902021-10-27 16:15:04 +010048#define RD_LOCAL_LPI_ENABLED BIT(0)
Valentin Schneiderd23bc2b2021-10-27 16:15:05 +010049#define RD_LOCAL_PENDTABLE_PREALLOCATED BIT(1)
50#define RD_LOCAL_MEMRESERVE_DONE BIT(2)
Valentin Schneiderc0cdc8902021-10-27 16:15:04 +010051
Marc Zyngiera13b0402016-12-19 17:15:24 +000052static u32 lpi_id_bits;
53
54/*
55 * We allocate memory for PROPBASE to cover 2 ^ lpi_id_bits LPIs to
56 * deal with (one configuration byte per interrupt). PENDBASE has to
57 * be 64kB aligned (one bit per LPI, plus 8192 bits for SPI/PPI/SGI).
58 */
59#define LPI_NRBITS lpi_id_bits
60#define LPI_PROPBASE_SZ ALIGN(BIT(LPI_NRBITS), SZ_64K)
61#define LPI_PENDBASE_SZ ALIGN(BIT(LPI_NRBITS) / 8, SZ_64K)
62
Mark Rutlanda6156e72024-06-17 12:18:39 +010063static u8 __ro_after_init lpi_prop_prio;
Marc Zyngiera13b0402016-12-19 17:15:24 +000064
Marc Zyngiercc2d3212014-11-24 14:35:11 +000065/*
66 * Collection structure - just an ID, and a redistributor address to
67 * ping. We use one per CPU as a bag of interrupts assigned to this
68 * CPU.
69 */
70struct its_collection {
71 u64 target_address;
72 u16 col_id;
73};
74
75/*
Shanker Donthineni93473592016-06-06 18:17:30 -050076 * The ITS_BASER structure - contains memory information, cached
77 * value of BASER register configuration and ITS page size.
Shanker Donthineni466b7d12016-03-09 22:10:49 -060078 */
79struct its_baser {
80 void *base;
81 u64 val;
82 u32 order;
Shanker Donthineni93473592016-06-06 18:17:30 -050083 u32 psz;
Shanker Donthineni466b7d12016-03-09 22:10:49 -060084};
85
Ard Biesheuvel558b0162017-10-17 17:55:56 +010086struct its_device;
87
Shanker Donthineni466b7d12016-03-09 22:10:49 -060088/*
Marc Zyngiercc2d3212014-11-24 14:35:11 +000089 * The ITS structure - contains most of the infrastructure, with the
Marc Zyngier841514a2015-07-28 14:46:20 +010090 * top-level MSI domain, the command queue, the collections, and the
91 * list of devices writing to it.
Marc Zyngier9791ec72019-01-29 10:02:33 +000092 *
93 * dev_alloc_lock has to be taken for device allocations, while the
94 * spinlock must be taken to parse data structures such as the device
95 * list.
Marc Zyngiercc2d3212014-11-24 14:35:11 +000096 */
97struct its_node {
98 raw_spinlock_t lock;
Marc Zyngier9791ec72019-01-29 10:02:33 +000099 struct mutex dev_alloc_lock;
Marc Zyngiercc2d3212014-11-24 14:35:11 +0000100 struct list_head entry;
Marc Zyngiercc2d3212014-11-24 14:35:11 +0000101 void __iomem *base;
Marc Zyngier5e46a482020-03-04 20:33:14 +0000102 void __iomem *sgir_base;
Tomasz Nowickidb40f0a2016-09-12 20:32:24 +0200103 phys_addr_t phys_base;
Marc Zyngiercc2d3212014-11-24 14:35:11 +0000104 struct its_cmd_block *cmd_base;
105 struct its_cmd_block *cmd_write;
Shanker Donthineni466b7d12016-03-09 22:10:49 -0600106 struct its_baser tables[GITS_BASER_NR_REGS];
Marc Zyngiercc2d3212014-11-24 14:35:11 +0000107 struct its_collection *collections;
Ard Biesheuvel558b0162017-10-17 17:55:56 +0100108 struct fwnode_handle *fwnode_handle;
109 u64 (*get_msi_base)(struct its_device *its_dev);
Marc Zyngier0dd57fe2019-11-08 16:57:58 +0000110 u64 typer;
Derek Basehoredba0bc72018-02-28 21:48:18 -0800111 u64 cbaser_save;
112 u32 ctlr_save;
Marc Zyngier5e516842019-12-24 11:10:28 +0000113 u32 mpidr;
Marc Zyngiercc2d3212014-11-24 14:35:11 +0000114 struct list_head its_device_list;
115 u64 flags;
Marc Zyngierdebf6d02017-10-08 18:44:42 +0100116 unsigned long list_nr;
Ganapatrao Kulkarnifbf8f402016-05-25 15:29:20 +0200117 int numa_node;
Ard Biesheuvel558b0162017-10-17 17:55:56 +0100118 unsigned int msi_domain_flags;
119 u32 pre_its_base; /* for Socionext Synquacer */
Marc Zyngier5c9a8822017-07-28 21:20:37 +0100120 int vlpi_redist_offset;
Marc Zyngiercc2d3212014-11-24 14:35:11 +0000121};
122
Marc Zyngier0dd57fe2019-11-08 16:57:58 +0000123#define is_v4(its) (!!((its)->typer & GITS_TYPER_VLPIS))
Marc Zyngier5e516842019-12-24 11:10:28 +0000124#define is_v4_1(its) (!!((its)->typer & GITS_TYPER_VMAPP))
Marc Zyngier576a8342019-11-08 16:58:00 +0000125#define device_ids(its) (FIELD_GET(GITS_TYPER_DEVBITS, (its)->typer) + 1)
Marc Zyngier0dd57fe2019-11-08 16:57:58 +0000126
Marc Zyngiercc2d3212014-11-24 14:35:11 +0000127#define ITS_ITT_ALIGN SZ_256
128
Shanker Donthineni32bd44d2017-10-07 15:43:48 -0500129/* The maximum number of VPEID bits supported by VLPI commands */
Marc Zyngierf2d83402019-12-24 11:10:25 +0000130#define ITS_MAX_VPEID_BITS \
131 ({ \
132 int nvpeid = 16; \
133 if (gic_rdists->has_rvpeid && \
134 gic_rdists->gicd_typer2 & GICD_TYPER2_VIL) \
135 nvpeid = 1 + (gic_rdists->gicd_typer2 & \
136 GICD_TYPER2_VID); \
137 \
138 nvpeid; \
139 })
Shanker Donthineni32bd44d2017-10-07 15:43:48 -0500140#define ITS_MAX_VPEID (1 << (ITS_MAX_VPEID_BITS))
141
Shanker Donthineni2eca0d62016-02-16 18:00:36 -0600142/* Convert page order to size in bytes */
143#define PAGE_ORDER_TO_SIZE(o) (PAGE_SIZE << (o))
144
Marc Zyngier591e5be2015-07-17 10:46:42 +0100145struct event_lpi_map {
146 unsigned long *lpi_map;
147 u16 *col_map;
148 irq_hw_number_t lpi_base;
149 int nr_lpis;
Marc Zyngier11635fa2019-11-08 16:58:05 +0000150 raw_spinlock_t vlpi_lock;
Marc Zyngierd011e4e2016-12-20 09:44:41 +0000151 struct its_vm *vm;
152 struct its_vlpi_map *vlpi_maps;
153 int nr_vlpis;
Marc Zyngier591e5be2015-07-17 10:46:42 +0100154};
155
Marc Zyngiercc2d3212014-11-24 14:35:11 +0000156/*
Marc Zyngierd011e4e2016-12-20 09:44:41 +0000157 * The ITS view of a device - belongs to an ITS, owns an interrupt
158 * translation table, and a list of interrupts. If it some of its
159 * LPIs are injected into a guest (GICv4), the event_map.vm field
160 * indicates which one.
Marc Zyngiercc2d3212014-11-24 14:35:11 +0000161 */
162struct its_device {
163 struct list_head entry;
164 struct its_node *its;
Marc Zyngier591e5be2015-07-17 10:46:42 +0100165 struct event_lpi_map event_map;
Marc Zyngiercc2d3212014-11-24 14:35:11 +0000166 void *itt;
Marc Zyngiercc2d3212014-11-24 14:35:11 +0000167 u32 nr_ites;
168 u32 device_id;
Marc Zyngier9791ec72019-01-29 10:02:33 +0000169 bool shared;
Marc Zyngiercc2d3212014-11-24 14:35:11 +0000170};
171
Marc Zyngier20b3d542016-12-20 15:23:22 +0000172static struct {
173 raw_spinlock_t lock;
174 struct its_device *dev;
175 struct its_vpe **vpes;
176 int next_victim;
177} vpe_proxy;
178
Marc Zyngier2f13ff12020-05-15 17:57:51 +0100179struct cpu_lpi_count {
180 atomic_t managed;
181 atomic_t unmanaged;
182};
183
184static DEFINE_PER_CPU(struct cpu_lpi_count, cpu_lpi_count);
185
Marc Zyngier1ac19ca2014-11-24 14:35:14 +0000186static LIST_HEAD(its_nodes);
Sebastian Andrzej Siewiora8db7452018-07-18 17:42:04 +0200187static DEFINE_RAW_SPINLOCK(its_lock);
Marc Zyngier1ac19ca2014-11-24 14:35:14 +0000188static struct rdists *gic_rdists;
Tomasz Nowickidb40f0a2016-09-12 20:32:24 +0200189static struct irq_domain *its_parent;
Marc Zyngier1ac19ca2014-11-24 14:35:14 +0000190
Marc Zyngier3dfa5762016-12-19 17:25:54 +0000191static unsigned long its_list_map;
Marc Zyngier3171a472016-12-20 15:17:28 +0000192static u16 vmovp_seq_num;
193static DEFINE_RAW_SPINLOCK(vmovp_lock);
194
Marc Zyngier7d75bbb2016-12-20 13:55:54 +0000195static DEFINE_IDA(its_vpeid_ida);
Marc Zyngier3dfa5762016-12-19 17:25:54 +0000196
Marc Zyngier1ac19ca2014-11-24 14:35:14 +0000197#define gic_data_rdist() (raw_cpu_ptr(gic_rdists->rdist))
Marc Zyngier11e37d32018-07-27 13:38:54 +0100198#define gic_data_rdist_cpu(cpu) (per_cpu_ptr(gic_rdists->rdist, cpu))
Marc Zyngier1ac19ca2014-11-24 14:35:14 +0000199#define gic_data_rdist_rd_base() (gic_data_rdist()->rd_base)
Marc Zyngiere643d802016-12-20 15:09:31 +0000200#define gic_data_rdist_vlpi_base() (gic_data_rdist_rd_base() + SZ_128K)
Marc Zyngier1ac19ca2014-11-24 14:35:14 +0000201
Marc Zyngier009384b2020-03-04 20:33:23 +0000202/*
203 * Skip ITSs that have no vLPIs mapped, unless we're on GICv4.1, as we
204 * always have vSGIs mapped.
205 */
206static bool require_its_list_vmovp(struct its_vm *vm, struct its_node *its)
207{
208 return (gic_rdists->has_rvpeid || vm->vlpi_count[its->list_nr]);
209}
210
Marc Zyngier846297e2024-02-13 10:12:04 +0000211static bool rdists_support_shareable(void)
212{
213 return !(gic_rdists->flags & RDIST_FLAGS_FORCE_NON_SHAREABLE);
214}
215
Zenghui Yu84243122019-10-23 03:46:26 +0000216static u16 get_its_list(struct its_vm *vm)
217{
218 struct its_node *its;
219 unsigned long its_list = 0;
220
221 list_for_each_entry(its, &its_nodes, entry) {
Marc Zyngier0dd57fe2019-11-08 16:57:58 +0000222 if (!is_v4(its))
Zenghui Yu84243122019-10-23 03:46:26 +0000223 continue;
224
Marc Zyngier009384b2020-03-04 20:33:23 +0000225 if (require_its_list_vmovp(vm, its))
Zenghui Yu84243122019-10-23 03:46:26 +0000226 __set_bit(its->list_nr, &its_list);
227 }
228
229 return (u16)its_list;
230}
231
Marc Zyngier425c09b2019-11-08 16:57:57 +0000232static inline u32 its_get_event_id(struct irq_data *d)
233{
234 struct its_device *its_dev = irq_data_get_irq_chip_data(d);
235 return d->hwirq - its_dev->event_map.lpi_base;
236}
237
Marc Zyngier591e5be2015-07-17 10:46:42 +0100238static struct its_collection *dev_event_to_col(struct its_device *its_dev,
239 u32 event)
240{
241 struct its_node *its = its_dev->its;
242
243 return its->collections + its_dev->event_map.col_map[event];
244}
245
Marc Zyngierc1d4d5c2019-11-08 16:58:01 +0000246static struct its_vlpi_map *dev_event_to_vlpi_map(struct its_device *its_dev,
247 u32 event)
248{
249 if (WARN_ON_ONCE(event >= its_dev->event_map.nr_lpis))
250 return NULL;
251
252 return &its_dev->event_map.vlpi_maps[event];
253}
254
Marc Zyngierf4a81f52019-12-24 11:10:38 +0000255static struct its_vlpi_map *get_vlpi_map(struct irq_data *d)
256{
257 if (irqd_is_forwarded_to_vcpu(d)) {
258 struct its_device *its_dev = irq_data_get_irq_chip_data(d);
259 u32 event = its_get_event_id(d);
260
261 return dev_event_to_vlpi_map(its_dev, event);
262 }
263
264 return NULL;
265}
266
Marc Zyngierf3a059212020-03-04 20:33:10 +0000267static int vpe_to_cpuid_lock(struct its_vpe *vpe, unsigned long *flags)
Marc Zyngier425c09b2019-11-08 16:57:57 +0000268{
Marc Zyngierf3a059212020-03-04 20:33:10 +0000269 raw_spin_lock_irqsave(&vpe->vpe_lock, *flags);
270 return vpe->col_idx;
271}
272
273static void vpe_to_cpuid_unlock(struct its_vpe *vpe, unsigned long flags)
274{
275 raw_spin_unlock_irqrestore(&vpe->vpe_lock, flags);
276}
277
Marc Zyngier926846a2023-06-17 08:32:42 +0100278static struct irq_chip its_vpe_irq_chip;
279
Marc Zyngierf3a059212020-03-04 20:33:10 +0000280static int irq_to_cpuid_lock(struct irq_data *d, unsigned long *flags)
281{
Marc Zyngier926846a2023-06-17 08:32:42 +0100282 struct its_vpe *vpe = NULL;
Marc Zyngierf3a059212020-03-04 20:33:10 +0000283 int cpu;
284
Marc Zyngier926846a2023-06-17 08:32:42 +0100285 if (d->chip == &its_vpe_irq_chip) {
286 vpe = irq_data_get_irq_chip_data(d);
287 } else {
288 struct its_vlpi_map *map = get_vlpi_map(d);
289 if (map)
290 vpe = map->vpe;
291 }
292
293 if (vpe) {
294 cpu = vpe_to_cpuid_lock(vpe, flags);
Marc Zyngierf3a059212020-03-04 20:33:10 +0000295 } else {
296 /* Physical LPIs are already locked via the irq_desc lock */
297 struct its_device *its_dev = irq_data_get_irq_chip_data(d);
298 cpu = its_dev->event_map.col_map[its_get_event_id(d)];
299 /* Keep GCC quiet... */
300 *flags = 0;
301 }
302
303 return cpu;
304}
305
306static void irq_to_cpuid_unlock(struct irq_data *d, unsigned long flags)
307{
Marc Zyngier926846a2023-06-17 08:32:42 +0100308 struct its_vpe *vpe = NULL;
Marc Zyngier425c09b2019-11-08 16:57:57 +0000309
Marc Zyngier926846a2023-06-17 08:32:42 +0100310 if (d->chip == &its_vpe_irq_chip) {
311 vpe = irq_data_get_irq_chip_data(d);
312 } else {
313 struct its_vlpi_map *map = get_vlpi_map(d);
314 if (map)
315 vpe = map->vpe;
316 }
317
318 if (vpe)
319 vpe_to_cpuid_unlock(vpe, flags);
Marc Zyngier425c09b2019-11-08 16:57:57 +0000320}
321
Marc Zyngier83559b42018-06-22 10:52:52 +0100322static struct its_collection *valid_col(struct its_collection *col)
323{
Joe Perches20faba82019-07-09 22:04:18 -0700324 if (WARN_ON_ONCE(col->target_address & GENMASK_ULL(15, 0)))
Marc Zyngier83559b42018-06-22 10:52:52 +0100325 return NULL;
326
327 return col;
328}
329
Marc Zyngier205e0652018-06-22 10:52:53 +0100330static struct its_vpe *valid_vpe(struct its_node *its, struct its_vpe *vpe)
331{
332 if (valid_col(its->collections + vpe->col_idx))
333 return vpe;
334
335 return NULL;
336}
337
Marc Zyngiercc2d3212014-11-24 14:35:11 +0000338/*
339 * ITS command descriptors - parameters to be encoded in a command
340 * block.
341 */
342struct its_cmd_desc {
343 union {
344 struct {
345 struct its_device *dev;
346 u32 event_id;
347 } its_inv_cmd;
348
349 struct {
350 struct its_device *dev;
351 u32 event_id;
Marc Zyngier8d85dce2016-12-19 18:02:13 +0000352 } its_clear_cmd;
353
354 struct {
355 struct its_device *dev;
356 u32 event_id;
Marc Zyngiercc2d3212014-11-24 14:35:11 +0000357 } its_int_cmd;
358
359 struct {
360 struct its_device *dev;
361 int valid;
362 } its_mapd_cmd;
363
364 struct {
365 struct its_collection *col;
366 int valid;
367 } its_mapc_cmd;
368
369 struct {
370 struct its_device *dev;
371 u32 phys_id;
372 u32 event_id;
Marc Zyngier6a25ad32016-12-20 15:52:26 +0000373 } its_mapti_cmd;
Marc Zyngiercc2d3212014-11-24 14:35:11 +0000374
375 struct {
376 struct its_device *dev;
377 struct its_collection *col;
Marc Zyngier591e5be2015-07-17 10:46:42 +0100378 u32 event_id;
Marc Zyngiercc2d3212014-11-24 14:35:11 +0000379 } its_movi_cmd;
380
381 struct {
382 struct its_device *dev;
383 u32 event_id;
384 } its_discard_cmd;
385
386 struct {
387 struct its_collection *col;
388 } its_invall_cmd;
Marc Zyngierd011e4e2016-12-20 09:44:41 +0000389
390 struct {
391 struct its_vpe *vpe;
Marc Zyngiereb781922016-12-20 14:47:05 +0000392 } its_vinvall_cmd;
393
394 struct {
395 struct its_vpe *vpe;
396 struct its_collection *col;
397 bool valid;
398 } its_vmapp_cmd;
399
400 struct {
401 struct its_vpe *vpe;
Marc Zyngierd011e4e2016-12-20 09:44:41 +0000402 struct its_device *dev;
403 u32 virt_id;
404 u32 event_id;
405 bool db_enabled;
406 } its_vmapti_cmd;
407
408 struct {
409 struct its_vpe *vpe;
410 struct its_device *dev;
411 u32 event_id;
412 bool db_enabled;
413 } its_vmovi_cmd;
Marc Zyngier3171a472016-12-20 15:17:28 +0000414
415 struct {
416 struct its_vpe *vpe;
417 struct its_collection *col;
418 u16 seq_num;
419 u16 its_list;
420 } its_vmovp_cmd;
Marc Zyngierd97c97b2019-12-24 11:10:33 +0000421
422 struct {
423 struct its_vpe *vpe;
424 } its_invdb_cmd;
Marc Zyngiere252cf82020-03-04 20:33:16 +0000425
426 struct {
427 struct its_vpe *vpe;
428 u8 sgi;
429 u8 priority;
430 bool enable;
431 bool group;
432 bool clear;
433 } its_vsgi_cmd;
Marc Zyngiercc2d3212014-11-24 14:35:11 +0000434 };
435};
436
437/*
438 * The ITS command block, which is what the ITS actually parses.
439 */
440struct its_cmd_block {
Ben Dooks (Codethink)2bbdfcc2019-10-17 12:29:55 +0100441 union {
442 u64 raw_cmd[4];
443 __le64 raw_cmd_le[4];
444 };
Marc Zyngiercc2d3212014-11-24 14:35:11 +0000445};
446
447#define ITS_CMD_QUEUE_SZ SZ_64K
448#define ITS_CMD_QUEUE_NR_ENTRIES (ITS_CMD_QUEUE_SZ / sizeof(struct its_cmd_block))
449
Marc Zyngier67047f902017-07-28 21:16:58 +0100450typedef struct its_collection *(*its_cmd_builder_t)(struct its_node *,
451 struct its_cmd_block *,
Marc Zyngiercc2d3212014-11-24 14:35:11 +0000452 struct its_cmd_desc *);
453
Marc Zyngier67047f902017-07-28 21:16:58 +0100454typedef struct its_vpe *(*its_cmd_vbuilder_t)(struct its_node *,
455 struct its_cmd_block *,
Marc Zyngierd011e4e2016-12-20 09:44:41 +0000456 struct its_cmd_desc *);
457
Marc Zyngier4d36f132016-12-19 17:11:52 +0000458static void its_mask_encode(u64 *raw_cmd, u64 val, int h, int l)
459{
460 u64 mask = GENMASK_ULL(h, l);
461 *raw_cmd &= ~mask;
462 *raw_cmd |= (val << l) & mask;
463}
464
Marc Zyngiercc2d3212014-11-24 14:35:11 +0000465static void its_encode_cmd(struct its_cmd_block *cmd, u8 cmd_nr)
466{
Marc Zyngier4d36f132016-12-19 17:11:52 +0000467 its_mask_encode(&cmd->raw_cmd[0], cmd_nr, 7, 0);
Marc Zyngiercc2d3212014-11-24 14:35:11 +0000468}
469
470static void its_encode_devid(struct its_cmd_block *cmd, u32 devid)
471{
Marc Zyngier4d36f132016-12-19 17:11:52 +0000472 its_mask_encode(&cmd->raw_cmd[0], devid, 63, 32);
Marc Zyngiercc2d3212014-11-24 14:35:11 +0000473}
474
475static void its_encode_event_id(struct its_cmd_block *cmd, u32 id)
476{
Marc Zyngier4d36f132016-12-19 17:11:52 +0000477 its_mask_encode(&cmd->raw_cmd[1], id, 31, 0);
Marc Zyngiercc2d3212014-11-24 14:35:11 +0000478}
479
480static void its_encode_phys_id(struct its_cmd_block *cmd, u32 phys_id)
481{
Marc Zyngier4d36f132016-12-19 17:11:52 +0000482 its_mask_encode(&cmd->raw_cmd[1], phys_id, 63, 32);
Marc Zyngiercc2d3212014-11-24 14:35:11 +0000483}
484
485static void its_encode_size(struct its_cmd_block *cmd, u8 size)
486{
Marc Zyngier4d36f132016-12-19 17:11:52 +0000487 its_mask_encode(&cmd->raw_cmd[1], size, 4, 0);
Marc Zyngiercc2d3212014-11-24 14:35:11 +0000488}
489
490static void its_encode_itt(struct its_cmd_block *cmd, u64 itt_addr)
491{
Shanker Donthineni30ae9612017-10-09 11:46:55 -0500492 its_mask_encode(&cmd->raw_cmd[2], itt_addr >> 8, 51, 8);
Marc Zyngiercc2d3212014-11-24 14:35:11 +0000493}
494
495static void its_encode_valid(struct its_cmd_block *cmd, int valid)
496{
Marc Zyngier4d36f132016-12-19 17:11:52 +0000497 its_mask_encode(&cmd->raw_cmd[2], !!valid, 63, 63);
Marc Zyngiercc2d3212014-11-24 14:35:11 +0000498}
499
500static void its_encode_target(struct its_cmd_block *cmd, u64 target_addr)
501{
Shanker Donthineni30ae9612017-10-09 11:46:55 -0500502 its_mask_encode(&cmd->raw_cmd[2], target_addr >> 16, 51, 16);
Marc Zyngiercc2d3212014-11-24 14:35:11 +0000503}
504
505static void its_encode_collection(struct its_cmd_block *cmd, u16 col)
506{
Marc Zyngier4d36f132016-12-19 17:11:52 +0000507 its_mask_encode(&cmd->raw_cmd[2], col, 15, 0);
Marc Zyngiercc2d3212014-11-24 14:35:11 +0000508}
509
Marc Zyngierd011e4e2016-12-20 09:44:41 +0000510static void its_encode_vpeid(struct its_cmd_block *cmd, u16 vpeid)
511{
512 its_mask_encode(&cmd->raw_cmd[1], vpeid, 47, 32);
513}
514
515static void its_encode_virt_id(struct its_cmd_block *cmd, u32 virt_id)
516{
517 its_mask_encode(&cmd->raw_cmd[2], virt_id, 31, 0);
518}
519
520static void its_encode_db_phys_id(struct its_cmd_block *cmd, u32 db_phys_id)
521{
522 its_mask_encode(&cmd->raw_cmd[2], db_phys_id, 63, 32);
523}
524
525static void its_encode_db_valid(struct its_cmd_block *cmd, bool db_valid)
526{
527 its_mask_encode(&cmd->raw_cmd[2], db_valid, 0, 0);
528}
529
Marc Zyngier3171a472016-12-20 15:17:28 +0000530static void its_encode_seq_num(struct its_cmd_block *cmd, u16 seq_num)
531{
532 its_mask_encode(&cmd->raw_cmd[0], seq_num, 47, 32);
533}
534
535static void its_encode_its_list(struct its_cmd_block *cmd, u16 its_list)
536{
537 its_mask_encode(&cmd->raw_cmd[1], its_list, 15, 0);
538}
539
Marc Zyngiereb781922016-12-20 14:47:05 +0000540static void its_encode_vpt_addr(struct its_cmd_block *cmd, u64 vpt_pa)
541{
Shanker Donthineni30ae9612017-10-09 11:46:55 -0500542 its_mask_encode(&cmd->raw_cmd[3], vpt_pa >> 16, 51, 16);
Marc Zyngiereb781922016-12-20 14:47:05 +0000543}
544
545static void its_encode_vpt_size(struct its_cmd_block *cmd, u8 vpt_size)
546{
547 its_mask_encode(&cmd->raw_cmd[3], vpt_size, 4, 0);
548}
549
Marc Zyngier64edfaa2019-12-24 11:10:29 +0000550static void its_encode_vconf_addr(struct its_cmd_block *cmd, u64 vconf_pa)
551{
552 its_mask_encode(&cmd->raw_cmd[0], vconf_pa >> 16, 51, 16);
553}
554
555static void its_encode_alloc(struct its_cmd_block *cmd, bool alloc)
556{
557 its_mask_encode(&cmd->raw_cmd[0], alloc, 8, 8);
558}
559
560static void its_encode_ptz(struct its_cmd_block *cmd, bool ptz)
561{
562 its_mask_encode(&cmd->raw_cmd[0], ptz, 9, 9);
563}
564
565static void its_encode_vmapp_default_db(struct its_cmd_block *cmd,
566 u32 vpe_db_lpi)
567{
568 its_mask_encode(&cmd->raw_cmd[1], vpe_db_lpi, 31, 0);
569}
570
Marc Zyngierdd3f0502019-12-24 11:10:31 +0000571static void its_encode_vmovp_default_db(struct its_cmd_block *cmd,
572 u32 vpe_db_lpi)
573{
574 its_mask_encode(&cmd->raw_cmd[3], vpe_db_lpi, 31, 0);
575}
576
577static void its_encode_db(struct its_cmd_block *cmd, bool db)
578{
579 its_mask_encode(&cmd->raw_cmd[2], db, 63, 63);
580}
581
Marc Zyngiere252cf82020-03-04 20:33:16 +0000582static void its_encode_sgi_intid(struct its_cmd_block *cmd, u8 sgi)
583{
584 its_mask_encode(&cmd->raw_cmd[0], sgi, 35, 32);
585}
586
587static void its_encode_sgi_priority(struct its_cmd_block *cmd, u8 prio)
588{
589 its_mask_encode(&cmd->raw_cmd[0], prio >> 4, 23, 20);
590}
591
592static void its_encode_sgi_group(struct its_cmd_block *cmd, bool grp)
593{
594 its_mask_encode(&cmd->raw_cmd[0], grp, 10, 10);
595}
596
597static void its_encode_sgi_clear(struct its_cmd_block *cmd, bool clr)
598{
599 its_mask_encode(&cmd->raw_cmd[0], clr, 9, 9);
600}
601
602static void its_encode_sgi_enable(struct its_cmd_block *cmd, bool en)
603{
604 its_mask_encode(&cmd->raw_cmd[0], en, 8, 8);
605}
606
Marc Zyngiercc2d3212014-11-24 14:35:11 +0000607static inline void its_fixup_cmd(struct its_cmd_block *cmd)
608{
609 /* Let's fixup BE commands */
Ben Dooks (Codethink)2bbdfcc2019-10-17 12:29:55 +0100610 cmd->raw_cmd_le[0] = cpu_to_le64(cmd->raw_cmd[0]);
611 cmd->raw_cmd_le[1] = cpu_to_le64(cmd->raw_cmd[1]);
612 cmd->raw_cmd_le[2] = cpu_to_le64(cmd->raw_cmd[2]);
613 cmd->raw_cmd_le[3] = cpu_to_le64(cmd->raw_cmd[3]);
Marc Zyngiercc2d3212014-11-24 14:35:11 +0000614}
615
Marc Zyngier67047f902017-07-28 21:16:58 +0100616static struct its_collection *its_build_mapd_cmd(struct its_node *its,
617 struct its_cmd_block *cmd,
Marc Zyngiercc2d3212014-11-24 14:35:11 +0000618 struct its_cmd_desc *desc)
619{
620 unsigned long itt_addr;
Marc Zyngierc8481262014-12-12 10:51:24 +0000621 u8 size = ilog2(desc->its_mapd_cmd.dev->nr_ites);
Marc Zyngiercc2d3212014-11-24 14:35:11 +0000622
623 itt_addr = virt_to_phys(desc->its_mapd_cmd.dev->itt);
624 itt_addr = ALIGN(itt_addr, ITS_ITT_ALIGN);
625
626 its_encode_cmd(cmd, GITS_CMD_MAPD);
627 its_encode_devid(cmd, desc->its_mapd_cmd.dev->device_id);
628 its_encode_size(cmd, size - 1);
629 its_encode_itt(cmd, itt_addr);
630 its_encode_valid(cmd, desc->its_mapd_cmd.valid);
631
632 its_fixup_cmd(cmd);
633
Marc Zyngier591e5be2015-07-17 10:46:42 +0100634 return NULL;
Marc Zyngiercc2d3212014-11-24 14:35:11 +0000635}
636
Marc Zyngier67047f902017-07-28 21:16:58 +0100637static struct its_collection *its_build_mapc_cmd(struct its_node *its,
638 struct its_cmd_block *cmd,
Marc Zyngiercc2d3212014-11-24 14:35:11 +0000639 struct its_cmd_desc *desc)
640{
641 its_encode_cmd(cmd, GITS_CMD_MAPC);
642 its_encode_collection(cmd, desc->its_mapc_cmd.col->col_id);
643 its_encode_target(cmd, desc->its_mapc_cmd.col->target_address);
644 its_encode_valid(cmd, desc->its_mapc_cmd.valid);
645
646 its_fixup_cmd(cmd);
647
648 return desc->its_mapc_cmd.col;
649}
650
Marc Zyngier67047f902017-07-28 21:16:58 +0100651static struct its_collection *its_build_mapti_cmd(struct its_node *its,
652 struct its_cmd_block *cmd,
Marc Zyngiercc2d3212014-11-24 14:35:11 +0000653 struct its_cmd_desc *desc)
654{
Marc Zyngier591e5be2015-07-17 10:46:42 +0100655 struct its_collection *col;
656
Marc Zyngier6a25ad32016-12-20 15:52:26 +0000657 col = dev_event_to_col(desc->its_mapti_cmd.dev,
658 desc->its_mapti_cmd.event_id);
Marc Zyngier591e5be2015-07-17 10:46:42 +0100659
Marc Zyngier6a25ad32016-12-20 15:52:26 +0000660 its_encode_cmd(cmd, GITS_CMD_MAPTI);
661 its_encode_devid(cmd, desc->its_mapti_cmd.dev->device_id);
662 its_encode_event_id(cmd, desc->its_mapti_cmd.event_id);
663 its_encode_phys_id(cmd, desc->its_mapti_cmd.phys_id);
Marc Zyngier591e5be2015-07-17 10:46:42 +0100664 its_encode_collection(cmd, col->col_id);
Marc Zyngiercc2d3212014-11-24 14:35:11 +0000665
666 its_fixup_cmd(cmd);
667
Marc Zyngier83559b42018-06-22 10:52:52 +0100668 return valid_col(col);
Marc Zyngiercc2d3212014-11-24 14:35:11 +0000669}
670
Marc Zyngier67047f902017-07-28 21:16:58 +0100671static struct its_collection *its_build_movi_cmd(struct its_node *its,
672 struct its_cmd_block *cmd,
Marc Zyngiercc2d3212014-11-24 14:35:11 +0000673 struct its_cmd_desc *desc)
674{
Marc Zyngier591e5be2015-07-17 10:46:42 +0100675 struct its_collection *col;
676
677 col = dev_event_to_col(desc->its_movi_cmd.dev,
678 desc->its_movi_cmd.event_id);
679
Marc Zyngiercc2d3212014-11-24 14:35:11 +0000680 its_encode_cmd(cmd, GITS_CMD_MOVI);
681 its_encode_devid(cmd, desc->its_movi_cmd.dev->device_id);
Marc Zyngier591e5be2015-07-17 10:46:42 +0100682 its_encode_event_id(cmd, desc->its_movi_cmd.event_id);
Marc Zyngiercc2d3212014-11-24 14:35:11 +0000683 its_encode_collection(cmd, desc->its_movi_cmd.col->col_id);
684
685 its_fixup_cmd(cmd);
686
Marc Zyngier83559b42018-06-22 10:52:52 +0100687 return valid_col(col);
Marc Zyngiercc2d3212014-11-24 14:35:11 +0000688}
689
Marc Zyngier67047f902017-07-28 21:16:58 +0100690static struct its_collection *its_build_discard_cmd(struct its_node *its,
691 struct its_cmd_block *cmd,
Marc Zyngiercc2d3212014-11-24 14:35:11 +0000692 struct its_cmd_desc *desc)
693{
Marc Zyngier591e5be2015-07-17 10:46:42 +0100694 struct its_collection *col;
695
696 col = dev_event_to_col(desc->its_discard_cmd.dev,
697 desc->its_discard_cmd.event_id);
698
Marc Zyngiercc2d3212014-11-24 14:35:11 +0000699 its_encode_cmd(cmd, GITS_CMD_DISCARD);
700 its_encode_devid(cmd, desc->its_discard_cmd.dev->device_id);
701 its_encode_event_id(cmd, desc->its_discard_cmd.event_id);
702
703 its_fixup_cmd(cmd);
704
Marc Zyngier83559b42018-06-22 10:52:52 +0100705 return valid_col(col);
Marc Zyngiercc2d3212014-11-24 14:35:11 +0000706}
707
Marc Zyngier67047f902017-07-28 21:16:58 +0100708static struct its_collection *its_build_inv_cmd(struct its_node *its,
709 struct its_cmd_block *cmd,
Marc Zyngiercc2d3212014-11-24 14:35:11 +0000710 struct its_cmd_desc *desc)
711{
Marc Zyngier591e5be2015-07-17 10:46:42 +0100712 struct its_collection *col;
713
714 col = dev_event_to_col(desc->its_inv_cmd.dev,
715 desc->its_inv_cmd.event_id);
716
Marc Zyngiercc2d3212014-11-24 14:35:11 +0000717 its_encode_cmd(cmd, GITS_CMD_INV);
718 its_encode_devid(cmd, desc->its_inv_cmd.dev->device_id);
719 its_encode_event_id(cmd, desc->its_inv_cmd.event_id);
720
721 its_fixup_cmd(cmd);
722
Marc Zyngier83559b42018-06-22 10:52:52 +0100723 return valid_col(col);
Marc Zyngiercc2d3212014-11-24 14:35:11 +0000724}
725
Marc Zyngier67047f902017-07-28 21:16:58 +0100726static struct its_collection *its_build_int_cmd(struct its_node *its,
727 struct its_cmd_block *cmd,
Marc Zyngier8d85dce2016-12-19 18:02:13 +0000728 struct its_cmd_desc *desc)
729{
730 struct its_collection *col;
731
732 col = dev_event_to_col(desc->its_int_cmd.dev,
733 desc->its_int_cmd.event_id);
734
735 its_encode_cmd(cmd, GITS_CMD_INT);
736 its_encode_devid(cmd, desc->its_int_cmd.dev->device_id);
737 its_encode_event_id(cmd, desc->its_int_cmd.event_id);
738
739 its_fixup_cmd(cmd);
740
Marc Zyngier83559b42018-06-22 10:52:52 +0100741 return valid_col(col);
Marc Zyngier8d85dce2016-12-19 18:02:13 +0000742}
743
Marc Zyngier67047f902017-07-28 21:16:58 +0100744static struct its_collection *its_build_clear_cmd(struct its_node *its,
745 struct its_cmd_block *cmd,
Marc Zyngier8d85dce2016-12-19 18:02:13 +0000746 struct its_cmd_desc *desc)
747{
748 struct its_collection *col;
749
750 col = dev_event_to_col(desc->its_clear_cmd.dev,
751 desc->its_clear_cmd.event_id);
752
753 its_encode_cmd(cmd, GITS_CMD_CLEAR);
754 its_encode_devid(cmd, desc->its_clear_cmd.dev->device_id);
755 its_encode_event_id(cmd, desc->its_clear_cmd.event_id);
756
757 its_fixup_cmd(cmd);
758
Marc Zyngier83559b42018-06-22 10:52:52 +0100759 return valid_col(col);
Marc Zyngier8d85dce2016-12-19 18:02:13 +0000760}
761
Marc Zyngier67047f902017-07-28 21:16:58 +0100762static struct its_collection *its_build_invall_cmd(struct its_node *its,
763 struct its_cmd_block *cmd,
Marc Zyngiercc2d3212014-11-24 14:35:11 +0000764 struct its_cmd_desc *desc)
765{
766 its_encode_cmd(cmd, GITS_CMD_INVALL);
Zenghui Yu10794522019-12-02 15:10:21 +0800767 its_encode_collection(cmd, desc->its_invall_cmd.col->col_id);
Marc Zyngiercc2d3212014-11-24 14:35:11 +0000768
769 its_fixup_cmd(cmd);
770
Wudi Wangb383a422021-12-08 09:54:29 +0800771 return desc->its_invall_cmd.col;
Marc Zyngiercc2d3212014-11-24 14:35:11 +0000772}
773
Marc Zyngier67047f902017-07-28 21:16:58 +0100774static struct its_vpe *its_build_vinvall_cmd(struct its_node *its,
775 struct its_cmd_block *cmd,
Marc Zyngiereb781922016-12-20 14:47:05 +0000776 struct its_cmd_desc *desc)
777{
778 its_encode_cmd(cmd, GITS_CMD_VINVALL);
779 its_encode_vpeid(cmd, desc->its_vinvall_cmd.vpe->vpe_id);
780
781 its_fixup_cmd(cmd);
782
Marc Zyngier205e0652018-06-22 10:52:53 +0100783 return valid_vpe(its, desc->its_vinvall_cmd.vpe);
Marc Zyngiereb781922016-12-20 14:47:05 +0000784}
785
Marc Zyngier67047f902017-07-28 21:16:58 +0100786static struct its_vpe *its_build_vmapp_cmd(struct its_node *its,
787 struct its_cmd_block *cmd,
Marc Zyngiereb781922016-12-20 14:47:05 +0000788 struct its_cmd_desc *desc)
789{
Nianyao Tang80e99632024-04-06 02:27:37 +0000790 struct its_vpe *vpe = valid_vpe(its, desc->its_vmapp_cmd.vpe);
Marc Zyngier64edfaa2019-12-24 11:10:29 +0000791 unsigned long vpt_addr, vconf_addr;
Marc Zyngier5c9a8822017-07-28 21:20:37 +0100792 u64 target;
Marc Zyngier64edfaa2019-12-24 11:10:29 +0000793 bool alloc;
Marc Zyngiereb781922016-12-20 14:47:05 +0000794
795 its_encode_cmd(cmd, GITS_CMD_VMAPP);
796 its_encode_vpeid(cmd, desc->its_vmapp_cmd.vpe->vpe_id);
797 its_encode_valid(cmd, desc->its_vmapp_cmd.valid);
Marc Zyngier64edfaa2019-12-24 11:10:29 +0000798
799 if (!desc->its_vmapp_cmd.valid) {
800 if (is_v4_1(its)) {
801 alloc = !atomic_dec_return(&desc->its_vmapp_cmd.vpe->vmapp_count);
802 its_encode_alloc(cmd, alloc);
Nianyao Tang80e99632024-04-06 02:27:37 +0000803 /*
804 * Unmapping a VPE is self-synchronizing on GICv4.1,
805 * no need to issue a VSYNC.
806 */
807 vpe = NULL;
Marc Zyngier64edfaa2019-12-24 11:10:29 +0000808 }
809
810 goto out;
811 }
812
813 vpt_addr = virt_to_phys(page_address(desc->its_vmapp_cmd.vpe->vpt_page));
814 target = desc->its_vmapp_cmd.col->target_address + its->vlpi_redist_offset;
815
Marc Zyngier5c9a8822017-07-28 21:20:37 +0100816 its_encode_target(cmd, target);
Marc Zyngiereb781922016-12-20 14:47:05 +0000817 its_encode_vpt_addr(cmd, vpt_addr);
818 its_encode_vpt_size(cmd, LPI_NRBITS - 1);
819
Marc Zyngier64edfaa2019-12-24 11:10:29 +0000820 if (!is_v4_1(its))
821 goto out;
822
823 vconf_addr = virt_to_phys(page_address(desc->its_vmapp_cmd.vpe->its_vm->vprop_page));
824
825 alloc = !atomic_fetch_inc(&desc->its_vmapp_cmd.vpe->vmapp_count);
826
827 its_encode_alloc(cmd, alloc);
828
Shenming Luc21bc062021-03-22 14:01:54 +0800829 /*
830 * GICv4.1 provides a way to get the VLPI state, which needs the vPE
831 * to be unmapped first, and in this case, we may remap the vPE
832 * back while the VPT is not empty. So we can't assume that the
833 * VPT is empty on map. This is why we never advertise PTZ.
834 */
835 its_encode_ptz(cmd, false);
Marc Zyngier64edfaa2019-12-24 11:10:29 +0000836 its_encode_vconf_addr(cmd, vconf_addr);
837 its_encode_vmapp_default_db(cmd, desc->its_vmapp_cmd.vpe->vpe_db_lpi);
838
839out:
Marc Zyngiereb781922016-12-20 14:47:05 +0000840 its_fixup_cmd(cmd);
841
Nianyao Tang80e99632024-04-06 02:27:37 +0000842 return vpe;
Marc Zyngiereb781922016-12-20 14:47:05 +0000843}
844
Marc Zyngier67047f902017-07-28 21:16:58 +0100845static struct its_vpe *its_build_vmapti_cmd(struct its_node *its,
846 struct its_cmd_block *cmd,
Marc Zyngierd011e4e2016-12-20 09:44:41 +0000847 struct its_cmd_desc *desc)
848{
849 u32 db;
850
Marc Zyngier3858d4d2019-12-24 11:10:37 +0000851 if (!is_v4_1(its) && desc->its_vmapti_cmd.db_enabled)
Marc Zyngierd011e4e2016-12-20 09:44:41 +0000852 db = desc->its_vmapti_cmd.vpe->vpe_db_lpi;
853 else
854 db = 1023;
855
856 its_encode_cmd(cmd, GITS_CMD_VMAPTI);
857 its_encode_devid(cmd, desc->its_vmapti_cmd.dev->device_id);
858 its_encode_vpeid(cmd, desc->its_vmapti_cmd.vpe->vpe_id);
859 its_encode_event_id(cmd, desc->its_vmapti_cmd.event_id);
860 its_encode_db_phys_id(cmd, db);
861 its_encode_virt_id(cmd, desc->its_vmapti_cmd.virt_id);
862
863 its_fixup_cmd(cmd);
864
Marc Zyngier205e0652018-06-22 10:52:53 +0100865 return valid_vpe(its, desc->its_vmapti_cmd.vpe);
Marc Zyngierd011e4e2016-12-20 09:44:41 +0000866}
867
Marc Zyngier67047f902017-07-28 21:16:58 +0100868static struct its_vpe *its_build_vmovi_cmd(struct its_node *its,
869 struct its_cmd_block *cmd,
Marc Zyngierd011e4e2016-12-20 09:44:41 +0000870 struct its_cmd_desc *desc)
871{
872 u32 db;
873
Marc Zyngier3858d4d2019-12-24 11:10:37 +0000874 if (!is_v4_1(its) && desc->its_vmovi_cmd.db_enabled)
Marc Zyngierd011e4e2016-12-20 09:44:41 +0000875 db = desc->its_vmovi_cmd.vpe->vpe_db_lpi;
876 else
877 db = 1023;
878
879 its_encode_cmd(cmd, GITS_CMD_VMOVI);
880 its_encode_devid(cmd, desc->its_vmovi_cmd.dev->device_id);
881 its_encode_vpeid(cmd, desc->its_vmovi_cmd.vpe->vpe_id);
882 its_encode_event_id(cmd, desc->its_vmovi_cmd.event_id);
883 its_encode_db_phys_id(cmd, db);
884 its_encode_db_valid(cmd, true);
885
886 its_fixup_cmd(cmd);
887
Marc Zyngier205e0652018-06-22 10:52:53 +0100888 return valid_vpe(its, desc->its_vmovi_cmd.vpe);
Marc Zyngierd011e4e2016-12-20 09:44:41 +0000889}
890
Marc Zyngier67047f902017-07-28 21:16:58 +0100891static struct its_vpe *its_build_vmovp_cmd(struct its_node *its,
892 struct its_cmd_block *cmd,
Marc Zyngier3171a472016-12-20 15:17:28 +0000893 struct its_cmd_desc *desc)
894{
Marc Zyngier5c9a8822017-07-28 21:20:37 +0100895 u64 target;
896
897 target = desc->its_vmovp_cmd.col->target_address + its->vlpi_redist_offset;
Marc Zyngier3171a472016-12-20 15:17:28 +0000898 its_encode_cmd(cmd, GITS_CMD_VMOVP);
899 its_encode_seq_num(cmd, desc->its_vmovp_cmd.seq_num);
900 its_encode_its_list(cmd, desc->its_vmovp_cmd.its_list);
901 its_encode_vpeid(cmd, desc->its_vmovp_cmd.vpe->vpe_id);
Marc Zyngier5c9a8822017-07-28 21:20:37 +0100902 its_encode_target(cmd, target);
Marc Zyngier3171a472016-12-20 15:17:28 +0000903
Marc Zyngierdd3f0502019-12-24 11:10:31 +0000904 if (is_v4_1(its)) {
905 its_encode_db(cmd, true);
906 its_encode_vmovp_default_db(cmd, desc->its_vmovp_cmd.vpe->vpe_db_lpi);
907 }
908
Marc Zyngier3171a472016-12-20 15:17:28 +0000909 its_fixup_cmd(cmd);
910
Marc Zyngier205e0652018-06-22 10:52:53 +0100911 return valid_vpe(its, desc->its_vmovp_cmd.vpe);
Marc Zyngier3171a472016-12-20 15:17:28 +0000912}
913
Marc Zyngier28614692019-11-08 16:58:02 +0000914static struct its_vpe *its_build_vinv_cmd(struct its_node *its,
915 struct its_cmd_block *cmd,
916 struct its_cmd_desc *desc)
917{
918 struct its_vlpi_map *map;
919
920 map = dev_event_to_vlpi_map(desc->its_inv_cmd.dev,
921 desc->its_inv_cmd.event_id);
922
923 its_encode_cmd(cmd, GITS_CMD_INV);
924 its_encode_devid(cmd, desc->its_inv_cmd.dev->device_id);
925 its_encode_event_id(cmd, desc->its_inv_cmd.event_id);
926
927 its_fixup_cmd(cmd);
928
929 return valid_vpe(its, map->vpe);
930}
931
Marc Zyngiered0e4aa2019-11-08 16:58:03 +0000932static struct its_vpe *its_build_vint_cmd(struct its_node *its,
933 struct its_cmd_block *cmd,
934 struct its_cmd_desc *desc)
935{
936 struct its_vlpi_map *map;
937
938 map = dev_event_to_vlpi_map(desc->its_int_cmd.dev,
939 desc->its_int_cmd.event_id);
940
941 its_encode_cmd(cmd, GITS_CMD_INT);
942 its_encode_devid(cmd, desc->its_int_cmd.dev->device_id);
943 its_encode_event_id(cmd, desc->its_int_cmd.event_id);
944
945 its_fixup_cmd(cmd);
946
947 return valid_vpe(its, map->vpe);
948}
949
950static struct its_vpe *its_build_vclear_cmd(struct its_node *its,
951 struct its_cmd_block *cmd,
952 struct its_cmd_desc *desc)
953{
954 struct its_vlpi_map *map;
955
956 map = dev_event_to_vlpi_map(desc->its_clear_cmd.dev,
957 desc->its_clear_cmd.event_id);
958
959 its_encode_cmd(cmd, GITS_CMD_CLEAR);
960 its_encode_devid(cmd, desc->its_clear_cmd.dev->device_id);
961 its_encode_event_id(cmd, desc->its_clear_cmd.event_id);
962
963 its_fixup_cmd(cmd);
964
965 return valid_vpe(its, map->vpe);
966}
967
Marc Zyngierd97c97b2019-12-24 11:10:33 +0000968static struct its_vpe *its_build_invdb_cmd(struct its_node *its,
969 struct its_cmd_block *cmd,
970 struct its_cmd_desc *desc)
971{
972 if (WARN_ON(!is_v4_1(its)))
973 return NULL;
974
975 its_encode_cmd(cmd, GITS_CMD_INVDB);
976 its_encode_vpeid(cmd, desc->its_invdb_cmd.vpe->vpe_id);
977
978 its_fixup_cmd(cmd);
979
980 return valid_vpe(its, desc->its_invdb_cmd.vpe);
981}
982
Marc Zyngiere252cf82020-03-04 20:33:16 +0000983static struct its_vpe *its_build_vsgi_cmd(struct its_node *its,
984 struct its_cmd_block *cmd,
985 struct its_cmd_desc *desc)
986{
987 if (WARN_ON(!is_v4_1(its)))
988 return NULL;
989
990 its_encode_cmd(cmd, GITS_CMD_VSGI);
991 its_encode_vpeid(cmd, desc->its_vsgi_cmd.vpe->vpe_id);
992 its_encode_sgi_intid(cmd, desc->its_vsgi_cmd.sgi);
993 its_encode_sgi_priority(cmd, desc->its_vsgi_cmd.priority);
994 its_encode_sgi_group(cmd, desc->its_vsgi_cmd.group);
995 its_encode_sgi_clear(cmd, desc->its_vsgi_cmd.clear);
996 its_encode_sgi_enable(cmd, desc->its_vsgi_cmd.enable);
997
998 its_fixup_cmd(cmd);
999
1000 return valid_vpe(its, desc->its_vsgi_cmd.vpe);
1001}
1002
Marc Zyngiercc2d3212014-11-24 14:35:11 +00001003static u64 its_cmd_ptr_to_offset(struct its_node *its,
1004 struct its_cmd_block *ptr)
1005{
1006 return (ptr - its->cmd_base) * sizeof(*ptr);
1007}
1008
1009static int its_queue_full(struct its_node *its)
1010{
1011 int widx;
1012 int ridx;
1013
1014 widx = its->cmd_write - its->cmd_base;
1015 ridx = readl_relaxed(its->base + GITS_CREADR) / sizeof(struct its_cmd_block);
1016
1017 /* This is incredibly unlikely to happen, unless the ITS locks up. */
1018 if (((widx + 1) % ITS_CMD_QUEUE_NR_ENTRIES) == ridx)
1019 return 1;
1020
1021 return 0;
1022}
1023
1024static struct its_cmd_block *its_allocate_entry(struct its_node *its)
1025{
1026 struct its_cmd_block *cmd;
1027 u32 count = 1000000; /* 1s! */
1028
1029 while (its_queue_full(its)) {
1030 count--;
1031 if (!count) {
1032 pr_err_ratelimited("ITS queue not draining\n");
1033 return NULL;
1034 }
1035 cpu_relax();
1036 udelay(1);
1037 }
1038
1039 cmd = its->cmd_write++;
1040
1041 /* Handle queue wrapping */
1042 if (its->cmd_write == (its->cmd_base + ITS_CMD_QUEUE_NR_ENTRIES))
1043 its->cmd_write = its->cmd_base;
1044
Marc Zyngier34d677a2016-12-19 17:16:45 +00001045 /* Clear command */
1046 cmd->raw_cmd[0] = 0;
1047 cmd->raw_cmd[1] = 0;
1048 cmd->raw_cmd[2] = 0;
1049 cmd->raw_cmd[3] = 0;
1050
Marc Zyngiercc2d3212014-11-24 14:35:11 +00001051 return cmd;
1052}
1053
1054static struct its_cmd_block *its_post_commands(struct its_node *its)
1055{
1056 u64 wr = its_cmd_ptr_to_offset(its, its->cmd_write);
1057
1058 writel_relaxed(wr, its->base + GITS_CWRITER);
1059
1060 return its->cmd_write;
1061}
1062
1063static void its_flush_cmd(struct its_node *its, struct its_cmd_block *cmd)
1064{
1065 /*
1066 * Make sure the commands written to memory are observable by
1067 * the ITS.
1068 */
1069 if (its->flags & ITS_FLAGS_CMDQ_NEEDS_FLUSHING)
Vladimir Murzin328191c2016-11-02 11:54:05 +00001070 gic_flush_dcache_to_poc(cmd, sizeof(*cmd));
Marc Zyngiercc2d3212014-11-24 14:35:11 +00001071 else
1072 dsb(ishst);
1073}
1074
Marc Zyngiera19b4622017-08-04 17:45:50 +01001075static int its_wait_for_range_completion(struct its_node *its,
Heyi Guoa050fa52019-05-13 19:42:06 +08001076 u64 prev_idx,
Marc Zyngiera19b4622017-08-04 17:45:50 +01001077 struct its_cmd_block *to)
Marc Zyngiercc2d3212014-11-24 14:35:11 +00001078{
Heyi Guoa050fa52019-05-13 19:42:06 +08001079 u64 rd_idx, to_idx, linear_idx;
Marc Zyngiercc2d3212014-11-24 14:35:11 +00001080 u32 count = 1000000; /* 1s! */
1081
Heyi Guoa050fa52019-05-13 19:42:06 +08001082 /* Linearize to_idx if the command set has wrapped around */
Marc Zyngiercc2d3212014-11-24 14:35:11 +00001083 to_idx = its_cmd_ptr_to_offset(its, to);
Heyi Guoa050fa52019-05-13 19:42:06 +08001084 if (to_idx < prev_idx)
1085 to_idx += ITS_CMD_QUEUE_SZ;
1086
1087 linear_idx = prev_idx;
Marc Zyngiercc2d3212014-11-24 14:35:11 +00001088
1089 while (1) {
Heyi Guoa050fa52019-05-13 19:42:06 +08001090 s64 delta;
1091
Marc Zyngiercc2d3212014-11-24 14:35:11 +00001092 rd_idx = readl_relaxed(its->base + GITS_CREADR);
Marc Zyngier9bdd8b12017-08-19 10:16:02 +01001093
Heyi Guoa050fa52019-05-13 19:42:06 +08001094 /*
1095 * Compute the read pointer progress, taking the
1096 * potential wrap-around into account.
1097 */
1098 delta = rd_idx - prev_idx;
1099 if (rd_idx < prev_idx)
1100 delta += ITS_CMD_QUEUE_SZ;
Marc Zyngier9bdd8b12017-08-19 10:16:02 +01001101
Heyi Guoa050fa52019-05-13 19:42:06 +08001102 linear_idx += delta;
1103 if (linear_idx >= to_idx)
Marc Zyngiercc2d3212014-11-24 14:35:11 +00001104 break;
1105
1106 count--;
1107 if (!count) {
Heyi Guoa050fa52019-05-13 19:42:06 +08001108 pr_err_ratelimited("ITS queue timeout (%llu %llu)\n",
1109 to_idx, linear_idx);
Marc Zyngiera19b4622017-08-04 17:45:50 +01001110 return -1;
Marc Zyngiercc2d3212014-11-24 14:35:11 +00001111 }
Heyi Guoa050fa52019-05-13 19:42:06 +08001112 prev_idx = rd_idx;
Marc Zyngiercc2d3212014-11-24 14:35:11 +00001113 cpu_relax();
1114 udelay(1);
1115 }
Marc Zyngiera19b4622017-08-04 17:45:50 +01001116
1117 return 0;
Marc Zyngiercc2d3212014-11-24 14:35:11 +00001118}
1119
Marc Zyngiere4f90942016-12-19 17:56:32 +00001120/* Warning, macro hell follows */
1121#define BUILD_SINGLE_CMD_FUNC(name, buildtype, synctype, buildfn) \
1122void name(struct its_node *its, \
1123 buildtype builder, \
1124 struct its_cmd_desc *desc) \
1125{ \
1126 struct its_cmd_block *cmd, *sync_cmd, *next_cmd; \
1127 synctype *sync_obj; \
1128 unsigned long flags; \
Heyi Guoa050fa52019-05-13 19:42:06 +08001129 u64 rd_idx; \
Marc Zyngiere4f90942016-12-19 17:56:32 +00001130 \
1131 raw_spin_lock_irqsave(&its->lock, flags); \
1132 \
1133 cmd = its_allocate_entry(its); \
1134 if (!cmd) { /* We're soooooo screewed... */ \
1135 raw_spin_unlock_irqrestore(&its->lock, flags); \
1136 return; \
1137 } \
Marc Zyngier67047f902017-07-28 21:16:58 +01001138 sync_obj = builder(its, cmd, desc); \
Marc Zyngiere4f90942016-12-19 17:56:32 +00001139 its_flush_cmd(its, cmd); \
1140 \
1141 if (sync_obj) { \
1142 sync_cmd = its_allocate_entry(its); \
1143 if (!sync_cmd) \
1144 goto post; \
1145 \
Marc Zyngier67047f902017-07-28 21:16:58 +01001146 buildfn(its, sync_cmd, sync_obj); \
Marc Zyngiere4f90942016-12-19 17:56:32 +00001147 its_flush_cmd(its, sync_cmd); \
1148 } \
1149 \
1150post: \
Heyi Guoa050fa52019-05-13 19:42:06 +08001151 rd_idx = readl_relaxed(its->base + GITS_CREADR); \
Marc Zyngiere4f90942016-12-19 17:56:32 +00001152 next_cmd = its_post_commands(its); \
1153 raw_spin_unlock_irqrestore(&its->lock, flags); \
1154 \
Heyi Guoa050fa52019-05-13 19:42:06 +08001155 if (its_wait_for_range_completion(its, rd_idx, next_cmd)) \
Marc Zyngiera19b4622017-08-04 17:45:50 +01001156 pr_err_ratelimited("ITS cmd %ps failed\n", builder); \
Marc Zyngiercc2d3212014-11-24 14:35:11 +00001157}
1158
Marc Zyngier67047f902017-07-28 21:16:58 +01001159static void its_build_sync_cmd(struct its_node *its,
1160 struct its_cmd_block *sync_cmd,
Marc Zyngiere4f90942016-12-19 17:56:32 +00001161 struct its_collection *sync_col)
Marc Zyngiercc2d3212014-11-24 14:35:11 +00001162{
Marc Zyngiere4f90942016-12-19 17:56:32 +00001163 its_encode_cmd(sync_cmd, GITS_CMD_SYNC);
1164 its_encode_target(sync_cmd, sync_col->target_address);
Marc Zyngiercc2d3212014-11-24 14:35:11 +00001165
Marc Zyngiere4f90942016-12-19 17:56:32 +00001166 its_fixup_cmd(sync_cmd);
1167}
Marc Zyngiercc2d3212014-11-24 14:35:11 +00001168
Marc Zyngiere4f90942016-12-19 17:56:32 +00001169static BUILD_SINGLE_CMD_FUNC(its_send_single_command, its_cmd_builder_t,
1170 struct its_collection, its_build_sync_cmd)
Marc Zyngiercc2d3212014-11-24 14:35:11 +00001171
Marc Zyngier67047f902017-07-28 21:16:58 +01001172static void its_build_vsync_cmd(struct its_node *its,
1173 struct its_cmd_block *sync_cmd,
Marc Zyngierd011e4e2016-12-20 09:44:41 +00001174 struct its_vpe *sync_vpe)
1175{
1176 its_encode_cmd(sync_cmd, GITS_CMD_VSYNC);
1177 its_encode_vpeid(sync_cmd, sync_vpe->vpe_id);
Marc Zyngiercc2d3212014-11-24 14:35:11 +00001178
Marc Zyngierd011e4e2016-12-20 09:44:41 +00001179 its_fixup_cmd(sync_cmd);
1180}
Marc Zyngiercc2d3212014-11-24 14:35:11 +00001181
Marc Zyngierd011e4e2016-12-20 09:44:41 +00001182static BUILD_SINGLE_CMD_FUNC(its_send_single_vcommand, its_cmd_vbuilder_t,
1183 struct its_vpe, its_build_vsync_cmd)
1184
Marc Zyngier8d85dce2016-12-19 18:02:13 +00001185static void its_send_int(struct its_device *dev, u32 event_id)
1186{
1187 struct its_cmd_desc desc;
1188
1189 desc.its_int_cmd.dev = dev;
1190 desc.its_int_cmd.event_id = event_id;
1191
1192 its_send_single_command(dev->its, its_build_int_cmd, &desc);
1193}
1194
1195static void its_send_clear(struct its_device *dev, u32 event_id)
1196{
1197 struct its_cmd_desc desc;
1198
1199 desc.its_clear_cmd.dev = dev;
1200 desc.its_clear_cmd.event_id = event_id;
1201
1202 its_send_single_command(dev->its, its_build_clear_cmd, &desc);
Marc Zyngiercc2d3212014-11-24 14:35:11 +00001203}
1204
1205static void its_send_inv(struct its_device *dev, u32 event_id)
1206{
1207 struct its_cmd_desc desc;
1208
1209 desc.its_inv_cmd.dev = dev;
1210 desc.its_inv_cmd.event_id = event_id;
1211
1212 its_send_single_command(dev->its, its_build_inv_cmd, &desc);
1213}
1214
1215static void its_send_mapd(struct its_device *dev, int valid)
1216{
1217 struct its_cmd_desc desc;
1218
1219 desc.its_mapd_cmd.dev = dev;
1220 desc.its_mapd_cmd.valid = !!valid;
1221
1222 its_send_single_command(dev->its, its_build_mapd_cmd, &desc);
1223}
1224
1225static void its_send_mapc(struct its_node *its, struct its_collection *col,
1226 int valid)
1227{
1228 struct its_cmd_desc desc;
1229
1230 desc.its_mapc_cmd.col = col;
1231 desc.its_mapc_cmd.valid = !!valid;
1232
1233 its_send_single_command(its, its_build_mapc_cmd, &desc);
1234}
1235
Marc Zyngier6a25ad32016-12-20 15:52:26 +00001236static void its_send_mapti(struct its_device *dev, u32 irq_id, u32 id)
Marc Zyngiercc2d3212014-11-24 14:35:11 +00001237{
1238 struct its_cmd_desc desc;
1239
Marc Zyngier6a25ad32016-12-20 15:52:26 +00001240 desc.its_mapti_cmd.dev = dev;
1241 desc.its_mapti_cmd.phys_id = irq_id;
1242 desc.its_mapti_cmd.event_id = id;
Marc Zyngiercc2d3212014-11-24 14:35:11 +00001243
Marc Zyngier6a25ad32016-12-20 15:52:26 +00001244 its_send_single_command(dev->its, its_build_mapti_cmd, &desc);
Marc Zyngiercc2d3212014-11-24 14:35:11 +00001245}
1246
1247static void its_send_movi(struct its_device *dev,
1248 struct its_collection *col, u32 id)
1249{
1250 struct its_cmd_desc desc;
1251
1252 desc.its_movi_cmd.dev = dev;
1253 desc.its_movi_cmd.col = col;
Marc Zyngier591e5be2015-07-17 10:46:42 +01001254 desc.its_movi_cmd.event_id = id;
Marc Zyngiercc2d3212014-11-24 14:35:11 +00001255
1256 its_send_single_command(dev->its, its_build_movi_cmd, &desc);
1257}
1258
1259static void its_send_discard(struct its_device *dev, u32 id)
1260{
1261 struct its_cmd_desc desc;
1262
1263 desc.its_discard_cmd.dev = dev;
1264 desc.its_discard_cmd.event_id = id;
1265
1266 its_send_single_command(dev->its, its_build_discard_cmd, &desc);
1267}
1268
1269static void its_send_invall(struct its_node *its, struct its_collection *col)
1270{
1271 struct its_cmd_desc desc;
1272
1273 desc.its_invall_cmd.col = col;
1274
1275 its_send_single_command(its, its_build_invall_cmd, &desc);
1276}
Marc Zyngierc48ed512014-11-24 14:35:12 +00001277
Marc Zyngierd011e4e2016-12-20 09:44:41 +00001278static void its_send_vmapti(struct its_device *dev, u32 id)
1279{
Marc Zyngierc1d4d5c2019-11-08 16:58:01 +00001280 struct its_vlpi_map *map = dev_event_to_vlpi_map(dev, id);
Marc Zyngierd011e4e2016-12-20 09:44:41 +00001281 struct its_cmd_desc desc;
1282
1283 desc.its_vmapti_cmd.vpe = map->vpe;
1284 desc.its_vmapti_cmd.dev = dev;
1285 desc.its_vmapti_cmd.virt_id = map->vintid;
1286 desc.its_vmapti_cmd.event_id = id;
1287 desc.its_vmapti_cmd.db_enabled = map->db_enabled;
1288
1289 its_send_single_vcommand(dev->its, its_build_vmapti_cmd, &desc);
1290}
1291
1292static void its_send_vmovi(struct its_device *dev, u32 id)
1293{
Marc Zyngierc1d4d5c2019-11-08 16:58:01 +00001294 struct its_vlpi_map *map = dev_event_to_vlpi_map(dev, id);
Marc Zyngierd011e4e2016-12-20 09:44:41 +00001295 struct its_cmd_desc desc;
1296
1297 desc.its_vmovi_cmd.vpe = map->vpe;
1298 desc.its_vmovi_cmd.dev = dev;
1299 desc.its_vmovi_cmd.event_id = id;
1300 desc.its_vmovi_cmd.db_enabled = map->db_enabled;
1301
1302 its_send_single_vcommand(dev->its, its_build_vmovi_cmd, &desc);
1303}
1304
Marc Zyngier75fd9512017-10-08 18:46:39 +01001305static void its_send_vmapp(struct its_node *its,
1306 struct its_vpe *vpe, bool valid)
Marc Zyngiereb781922016-12-20 14:47:05 +00001307{
1308 struct its_cmd_desc desc;
Marc Zyngiereb781922016-12-20 14:47:05 +00001309
1310 desc.its_vmapp_cmd.vpe = vpe;
1311 desc.its_vmapp_cmd.valid = valid;
Marc Zyngier75fd9512017-10-08 18:46:39 +01001312 desc.its_vmapp_cmd.col = &its->collections[vpe->col_idx];
Marc Zyngiereb781922016-12-20 14:47:05 +00001313
Marc Zyngier75fd9512017-10-08 18:46:39 +01001314 its_send_single_vcommand(its, its_build_vmapp_cmd, &desc);
Marc Zyngiereb781922016-12-20 14:47:05 +00001315}
1316
Marc Zyngier3171a472016-12-20 15:17:28 +00001317static void its_send_vmovp(struct its_vpe *vpe)
1318{
Zenghui Yu84243122019-10-23 03:46:26 +00001319 struct its_cmd_desc desc = {};
Marc Zyngier3171a472016-12-20 15:17:28 +00001320 struct its_node *its;
Marc Zyngier3171a472016-12-20 15:17:28 +00001321 int col_id = vpe->col_idx;
1322
1323 desc.its_vmovp_cmd.vpe = vpe;
Marc Zyngier3171a472016-12-20 15:17:28 +00001324
1325 if (!its_list_map) {
1326 its = list_first_entry(&its_nodes, struct its_node, entry);
Marc Zyngier3171a472016-12-20 15:17:28 +00001327 desc.its_vmovp_cmd.col = &its->collections[col_id];
1328 its_send_single_vcommand(its, its_build_vmovp_cmd, &desc);
1329 return;
1330 }
1331
1332 /*
1333 * Yet another marvel of the architecture. If using the
1334 * its_list "feature", we need to make sure that all ITSs
1335 * receive all VMOVP commands in the same order. The only way
1336 * to guarantee this is to make vmovp a serialization point.
1337 *
1338 * Wall <-- Head.
1339 */
Marc Zyngierf0eb1542024-07-05 10:31:54 +01001340 guard(raw_spinlock)(&vmovp_lock);
Marc Zyngier3171a472016-12-20 15:17:28 +00001341 desc.its_vmovp_cmd.seq_num = vmovp_seq_num++;
Zenghui Yu84243122019-10-23 03:46:26 +00001342 desc.its_vmovp_cmd.its_list = get_its_list(vpe->its_vm);
Marc Zyngier3171a472016-12-20 15:17:28 +00001343
1344 /* Emit VMOVPs */
1345 list_for_each_entry(its, &its_nodes, entry) {
Marc Zyngier0dd57fe2019-11-08 16:57:58 +00001346 if (!is_v4(its))
Marc Zyngier3171a472016-12-20 15:17:28 +00001347 continue;
1348
Marc Zyngier009384b2020-03-04 20:33:23 +00001349 if (!require_its_list_vmovp(vpe->its_vm, its))
Marc Zyngier2247e1b2017-10-08 18:50:36 +01001350 continue;
1351
Marc Zyngier3171a472016-12-20 15:17:28 +00001352 desc.its_vmovp_cmd.col = &its->collections[col_id];
1353 its_send_single_vcommand(its, its_build_vmovp_cmd, &desc);
1354 }
Marc Zyngier3171a472016-12-20 15:17:28 +00001355}
1356
Marc Zyngier40619a22017-10-08 15:16:09 +01001357static void its_send_vinvall(struct its_node *its, struct its_vpe *vpe)
Marc Zyngiereb781922016-12-20 14:47:05 +00001358{
1359 struct its_cmd_desc desc;
Marc Zyngiereb781922016-12-20 14:47:05 +00001360
1361 desc.its_vinvall_cmd.vpe = vpe;
Marc Zyngier40619a22017-10-08 15:16:09 +01001362 its_send_single_vcommand(its, its_build_vinvall_cmd, &desc);
Marc Zyngiereb781922016-12-20 14:47:05 +00001363}
1364
Marc Zyngier28614692019-11-08 16:58:02 +00001365static void its_send_vinv(struct its_device *dev, u32 event_id)
1366{
1367 struct its_cmd_desc desc;
1368
1369 /*
1370 * There is no real VINV command. This is just a normal INV,
1371 * with a VSYNC instead of a SYNC.
1372 */
1373 desc.its_inv_cmd.dev = dev;
1374 desc.its_inv_cmd.event_id = event_id;
1375
1376 its_send_single_vcommand(dev->its, its_build_vinv_cmd, &desc);
1377}
1378
Marc Zyngiered0e4aa2019-11-08 16:58:03 +00001379static void its_send_vint(struct its_device *dev, u32 event_id)
1380{
1381 struct its_cmd_desc desc;
1382
1383 /*
1384 * There is no real VINT command. This is just a normal INT,
1385 * with a VSYNC instead of a SYNC.
1386 */
1387 desc.its_int_cmd.dev = dev;
1388 desc.its_int_cmd.event_id = event_id;
1389
1390 its_send_single_vcommand(dev->its, its_build_vint_cmd, &desc);
1391}
1392
1393static void its_send_vclear(struct its_device *dev, u32 event_id)
1394{
1395 struct its_cmd_desc desc;
1396
1397 /*
1398 * There is no real VCLEAR command. This is just a normal CLEAR,
1399 * with a VSYNC instead of a SYNC.
1400 */
1401 desc.its_clear_cmd.dev = dev;
1402 desc.its_clear_cmd.event_id = event_id;
1403
1404 its_send_single_vcommand(dev->its, its_build_vclear_cmd, &desc);
1405}
1406
Marc Zyngierd97c97b2019-12-24 11:10:33 +00001407static void its_send_invdb(struct its_node *its, struct its_vpe *vpe)
1408{
1409 struct its_cmd_desc desc;
1410
1411 desc.its_invdb_cmd.vpe = vpe;
1412 its_send_single_vcommand(its, its_build_invdb_cmd, &desc);
1413}
1414
Marc Zyngierc48ed512014-11-24 14:35:12 +00001415/*
1416 * irqchip functions - assumes MSI, mostly.
1417 */
Marc Zyngier015ec032016-12-20 09:54:57 +00001418static void lpi_write_config(struct irq_data *d, u8 clr, u8 set)
Marc Zyngierc48ed512014-11-24 14:35:12 +00001419{
Marc Zyngierc1d4d5c2019-11-08 16:58:01 +00001420 struct its_vlpi_map *map = get_vlpi_map(d);
Marc Zyngier015ec032016-12-20 09:54:57 +00001421 irq_hw_number_t hwirq;
Marc Zyngiere1a2e202018-07-27 14:36:00 +01001422 void *va;
Marc Zyngieradcdb942016-12-19 19:18:13 +00001423 u8 *cfg;
Marc Zyngierc48ed512014-11-24 14:35:12 +00001424
Marc Zyngierc1d4d5c2019-11-08 16:58:01 +00001425 if (map) {
1426 va = page_address(map->vm->vprop_page);
Marc Zyngierd4d7b4a2017-10-26 10:44:07 +01001427 hwirq = map->vintid;
1428
1429 /* Remember the updated property */
1430 map->properties &= ~clr;
1431 map->properties |= set | LPI_PROP_GROUP1;
Marc Zyngier015ec032016-12-20 09:54:57 +00001432 } else {
Marc Zyngiere1a2e202018-07-27 14:36:00 +01001433 va = gic_rdists->prop_table_va;
Marc Zyngier015ec032016-12-20 09:54:57 +00001434 hwirq = d->hwirq;
1435 }
Marc Zyngieradcdb942016-12-19 19:18:13 +00001436
Marc Zyngiere1a2e202018-07-27 14:36:00 +01001437 cfg = va + hwirq - 8192;
Marc Zyngieradcdb942016-12-19 19:18:13 +00001438 *cfg &= ~clr;
Marc Zyngier015ec032016-12-20 09:54:57 +00001439 *cfg |= set | LPI_PROP_GROUP1;
Marc Zyngierc48ed512014-11-24 14:35:12 +00001440
1441 /*
1442 * Make the above write visible to the redistributors.
1443 * And yes, we're flushing exactly: One. Single. Byte.
1444 * Humpf...
1445 */
1446 if (gic_rdists->flags & RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING)
Vladimir Murzin328191c2016-11-02 11:54:05 +00001447 gic_flush_dcache_to_poc(cfg, sizeof(*cfg));
Marc Zyngierc48ed512014-11-24 14:35:12 +00001448 else
1449 dsb(ishst);
Marc Zyngier015ec032016-12-20 09:54:57 +00001450}
1451
Marc Zyngier2f4f0642019-11-08 16:57:56 +00001452static void wait_for_syncr(void __iomem *rdbase)
1453{
Heyi Guo04d80db2020-02-25 17:00:23 +08001454 while (readl_relaxed(rdbase + GICR_SYNCR) & 1)
Marc Zyngier2f4f0642019-11-08 16:57:56 +00001455 cpu_relax();
1456}
1457
Marc Zyngier926846a2023-06-17 08:32:42 +01001458static void __direct_lpi_inv(struct irq_data *d, u64 val)
1459{
1460 void __iomem *rdbase;
1461 unsigned long flags;
1462 int cpu;
1463
1464 /* Target the redistributor this LPI is currently routed to */
1465 cpu = irq_to_cpuid_lock(d, &flags);
1466 raw_spin_lock(&gic_data_rdist_cpu(cpu)->rd_lock);
1467
1468 rdbase = per_cpu_ptr(gic_rdists->rdist, cpu)->rd_base;
1469 gic_write_lpir(val, rdbase + GICR_INVLPIR);
1470 wait_for_syncr(rdbase);
1471
1472 raw_spin_unlock(&gic_data_rdist_cpu(cpu)->rd_lock);
1473 irq_to_cpuid_unlock(d, flags);
1474}
1475
Marc Zyngier425c09b2019-11-08 16:57:57 +00001476static void direct_lpi_inv(struct irq_data *d)
1477{
Marc Zyngierf4a81f52019-12-24 11:10:38 +00001478 struct its_vlpi_map *map = get_vlpi_map(d);
Marc Zyngierf4a81f52019-12-24 11:10:38 +00001479 u64 val;
1480
1481 if (map) {
1482 struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1483
1484 WARN_ON(!is_v4_1(its_dev->its));
1485
1486 val = GICR_INVLPIR_V;
1487 val |= FIELD_PREP(GICR_INVLPIR_VPEID, map->vpe->vpe_id);
1488 val |= FIELD_PREP(GICR_INVLPIR_INTID, map->vintid);
1489 } else {
1490 val = d->hwirq;
1491 }
Marc Zyngier425c09b2019-11-08 16:57:57 +00001492
Marc Zyngier926846a2023-06-17 08:32:42 +01001493 __direct_lpi_inv(d, val);
Marc Zyngier425c09b2019-11-08 16:57:57 +00001494}
1495
Marc Zyngier015ec032016-12-20 09:54:57 +00001496static void lpi_update_config(struct irq_data *d, u8 clr, u8 set)
1497{
1498 struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1499
1500 lpi_write_config(d, clr, set);
Marc Zyngierf4a81f52019-12-24 11:10:38 +00001501 if (gic_rdists->has_direct_lpi &&
1502 (is_v4_1(its_dev->its) || !irqd_is_forwarded_to_vcpu(d)))
Marc Zyngier425c09b2019-11-08 16:57:57 +00001503 direct_lpi_inv(d);
Marc Zyngier28614692019-11-08 16:58:02 +00001504 else if (!irqd_is_forwarded_to_vcpu(d))
Marc Zyngier425c09b2019-11-08 16:57:57 +00001505 its_send_inv(its_dev, its_get_event_id(d));
Marc Zyngier28614692019-11-08 16:58:02 +00001506 else
1507 its_send_vinv(its_dev, its_get_event_id(d));
Marc Zyngierc48ed512014-11-24 14:35:12 +00001508}
1509
Marc Zyngier015ec032016-12-20 09:54:57 +00001510static void its_vlpi_set_doorbell(struct irq_data *d, bool enable)
1511{
1512 struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1513 u32 event = its_get_event_id(d);
Marc Zyngierc1d4d5c2019-11-08 16:58:01 +00001514 struct its_vlpi_map *map;
Marc Zyngier015ec032016-12-20 09:54:57 +00001515
Marc Zyngier3858d4d2019-12-24 11:10:37 +00001516 /*
1517 * GICv4.1 does away with the per-LPI nonsense, nothing to do
1518 * here.
1519 */
1520 if (is_v4_1(its_dev->its))
1521 return;
1522
Marc Zyngierc1d4d5c2019-11-08 16:58:01 +00001523 map = dev_event_to_vlpi_map(its_dev, event);
1524
1525 if (map->db_enabled == enable)
Marc Zyngier015ec032016-12-20 09:54:57 +00001526 return;
1527
Marc Zyngierc1d4d5c2019-11-08 16:58:01 +00001528 map->db_enabled = enable;
Marc Zyngier015ec032016-12-20 09:54:57 +00001529
1530 /*
1531 * More fun with the architecture:
1532 *
1533 * Ideally, we'd issue a VMAPTI to set the doorbell to its LPI
1534 * value or to 1023, depending on the enable bit. But that
Ingo Molnara359f752021-03-22 04:21:30 +01001535 * would be issuing a mapping for an /existing/ DevID+EventID
Marc Zyngier015ec032016-12-20 09:54:57 +00001536 * pair, which is UNPREDICTABLE. Instead, let's issue a VMOVI
1537 * to the /same/ vPE, using this opportunity to adjust the
1538 * doorbell. Mouahahahaha. We loves it, Precious.
1539 */
1540 its_send_vmovi(its_dev, event);
Marc Zyngierc48ed512014-11-24 14:35:12 +00001541}
1542
1543static void its_mask_irq(struct irq_data *d)
1544{
Marc Zyngier015ec032016-12-20 09:54:57 +00001545 if (irqd_is_forwarded_to_vcpu(d))
1546 its_vlpi_set_doorbell(d, false);
1547
Marc Zyngieradcdb942016-12-19 19:18:13 +00001548 lpi_update_config(d, LPI_PROP_ENABLED, 0);
Marc Zyngierc48ed512014-11-24 14:35:12 +00001549}
1550
1551static void its_unmask_irq(struct irq_data *d)
1552{
Marc Zyngier015ec032016-12-20 09:54:57 +00001553 if (irqd_is_forwarded_to_vcpu(d))
1554 its_vlpi_set_doorbell(d, true);
1555
Marc Zyngieradcdb942016-12-19 19:18:13 +00001556 lpi_update_config(d, 0, LPI_PROP_ENABLED);
Marc Zyngierc48ed512014-11-24 14:35:12 +00001557}
1558
Marc Zyngier2f13ff12020-05-15 17:57:51 +01001559static __maybe_unused u32 its_read_lpi_count(struct irq_data *d, int cpu)
1560{
1561 if (irqd_affinity_is_managed(d))
1562 return atomic_read(&per_cpu_ptr(&cpu_lpi_count, cpu)->managed);
1563
1564 return atomic_read(&per_cpu_ptr(&cpu_lpi_count, cpu)->unmanaged);
1565}
1566
1567static void its_inc_lpi_count(struct irq_data *d, int cpu)
1568{
1569 if (irqd_affinity_is_managed(d))
1570 atomic_inc(&per_cpu_ptr(&cpu_lpi_count, cpu)->managed);
1571 else
1572 atomic_inc(&per_cpu_ptr(&cpu_lpi_count, cpu)->unmanaged);
1573}
1574
1575static void its_dec_lpi_count(struct irq_data *d, int cpu)
1576{
1577 if (irqd_affinity_is_managed(d))
1578 atomic_dec(&per_cpu_ptr(&cpu_lpi_count, cpu)->managed);
1579 else
1580 atomic_dec(&per_cpu_ptr(&cpu_lpi_count, cpu)->unmanaged);
1581}
1582
Marc Zyngierc5d60822020-05-15 17:57:52 +01001583static unsigned int cpumask_pick_least_loaded(struct irq_data *d,
1584 const struct cpumask *cpu_mask)
1585{
1586 unsigned int cpu = nr_cpu_ids, tmp;
1587 int count = S32_MAX;
1588
1589 for_each_cpu(tmp, cpu_mask) {
1590 int this_count = its_read_lpi_count(d, tmp);
1591 if (this_count < count) {
1592 cpu = tmp;
1593 count = this_count;
1594 }
1595 }
1596
1597 return cpu;
1598}
1599
1600/*
1601 * As suggested by Thomas Gleixner in:
1602 * https://lore.kernel.org/r/87h80q2aoc.fsf@nanos.tec.linutronix.de
1603 */
1604static int its_select_cpu(struct irq_data *d,
1605 const struct cpumask *aff_mask)
1606{
1607 struct its_device *its_dev = irq_data_get_irq_chip_data(d);
Pierre Gondoisf55a9b52022-09-12 16:18:57 +02001608 static DEFINE_RAW_SPINLOCK(tmpmask_lock);
1609 static struct cpumask __tmpmask;
1610 struct cpumask *tmpmask;
1611 unsigned long flags;
Marc Zyngierc5d60822020-05-15 17:57:52 +01001612 int cpu, node;
Marc Zyngierc5d60822020-05-15 17:57:52 +01001613 node = its_dev->its->numa_node;
Pierre Gondoisf55a9b52022-09-12 16:18:57 +02001614 tmpmask = &__tmpmask;
1615
1616 raw_spin_lock_irqsave(&tmpmask_lock, flags);
Marc Zyngierc5d60822020-05-15 17:57:52 +01001617
1618 if (!irqd_affinity_is_managed(d)) {
1619 /* First try the NUMA node */
1620 if (node != NUMA_NO_NODE) {
1621 /*
1622 * Try the intersection of the affinity mask and the
1623 * node mask (and the online mask, just to be safe).
1624 */
1625 cpumask_and(tmpmask, cpumask_of_node(node), aff_mask);
1626 cpumask_and(tmpmask, tmpmask, cpu_online_mask);
1627
1628 /*
1629 * Ideally, we would check if the mask is empty, and
1630 * try again on the full node here.
1631 *
1632 * But it turns out that the way ACPI describes the
1633 * affinity for ITSs only deals about memory, and
1634 * not target CPUs, so it cannot describe a single
1635 * ITS placed next to two NUMA nodes.
1636 *
1637 * Instead, just fallback on the online mask. This
1638 * diverges from Thomas' suggestion above.
1639 */
1640 cpu = cpumask_pick_least_loaded(d, tmpmask);
1641 if (cpu < nr_cpu_ids)
1642 goto out;
1643
1644 /* If we can't cross sockets, give up */
1645 if ((its_dev->its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_23144))
1646 goto out;
1647
1648 /* If the above failed, expand the search */
1649 }
1650
1651 /* Try the intersection of the affinity and online masks */
1652 cpumask_and(tmpmask, aff_mask, cpu_online_mask);
1653
1654 /* If that doesn't fly, the online mask is the last resort */
1655 if (cpumask_empty(tmpmask))
1656 cpumask_copy(tmpmask, cpu_online_mask);
1657
1658 cpu = cpumask_pick_least_loaded(d, tmpmask);
1659 } else {
Marc Zyngier3f893a52022-04-05 19:50:40 +01001660 cpumask_copy(tmpmask, aff_mask);
Marc Zyngierc5d60822020-05-15 17:57:52 +01001661
1662 /* If we cannot cross sockets, limit the search to that node */
1663 if ((its_dev->its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_23144) &&
1664 node != NUMA_NO_NODE)
1665 cpumask_and(tmpmask, tmpmask, cpumask_of_node(node));
1666
1667 cpu = cpumask_pick_least_loaded(d, tmpmask);
1668 }
1669out:
Pierre Gondoisf55a9b52022-09-12 16:18:57 +02001670 raw_spin_unlock_irqrestore(&tmpmask_lock, flags);
Marc Zyngierc5d60822020-05-15 17:57:52 +01001671
1672 pr_debug("IRQ%d -> %*pbl CPU%d\n", d->irq, cpumask_pr_args(aff_mask), cpu);
1673 return cpu;
1674}
1675
Marc Zyngierc48ed512014-11-24 14:35:12 +00001676static int its_set_affinity(struct irq_data *d, const struct cpumask *mask_val,
1677 bool force)
1678{
Marc Zyngierc48ed512014-11-24 14:35:12 +00001679 struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1680 struct its_collection *target_col;
1681 u32 id = its_get_event_id(d);
Marc Zyngierc5d60822020-05-15 17:57:52 +01001682 int cpu, prev_cpu;
Marc Zyngierc48ed512014-11-24 14:35:12 +00001683
Marc Zyngier015ec032016-12-20 09:54:57 +00001684 /* A forwarded interrupt should use irq_set_vcpu_affinity */
1685 if (irqd_is_forwarded_to_vcpu(d))
1686 return -EINVAL;
1687
Marc Zyngier2f13ff12020-05-15 17:57:51 +01001688 prev_cpu = its_dev->event_map.col_map[id];
1689 its_dec_lpi_count(d, prev_cpu);
1690
Marc Zyngierc5d60822020-05-15 17:57:52 +01001691 if (!force)
1692 cpu = its_select_cpu(d, mask_val);
1693 else
1694 cpu = cpumask_pick_least_loaded(d, mask_val);
Ganapatrao Kulkarnifbf8f402016-05-25 15:29:20 +02001695
Marc Zyngierc5d60822020-05-15 17:57:52 +01001696 if (cpu < 0 || cpu >= nr_cpu_ids)
Marc Zyngier2f13ff12020-05-15 17:57:51 +01001697 goto err;
Marc Zyngierc48ed512014-11-24 14:35:12 +00001698
MaJun8b8d94a2017-05-18 16:19:13 +08001699 /* don't set the affinity when the target cpu is same as current one */
Marc Zyngier2f13ff12020-05-15 17:57:51 +01001700 if (cpu != prev_cpu) {
MaJun8b8d94a2017-05-18 16:19:13 +08001701 target_col = &its_dev->its->collections[cpu];
1702 its_send_movi(its_dev, target_col, id);
1703 its_dev->event_map.col_map[id] = cpu;
Marc Zyngier0d224d32017-08-18 09:39:18 +01001704 irq_data_update_effective_affinity(d, cpumask_of(cpu));
MaJun8b8d94a2017-05-18 16:19:13 +08001705 }
Marc Zyngierc48ed512014-11-24 14:35:12 +00001706
Marc Zyngier2f13ff12020-05-15 17:57:51 +01001707 its_inc_lpi_count(d, cpu);
1708
Marc Zyngierc48ed512014-11-24 14:35:12 +00001709 return IRQ_SET_MASK_OK_DONE;
Marc Zyngier2f13ff12020-05-15 17:57:51 +01001710
1711err:
1712 its_inc_lpi_count(d, prev_cpu);
1713 return -EINVAL;
Marc Zyngierc48ed512014-11-24 14:35:12 +00001714}
1715
Ard Biesheuvel558b0162017-10-17 17:55:56 +01001716static u64 its_irq_get_msi_base(struct its_device *its_dev)
1717{
1718 struct its_node *its = its_dev->its;
1719
1720 return its->phys_base + GITS_TRANSLATER;
1721}
1722
Marc Zyngierb48ac832014-11-24 14:35:16 +00001723static void its_irq_compose_msi_msg(struct irq_data *d, struct msi_msg *msg)
1724{
1725 struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1726 struct its_node *its;
1727 u64 addr;
1728
1729 its = its_dev->its;
Ard Biesheuvel558b0162017-10-17 17:55:56 +01001730 addr = its->get_msi_base(its_dev);
Marc Zyngierb48ac832014-11-24 14:35:16 +00001731
Vladimir Murzinb11283e2016-11-02 11:54:03 +00001732 msg->address_lo = lower_32_bits(addr);
1733 msg->address_hi = upper_32_bits(addr);
Marc Zyngierb48ac832014-11-24 14:35:16 +00001734 msg->data = its_get_event_id(d);
Robin Murphy44bb7e22016-09-12 17:13:59 +01001735
Julien Grall35ae7df2019-05-01 14:58:21 +01001736 iommu_dma_compose_msi_msg(irq_data_get_msi_desc(d), msg);
Marc Zyngierb48ac832014-11-24 14:35:16 +00001737}
1738
Marc Zyngier8d85dce2016-12-19 18:02:13 +00001739static int its_irq_set_irqchip_state(struct irq_data *d,
1740 enum irqchip_irq_state which,
1741 bool state)
1742{
1743 struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1744 u32 event = its_get_event_id(d);
1745
1746 if (which != IRQCHIP_STATE_PENDING)
1747 return -EINVAL;
1748
Marc Zyngiered0e4aa2019-11-08 16:58:03 +00001749 if (irqd_is_forwarded_to_vcpu(d)) {
1750 if (state)
1751 its_send_vint(its_dev, event);
1752 else
1753 its_send_vclear(its_dev, event);
1754 } else {
1755 if (state)
1756 its_send_int(its_dev, event);
1757 else
1758 its_send_clear(its_dev, event);
1759 }
Marc Zyngier8d85dce2016-12-19 18:02:13 +00001760
1761 return 0;
1762}
1763
Marc Zyngier5f774f52020-07-31 11:33:13 +01001764static int its_irq_retrigger(struct irq_data *d)
1765{
1766 return !its_irq_set_irqchip_state(d, IRQCHIP_STATE_PENDING, true);
1767}
1768
Marc Zyngier009384b2020-03-04 20:33:23 +00001769/*
1770 * Two favourable cases:
1771 *
1772 * (a) Either we have a GICv4.1, and all vPEs have to be mapped at all times
1773 * for vSGI delivery
1774 *
1775 * (b) Or the ITSs do not use a list map, meaning that VMOVP is cheap enough
1776 * and we're better off mapping all VPEs always
1777 *
1778 * If neither (a) nor (b) is true, then we map vPEs on demand.
1779 *
1780 */
1781static bool gic_requires_eager_mapping(void)
1782{
1783 if (!its_list_map || gic_rdists->has_rvpeid)
1784 return true;
1785
1786 return false;
1787}
1788
Marc Zyngier2247e1b2017-10-08 18:50:36 +01001789static void its_map_vm(struct its_node *its, struct its_vm *vm)
1790{
Marc Zyngier009384b2020-03-04 20:33:23 +00001791 if (gic_requires_eager_mapping())
Marc Zyngier2247e1b2017-10-08 18:50:36 +01001792 return;
1793
Marc Zyngierf0eb1542024-07-05 10:31:54 +01001794 guard(raw_spinlock_irqsave)(&vm->vmapp_lock);
Marc Zyngier2247e1b2017-10-08 18:50:36 +01001795
1796 /*
1797 * If the VM wasn't mapped yet, iterate over the vpes and get
1798 * them mapped now.
1799 */
1800 vm->vlpi_count[its->list_nr]++;
1801
1802 if (vm->vlpi_count[its->list_nr] == 1) {
1803 int i;
1804
1805 for (i = 0; i < vm->nr_vpes; i++) {
1806 struct its_vpe *vpe = vm->vpes[i];
1807
Marc Zyngiera84a07fa2024-07-05 10:31:55 +01001808 scoped_guard(raw_spinlock, &vpe->vpe_lock)
1809 its_send_vmapp(its, vpe, true);
1810
Marc Zyngier2247e1b2017-10-08 18:50:36 +01001811 its_send_vinvall(its, vpe);
1812 }
1813 }
Marc Zyngier2247e1b2017-10-08 18:50:36 +01001814}
1815
1816static void its_unmap_vm(struct its_node *its, struct its_vm *vm)
1817{
Marc Zyngier2247e1b2017-10-08 18:50:36 +01001818 /* Not using the ITS list? Everything is always mapped. */
Marc Zyngier009384b2020-03-04 20:33:23 +00001819 if (gic_requires_eager_mapping())
Marc Zyngier2247e1b2017-10-08 18:50:36 +01001820 return;
1821
Marc Zyngierf0eb1542024-07-05 10:31:54 +01001822 guard(raw_spinlock_irqsave)(&vm->vmapp_lock);
Marc Zyngier2247e1b2017-10-08 18:50:36 +01001823
1824 if (!--vm->vlpi_count[its->list_nr]) {
1825 int i;
1826
Marc Zyngiera84a07fa2024-07-05 10:31:55 +01001827 for (i = 0; i < vm->nr_vpes; i++) {
1828 guard(raw_spinlock)(&vm->vpes[i]->vpe_lock);
Marc Zyngier2247e1b2017-10-08 18:50:36 +01001829 its_send_vmapp(its, vm->vpes[i], false);
Marc Zyngiera84a07fa2024-07-05 10:31:55 +01001830 }
Marc Zyngier2247e1b2017-10-08 18:50:36 +01001831 }
Marc Zyngier2247e1b2017-10-08 18:50:36 +01001832}
1833
Marc Zyngierd011e4e2016-12-20 09:44:41 +00001834static int its_vlpi_map(struct irq_data *d, struct its_cmd_info *info)
1835{
1836 struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1837 u32 event = its_get_event_id(d);
Marc Zyngierd011e4e2016-12-20 09:44:41 +00001838
1839 if (!info->map)
1840 return -EINVAL;
1841
Marc Zyngierd011e4e2016-12-20 09:44:41 +00001842 if (!its_dev->event_map.vm) {
1843 struct its_vlpi_map *maps;
1844
Kees Cook6396bb22018-06-12 14:03:40 -07001845 maps = kcalloc(its_dev->event_map.nr_lpis, sizeof(*maps),
Marc Zyngier11635fa2019-11-08 16:58:05 +00001846 GFP_ATOMIC);
Hagar Hemdanb97e8a22024-05-31 16:21:44 +00001847 if (!maps)
1848 return -ENOMEM;
Marc Zyngierd011e4e2016-12-20 09:44:41 +00001849
1850 its_dev->event_map.vm = info->map->vm;
1851 its_dev->event_map.vlpi_maps = maps;
1852 } else if (its_dev->event_map.vm != info->map->vm) {
Hagar Hemdanb97e8a22024-05-31 16:21:44 +00001853 return -EINVAL;
Marc Zyngierd011e4e2016-12-20 09:44:41 +00001854 }
1855
1856 /* Get our private copy of the mapping information */
1857 its_dev->event_map.vlpi_maps[event] = *info->map;
1858
1859 if (irqd_is_forwarded_to_vcpu(d)) {
1860 /* Already mapped, move it around */
1861 its_send_vmovi(its_dev, event);
1862 } else {
Marc Zyngier2247e1b2017-10-08 18:50:36 +01001863 /* Ensure all the VPEs are mapped on this ITS */
1864 its_map_vm(its_dev->its, info->map->vm);
1865
Marc Zyngierd4d7b4a2017-10-26 10:44:07 +01001866 /*
1867 * Flag the interrupt as forwarded so that we can
1868 * start poking the virtual property table.
1869 */
1870 irqd_set_forwarded_to_vcpu(d);
1871
1872 /* Write out the property to the prop table */
1873 lpi_write_config(d, 0xff, info->map->properties);
1874
Marc Zyngierd011e4e2016-12-20 09:44:41 +00001875 /* Drop the physical mapping */
1876 its_send_discard(its_dev, event);
1877
1878 /* and install the virtual one */
1879 its_send_vmapti(its_dev, event);
Marc Zyngierd011e4e2016-12-20 09:44:41 +00001880
1881 /* Increment the number of VLPIs */
1882 its_dev->event_map.nr_vlpis++;
1883 }
1884
Hagar Hemdanb97e8a22024-05-31 16:21:44 +00001885 return 0;
Marc Zyngierd011e4e2016-12-20 09:44:41 +00001886}
1887
1888static int its_vlpi_get(struct irq_data *d, struct its_cmd_info *info)
1889{
1890 struct its_device *its_dev = irq_data_get_irq_chip_data(d);
Marc Zyngier046b5052019-11-08 16:58:04 +00001891 struct its_vlpi_map *map;
Marc Zyngierd011e4e2016-12-20 09:44:41 +00001892
Marc Zyngier046b5052019-11-08 16:58:04 +00001893 map = get_vlpi_map(d);
1894
Hagar Hemdanb97e8a22024-05-31 16:21:44 +00001895 if (!its_dev->event_map.vm || !map)
1896 return -EINVAL;
Marc Zyngierd011e4e2016-12-20 09:44:41 +00001897
1898 /* Copy our mapping information to the incoming request */
Marc Zyngierc1d4d5c2019-11-08 16:58:01 +00001899 *info->map = *map;
Marc Zyngierd011e4e2016-12-20 09:44:41 +00001900
Hagar Hemdanb97e8a22024-05-31 16:21:44 +00001901 return 0;
Marc Zyngierd011e4e2016-12-20 09:44:41 +00001902}
1903
1904static int its_vlpi_unmap(struct irq_data *d)
1905{
1906 struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1907 u32 event = its_get_event_id(d);
Marc Zyngierd011e4e2016-12-20 09:44:41 +00001908
Hagar Hemdanb97e8a22024-05-31 16:21:44 +00001909 if (!its_dev->event_map.vm || !irqd_is_forwarded_to_vcpu(d))
1910 return -EINVAL;
Marc Zyngierd011e4e2016-12-20 09:44:41 +00001911
1912 /* Drop the virtual mapping */
1913 its_send_discard(its_dev, event);
1914
1915 /* and restore the physical one */
1916 irqd_clr_forwarded_to_vcpu(d);
1917 its_send_mapti(its_dev, d->hwirq, event);
Mark Rutlanda6156e72024-06-17 12:18:39 +01001918 lpi_update_config(d, 0xff, (lpi_prop_prio |
Marc Zyngierd011e4e2016-12-20 09:44:41 +00001919 LPI_PROP_ENABLED |
1920 LPI_PROP_GROUP1));
1921
Marc Zyngier2247e1b2017-10-08 18:50:36 +01001922 /* Potentially unmap the VM from this ITS */
1923 its_unmap_vm(its_dev->its, its_dev->event_map.vm);
1924
Marc Zyngierd011e4e2016-12-20 09:44:41 +00001925 /*
1926 * Drop the refcount and make the device available again if
1927 * this was the last VLPI.
1928 */
1929 if (!--its_dev->event_map.nr_vlpis) {
1930 its_dev->event_map.vm = NULL;
1931 kfree(its_dev->event_map.vlpi_maps);
1932 }
1933
Hagar Hemdanb97e8a22024-05-31 16:21:44 +00001934 return 0;
Marc Zyngierd011e4e2016-12-20 09:44:41 +00001935}
1936
Marc Zyngier015ec032016-12-20 09:54:57 +00001937static int its_vlpi_prop_update(struct irq_data *d, struct its_cmd_info *info)
1938{
1939 struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1940
1941 if (!its_dev->event_map.vm || !irqd_is_forwarded_to_vcpu(d))
1942 return -EINVAL;
1943
1944 if (info->cmd_type == PROP_UPDATE_AND_INV_VLPI)
1945 lpi_update_config(d, 0xff, info->config);
1946 else
1947 lpi_write_config(d, 0xff, info->config);
1948 its_vlpi_set_doorbell(d, !!(info->config & LPI_PROP_ENABLED));
1949
1950 return 0;
1951}
1952
Marc Zyngierc808eea2016-12-20 09:31:20 +00001953static int its_irq_set_vcpu_affinity(struct irq_data *d, void *vcpu_info)
1954{
1955 struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1956 struct its_cmd_info *info = vcpu_info;
1957
1958 /* Need a v4 ITS */
Marc Zyngier0dd57fe2019-11-08 16:57:58 +00001959 if (!is_v4(its_dev->its))
Marc Zyngierc808eea2016-12-20 09:31:20 +00001960 return -EINVAL;
1961
Hagar Hemdanb97e8a22024-05-31 16:21:44 +00001962 guard(raw_spinlock_irq)(&its_dev->event_map.vlpi_lock);
1963
Marc Zyngierd011e4e2016-12-20 09:44:41 +00001964 /* Unmap request? */
1965 if (!info)
1966 return its_vlpi_unmap(d);
1967
Marc Zyngierc808eea2016-12-20 09:31:20 +00001968 switch (info->cmd_type) {
1969 case MAP_VLPI:
Marc Zyngierd011e4e2016-12-20 09:44:41 +00001970 return its_vlpi_map(d, info);
Marc Zyngierc808eea2016-12-20 09:31:20 +00001971
1972 case GET_VLPI:
Marc Zyngierd011e4e2016-12-20 09:44:41 +00001973 return its_vlpi_get(d, info);
Marc Zyngierc808eea2016-12-20 09:31:20 +00001974
1975 case PROP_UPDATE_VLPI:
1976 case PROP_UPDATE_AND_INV_VLPI:
Marc Zyngier015ec032016-12-20 09:54:57 +00001977 return its_vlpi_prop_update(d, info);
Marc Zyngierc808eea2016-12-20 09:31:20 +00001978
1979 default:
1980 return -EINVAL;
1981 }
1982}
1983
Marc Zyngierc48ed512014-11-24 14:35:12 +00001984static struct irq_chip its_irq_chip = {
1985 .name = "ITS",
1986 .irq_mask = its_mask_irq,
1987 .irq_unmask = its_unmask_irq,
Ashok Kumar004fa082016-02-11 05:38:53 -08001988 .irq_eoi = irq_chip_eoi_parent,
Marc Zyngierc48ed512014-11-24 14:35:12 +00001989 .irq_set_affinity = its_set_affinity,
Marc Zyngierb48ac832014-11-24 14:35:16 +00001990 .irq_compose_msi_msg = its_irq_compose_msi_msg,
Marc Zyngier8d85dce2016-12-19 18:02:13 +00001991 .irq_set_irqchip_state = its_irq_set_irqchip_state,
Marc Zyngier5f774f52020-07-31 11:33:13 +01001992 .irq_retrigger = its_irq_retrigger,
Marc Zyngierc808eea2016-12-20 09:31:20 +00001993 .irq_set_vcpu_affinity = its_irq_set_vcpu_affinity,
Marc Zyngierb48ac832014-11-24 14:35:16 +00001994};
1995
Marc Zyngier880cb3c2018-05-27 16:14:15 +01001996
Marc Zyngierbf9529f2014-11-24 14:35:13 +00001997/*
1998 * How we allocate LPIs:
1999 *
Marc Zyngier880cb3c2018-05-27 16:14:15 +01002000 * lpi_range_list contains ranges of LPIs that are to available to
2001 * allocate from. To allocate LPIs, just pick the first range that
2002 * fits the required allocation, and reduce it by the required
2003 * amount. Once empty, remove the range from the list.
Marc Zyngierbf9529f2014-11-24 14:35:13 +00002004 *
Marc Zyngier880cb3c2018-05-27 16:14:15 +01002005 * To free a range of LPIs, add a free range to the list, sort it and
2006 * merge the result if the new range happens to be adjacent to an
2007 * already free block.
2008 *
2009 * The consequence of the above is that allocation is cost is low, but
2010 * freeing is expensive. We assumes that freeing rarely occurs.
2011 */
Jia He4cb205c2018-08-28 12:53:26 +08002012#define ITS_MAX_LPI_NRBITS 16 /* 64K LPIs */
Marc Zyngier880cb3c2018-05-27 16:14:15 +01002013
Marc Zyngier880cb3c2018-05-27 16:14:15 +01002014static DEFINE_MUTEX(lpi_range_lock);
2015static LIST_HEAD(lpi_range_list);
Marc Zyngierbf9529f2014-11-24 14:35:13 +00002016
Marc Zyngier880cb3c2018-05-27 16:14:15 +01002017struct lpi_range {
2018 struct list_head entry;
2019 u32 base_id;
2020 u32 span;
2021};
2022
2023static struct lpi_range *mk_lpi_range(u32 base, u32 span)
Marc Zyngierbf9529f2014-11-24 14:35:13 +00002024{
Marc Zyngier880cb3c2018-05-27 16:14:15 +01002025 struct lpi_range *range;
2026
Rasmus Villemoes1c73fac2019-03-12 18:33:48 +01002027 range = kmalloc(sizeof(*range), GFP_KERNEL);
Marc Zyngier880cb3c2018-05-27 16:14:15 +01002028 if (range) {
Marc Zyngier880cb3c2018-05-27 16:14:15 +01002029 range->base_id = base;
2030 range->span = span;
2031 }
2032
2033 return range;
Marc Zyngierbf9529f2014-11-24 14:35:13 +00002034}
2035
Marc Zyngier880cb3c2018-05-27 16:14:15 +01002036static int alloc_lpi_range(u32 nr_lpis, u32 *base)
2037{
2038 struct lpi_range *range, *tmp;
2039 int err = -ENOSPC;
2040
2041 mutex_lock(&lpi_range_lock);
2042
2043 list_for_each_entry_safe(range, tmp, &lpi_range_list, entry) {
2044 if (range->span >= nr_lpis) {
2045 *base = range->base_id;
2046 range->base_id += nr_lpis;
2047 range->span -= nr_lpis;
2048
2049 if (range->span == 0) {
2050 list_del(&range->entry);
2051 kfree(range);
2052 }
2053
2054 err = 0;
2055 break;
2056 }
2057 }
2058
2059 mutex_unlock(&lpi_range_lock);
2060
2061 pr_debug("ITS: alloc %u:%u\n", *base, nr_lpis);
2062 return err;
2063}
2064
Rasmus Villemoes12eade12019-03-12 18:33:49 +01002065static void merge_lpi_ranges(struct lpi_range *a, struct lpi_range *b)
2066{
2067 if (&a->entry == &lpi_range_list || &b->entry == &lpi_range_list)
2068 return;
2069 if (a->base_id + a->span != b->base_id)
2070 return;
2071 b->base_id = a->base_id;
2072 b->span += a->span;
2073 list_del(&a->entry);
2074 kfree(a);
2075}
2076
Marc Zyngier880cb3c2018-05-27 16:14:15 +01002077static int free_lpi_range(u32 base, u32 nr_lpis)
2078{
Rasmus Villemoes12eade12019-03-12 18:33:49 +01002079 struct lpi_range *new, *old;
Marc Zyngier880cb3c2018-05-27 16:14:15 +01002080
2081 new = mk_lpi_range(base, nr_lpis);
Rasmus Villemoesb31a3832019-03-12 18:33:47 +01002082 if (!new)
2083 return -ENOMEM;
Marc Zyngier880cb3c2018-05-27 16:14:15 +01002084
2085 mutex_lock(&lpi_range_lock);
2086
Rasmus Villemoes12eade12019-03-12 18:33:49 +01002087 list_for_each_entry_reverse(old, &lpi_range_list, entry) {
2088 if (old->base_id < base)
2089 break;
Marc Zyngier880cb3c2018-05-27 16:14:15 +01002090 }
Rasmus Villemoes12eade12019-03-12 18:33:49 +01002091 /*
2092 * old is the last element with ->base_id smaller than base,
2093 * so new goes right after it. If there are no elements with
2094 * ->base_id smaller than base, &old->entry ends up pointing
2095 * at the head of the list, and inserting new it the start of
2096 * the list is the right thing to do in that case as well.
2097 */
2098 list_add(&new->entry, &old->entry);
2099 /*
2100 * Now check if we can merge with the preceding and/or
2101 * following ranges.
2102 */
2103 merge_lpi_ranges(old, new);
2104 merge_lpi_ranges(new, list_next_entry(new, entry));
Marc Zyngier880cb3c2018-05-27 16:14:15 +01002105
Marc Zyngier880cb3c2018-05-27 16:14:15 +01002106 mutex_unlock(&lpi_range_lock);
Rasmus Villemoesb31a3832019-03-12 18:33:47 +01002107 return 0;
Marc Zyngierbf9529f2014-11-24 14:35:13 +00002108}
2109
Tomasz Nowicki04a0e4d2016-01-19 14:11:18 +01002110static int __init its_lpi_init(u32 id_bits)
Marc Zyngierbf9529f2014-11-24 14:35:13 +00002111{
Marc Zyngier880cb3c2018-05-27 16:14:15 +01002112 u32 lpis = (1UL << id_bits) - 8192;
Marc Zyngier12b29052018-05-31 09:01:59 +01002113 u32 numlpis;
Marc Zyngier880cb3c2018-05-27 16:14:15 +01002114 int err;
Marc Zyngierbf9529f2014-11-24 14:35:13 +00002115
Marc Zyngier12b29052018-05-31 09:01:59 +01002116 numlpis = 1UL << GICD_TYPER_NUM_LPIS(gic_rdists->gicd_typer);
2117
2118 if (numlpis > 2 && !WARN_ON(numlpis > lpis)) {
2119 lpis = numlpis;
2120 pr_info("ITS: Using hypervisor restricted LPI range [%u]\n",
2121 lpis);
2122 }
2123
Marc Zyngier880cb3c2018-05-27 16:14:15 +01002124 /*
2125 * Initializing the allocator is just the same as freeing the
2126 * full range of LPIs.
2127 */
2128 err = free_lpi_range(8192, lpis);
2129 pr_debug("ITS: Allocator initialized for %u LPIs\n", lpis);
2130 return err;
Marc Zyngierbf9529f2014-11-24 14:35:13 +00002131}
2132
Marc Zyngier38dd7c42018-05-27 17:03:03 +01002133static unsigned long *its_lpi_alloc(int nr_irqs, u32 *base, int *nr_ids)
Marc Zyngierbf9529f2014-11-24 14:35:13 +00002134{
2135 unsigned long *bitmap = NULL;
Marc Zyngier880cb3c2018-05-27 16:14:15 +01002136 int err = 0;
Marc Zyngierbf9529f2014-11-24 14:35:13 +00002137
2138 do {
Marc Zyngier38dd7c42018-05-27 17:03:03 +01002139 err = alloc_lpi_range(nr_irqs, base);
Marc Zyngier880cb3c2018-05-27 16:14:15 +01002140 if (!err)
Marc Zyngierbf9529f2014-11-24 14:35:13 +00002141 break;
2142
Marc Zyngier38dd7c42018-05-27 17:03:03 +01002143 nr_irqs /= 2;
2144 } while (nr_irqs > 0);
Marc Zyngierbf9529f2014-11-24 14:35:13 +00002145
Marc Zyngier45725e02019-01-29 15:19:23 +00002146 if (!nr_irqs)
2147 err = -ENOSPC;
2148
Marc Zyngier880cb3c2018-05-27 16:14:15 +01002149 if (err)
Marc Zyngierbf9529f2014-11-24 14:35:13 +00002150 goto out;
2151
Andy Shevchenkoff5fe882021-06-18 18:16:54 +03002152 bitmap = bitmap_zalloc(nr_irqs, GFP_ATOMIC);
Marc Zyngierbf9529f2014-11-24 14:35:13 +00002153 if (!bitmap)
2154 goto out;
2155
Marc Zyngier38dd7c42018-05-27 17:03:03 +01002156 *nr_ids = nr_irqs;
Marc Zyngierbf9529f2014-11-24 14:35:13 +00002157
2158out:
Marc Zyngierc8415b92015-10-02 16:44:05 +01002159 if (!bitmap)
2160 *base = *nr_ids = 0;
2161
Marc Zyngierbf9529f2014-11-24 14:35:13 +00002162 return bitmap;
2163}
2164
Marc Zyngier38dd7c42018-05-27 17:03:03 +01002165static void its_lpi_free(unsigned long *bitmap, u32 base, u32 nr_ids)
Marc Zyngierbf9529f2014-11-24 14:35:13 +00002166{
Marc Zyngier880cb3c2018-05-27 16:14:15 +01002167 WARN_ON(free_lpi_range(base, nr_ids));
Andy Shevchenkoff5fe882021-06-18 18:16:54 +03002168 bitmap_free(bitmap);
Marc Zyngierbf9529f2014-11-24 14:35:13 +00002169}
Marc Zyngier1ac19ca2014-11-24 14:35:14 +00002170
Marc Zyngier053be482018-07-27 15:02:27 +01002171static void gic_reset_prop_table(void *va)
2172{
Mark Rutlanda6156e72024-06-17 12:18:39 +01002173 /* Regular IRQ priority, Group-1, disabled */
2174 memset(va, lpi_prop_prio | LPI_PROP_GROUP1, LPI_PROPBASE_SZ);
Marc Zyngier053be482018-07-27 15:02:27 +01002175
2176 /* Make sure the GIC will observe the written configuration */
2177 gic_flush_dcache_to_poc(va, LPI_PROPBASE_SZ);
2178}
2179
Marc Zyngier0e5ccf92016-12-19 18:15:05 +00002180static struct page *its_allocate_prop_table(gfp_t gfp_flags)
2181{
2182 struct page *prop_page;
Marc Zyngier1ac19ca2014-11-24 14:35:14 +00002183
Marc Zyngier0e5ccf92016-12-19 18:15:05 +00002184 prop_page = alloc_pages(gfp_flags, get_order(LPI_PROPBASE_SZ));
2185 if (!prop_page)
2186 return NULL;
2187
Marc Zyngier053be482018-07-27 15:02:27 +01002188 gic_reset_prop_table(page_address(prop_page));
Marc Zyngier0e5ccf92016-12-19 18:15:05 +00002189
2190 return prop_page;
2191}
2192
Marc Zyngier7d75bbb2016-12-20 13:55:54 +00002193static void its_free_prop_table(struct page *prop_page)
2194{
2195 free_pages((unsigned long)page_address(prop_page),
2196 get_order(LPI_PROPBASE_SZ));
2197}
Marc Zyngier1ac19ca2014-11-24 14:35:14 +00002198
Marc Zyngier5e2c9f92018-07-27 16:23:18 +01002199static bool gic_check_reserved_range(phys_addr_t addr, unsigned long size)
2200{
2201 phys_addr_t start, end, addr_end;
2202 u64 i;
2203
2204 /*
2205 * We don't bother checking for a kdump kernel as by
2206 * construction, the LPI tables are out of this kernel's
2207 * memory map.
2208 */
2209 if (is_kdump_kernel())
2210 return true;
2211
2212 addr_end = addr + size - 1;
2213
Mike Rapoport9f3d5ea2020-10-13 16:58:25 -07002214 for_each_reserved_mem_range(i, &start, &end) {
Marc Zyngier5e2c9f92018-07-27 16:23:18 +01002215 if (addr >= start && addr_end <= end)
2216 return true;
2217 }
2218
2219 /* Not found, not a good sign... */
2220 pr_warn("GICv3: Expected reserved range [%pa:%pa], not found\n",
2221 &addr, &addr_end);
2222 add_taint(TAINT_CRAP, LOCKDEP_STILL_OK);
2223 return false;
2224}
2225
Marc Zyngier3fb68fa2018-07-27 16:21:18 +01002226static int gic_reserve_range(phys_addr_t addr, unsigned long size)
2227{
2228 if (efi_enabled(EFI_CONFIG_TABLES))
2229 return efi_mem_reserve_persistent(addr, size);
2230
2231 return 0;
2232}
2233
Marc Zyngier11e37d32018-07-27 13:38:54 +01002234static int __init its_setup_lpi_prop_table(void)
Marc Zyngier1ac19ca2014-11-24 14:35:14 +00002235{
Marc Zyngierc440a9d2018-07-27 15:40:13 +01002236 if (gic_rdists->flags & RDIST_FLAGS_RD_TABLES_PREALLOCATED) {
2237 u64 val;
Marc Zyngier1ac19ca2014-11-24 14:35:14 +00002238
Marc Zyngierc440a9d2018-07-27 15:40:13 +01002239 val = gicr_read_propbaser(gic_data_rdist_rd_base() + GICR_PROPBASER);
2240 lpi_id_bits = (val & GICR_PROPBASER_IDBITS_MASK) + 1;
2241
2242 gic_rdists->prop_table_pa = val & GENMASK_ULL(51, 12);
2243 gic_rdists->prop_table_va = memremap(gic_rdists->prop_table_pa,
2244 LPI_PROPBASE_SZ,
2245 MEMREMAP_WB);
2246 gic_reset_prop_table(gic_rdists->prop_table_va);
2247 } else {
2248 struct page *page;
2249
2250 lpi_id_bits = min_t(u32,
2251 GICD_TYPER_ID_BITS(gic_rdists->gicd_typer),
2252 ITS_MAX_LPI_NRBITS);
2253 page = its_allocate_prop_table(GFP_NOWAIT);
2254 if (!page) {
2255 pr_err("Failed to allocate PROPBASE\n");
2256 return -ENOMEM;
2257 }
2258
2259 gic_rdists->prop_table_pa = page_to_phys(page);
2260 gic_rdists->prop_table_va = page_address(page);
Marc Zyngier3fb68fa2018-07-27 16:21:18 +01002261 WARN_ON(gic_reserve_range(gic_rdists->prop_table_pa,
2262 LPI_PROPBASE_SZ));
Marc Zyngier1ac19ca2014-11-24 14:35:14 +00002263 }
2264
Marc Zyngiere1a2e202018-07-27 14:36:00 +01002265 pr_info("GICv3: using LPI property table @%pa\n",
2266 &gic_rdists->prop_table_pa);
Marc Zyngier1ac19ca2014-11-24 14:35:14 +00002267
Shanker Donthineni6c31e122017-06-22 18:19:14 -05002268 return its_lpi_init(lpi_id_bits);
Marc Zyngier1ac19ca2014-11-24 14:35:14 +00002269}
2270
2271static const char *its_base_type_string[] = {
2272 [GITS_BASER_TYPE_DEVICE] = "Devices",
2273 [GITS_BASER_TYPE_VCPU] = "Virtual CPUs",
Marc Zyngier4f46de92016-12-20 15:50:14 +00002274 [GITS_BASER_TYPE_RESERVED3] = "Reserved (3)",
Marc Zyngier1ac19ca2014-11-24 14:35:14 +00002275 [GITS_BASER_TYPE_COLLECTION] = "Interrupt Collections",
2276 [GITS_BASER_TYPE_RESERVED5] = "Reserved (5)",
2277 [GITS_BASER_TYPE_RESERVED6] = "Reserved (6)",
2278 [GITS_BASER_TYPE_RESERVED7] = "Reserved (7)",
2279};
2280
Shanker Donthineni2d81d422016-06-06 18:17:28 -05002281static u64 its_read_baser(struct its_node *its, struct its_baser *baser)
2282{
2283 u32 idx = baser - its->tables;
2284
Vladimir Murzin0968a612016-11-02 11:54:06 +00002285 return gits_read_baser(its->base + GITS_BASER + (idx << 3));
Shanker Donthineni2d81d422016-06-06 18:17:28 -05002286}
2287
2288static void its_write_baser(struct its_node *its, struct its_baser *baser,
2289 u64 val)
2290{
2291 u32 idx = baser - its->tables;
2292
Vladimir Murzin0968a612016-11-02 11:54:06 +00002293 gits_write_baser(val, its->base + GITS_BASER + (idx << 3));
Shanker Donthineni2d81d422016-06-06 18:17:28 -05002294 baser->val = its_read_baser(its, baser);
2295}
2296
Shanker Donthineni93473592016-06-06 18:17:30 -05002297static int its_setup_baser(struct its_node *its, struct its_baser *baser,
Marc Zyngierd5df9dc2020-03-13 11:01:15 +00002298 u64 cache, u64 shr, u32 order, bool indirect)
Shanker Donthineni93473592016-06-06 18:17:30 -05002299{
2300 u64 val = its_read_baser(its, baser);
2301 u64 esz = GITS_BASER_ENTRY_SIZE(val);
2302 u64 type = GITS_BASER_TYPE(val);
Shanker Donthineni30ae9612017-10-09 11:46:55 -05002303 u64 baser_phys, tmp;
Marc Zyngierd5df9dc2020-03-13 11:01:15 +00002304 u32 alloc_pages, psz;
Shanker Donthineni539d3782019-01-14 09:50:19 +00002305 struct page *page;
Shanker Donthineni93473592016-06-06 18:17:30 -05002306 void *base;
Shanker Donthineni93473592016-06-06 18:17:30 -05002307
Marc Zyngierd5df9dc2020-03-13 11:01:15 +00002308 psz = baser->psz;
Shanker Donthineni93473592016-06-06 18:17:30 -05002309 alloc_pages = (PAGE_ORDER_TO_SIZE(order) / psz);
2310 if (alloc_pages > GITS_BASER_PAGES_MAX) {
2311 pr_warn("ITS@%pa: %s too large, reduce ITS pages %u->%u\n",
2312 &its->phys_base, its_base_type_string[type],
2313 alloc_pages, GITS_BASER_PAGES_MAX);
2314 alloc_pages = GITS_BASER_PAGES_MAX;
2315 order = get_order(GITS_BASER_PAGES_MAX * psz);
2316 }
2317
Shanker Donthineni539d3782019-01-14 09:50:19 +00002318 page = alloc_pages_node(its->numa_node, GFP_KERNEL | __GFP_ZERO, order);
2319 if (!page)
Shanker Donthineni93473592016-06-06 18:17:30 -05002320 return -ENOMEM;
2321
Shanker Donthineni539d3782019-01-14 09:50:19 +00002322 base = (void *)page_address(page);
Shanker Donthineni30ae9612017-10-09 11:46:55 -05002323 baser_phys = virt_to_phys(base);
2324
2325 /* Check if the physical address of the memory is above 48bits */
2326 if (IS_ENABLED(CONFIG_ARM64_64K_PAGES) && (baser_phys >> 48)) {
2327
2328 /* 52bit PA is supported only when PageSize=64K */
2329 if (psz != SZ_64K) {
2330 pr_err("ITS: no 52bit PA support when psz=%d\n", psz);
2331 free_pages((unsigned long)base, order);
2332 return -ENXIO;
2333 }
2334
2335 /* Convert 52bit PA to 48bit field */
2336 baser_phys = GITS_BASER_PHYS_52_to_48(baser_phys);
2337 }
2338
Shanker Donthineni93473592016-06-06 18:17:30 -05002339retry_baser:
Shanker Donthineni30ae9612017-10-09 11:46:55 -05002340 val = (baser_phys |
Shanker Donthineni93473592016-06-06 18:17:30 -05002341 (type << GITS_BASER_TYPE_SHIFT) |
2342 ((esz - 1) << GITS_BASER_ENTRY_SIZE_SHIFT) |
2343 ((alloc_pages - 1) << GITS_BASER_PAGES_SHIFT) |
2344 cache |
2345 shr |
2346 GITS_BASER_VALID);
2347
Shanker Donthineni3faf24e2016-06-06 18:17:32 -05002348 val |= indirect ? GITS_BASER_INDIRECT : 0x0;
2349
Shanker Donthineni93473592016-06-06 18:17:30 -05002350 switch (psz) {
2351 case SZ_4K:
2352 val |= GITS_BASER_PAGE_SIZE_4K;
2353 break;
2354 case SZ_16K:
2355 val |= GITS_BASER_PAGE_SIZE_16K;
2356 break;
2357 case SZ_64K:
2358 val |= GITS_BASER_PAGE_SIZE_64K;
2359 break;
2360 }
2361
Fang Xiangd3badb12023-10-30 16:32:56 +08002362 if (!shr)
2363 gic_flush_dcache_to_poc(base, PAGE_ORDER_TO_SIZE(order));
2364
Shanker Donthineni93473592016-06-06 18:17:30 -05002365 its_write_baser(its, baser, val);
2366 tmp = baser->val;
2367
2368 if ((val ^ tmp) & GITS_BASER_SHAREABILITY_MASK) {
2369 /*
2370 * Shareability didn't stick. Just use
2371 * whatever the read reported, which is likely
2372 * to be the only thing this redistributor
2373 * supports. If that's zero, make it
2374 * non-cacheable as well.
2375 */
2376 shr = tmp & GITS_BASER_SHAREABILITY_MASK;
Fang Xiangd3badb12023-10-30 16:32:56 +08002377 if (!shr)
Shanker Donthineni93473592016-06-06 18:17:30 -05002378 cache = GITS_BASER_nC;
Fang Xiangd3badb12023-10-30 16:32:56 +08002379
Shanker Donthineni93473592016-06-06 18:17:30 -05002380 goto retry_baser;
2381 }
2382
Shanker Donthineni93473592016-06-06 18:17:30 -05002383 if (val != tmp) {
Vladimir Murzinb11283e2016-11-02 11:54:03 +00002384 pr_err("ITS@%pa: %s doesn't stick: %llx %llx\n",
Shanker Donthineni93473592016-06-06 18:17:30 -05002385 &its->phys_base, its_base_type_string[type],
Vladimir Murzinb11283e2016-11-02 11:54:03 +00002386 val, tmp);
Shanker Donthineni93473592016-06-06 18:17:30 -05002387 free_pages((unsigned long)base, order);
2388 return -ENXIO;
2389 }
2390
2391 baser->order = order;
2392 baser->base = base;
2393 baser->psz = psz;
Shanker Donthineni3faf24e2016-06-06 18:17:32 -05002394 tmp = indirect ? GITS_LVL1_ENTRY_SIZE : esz;
Shanker Donthineni93473592016-06-06 18:17:30 -05002395
Shanker Donthineni3faf24e2016-06-06 18:17:32 -05002396 pr_info("ITS@%pa: allocated %d %s @%lx (%s, esz %d, psz %dK, shr %d)\n",
Vladimir Murzind524eaa2016-11-02 11:54:04 +00002397 &its->phys_base, (int)(PAGE_ORDER_TO_SIZE(order) / (int)tmp),
Shanker Donthineni93473592016-06-06 18:17:30 -05002398 its_base_type_string[type],
2399 (unsigned long)virt_to_phys(base),
Shanker Donthineni3faf24e2016-06-06 18:17:32 -05002400 indirect ? "indirect" : "flat", (int)esz,
Shanker Donthineni93473592016-06-06 18:17:30 -05002401 psz / SZ_1K, (int)shr >> GITS_BASER_SHAREABILITY_SHIFT);
2402
2403 return 0;
2404}
2405
Marc Zyngier4cacac52016-12-19 18:18:34 +00002406static bool its_parse_indirect_baser(struct its_node *its,
2407 struct its_baser *baser,
Marc Zyngierd5df9dc2020-03-13 11:01:15 +00002408 u32 *order, u32 ids)
Shanker Donthineni4b75c452016-06-06 18:17:29 -05002409{
Marc Zyngier4cacac52016-12-19 18:18:34 +00002410 u64 tmp = its_read_baser(its, baser);
2411 u64 type = GITS_BASER_TYPE(tmp);
2412 u64 esz = GITS_BASER_ENTRY_SIZE(tmp);
Shanker Donthineni2fd632a2017-01-25 21:51:41 -06002413 u64 val = GITS_BASER_InnerShareable | GITS_BASER_RaWaWb;
Shanker Donthineni4b75c452016-06-06 18:17:29 -05002414 u32 new_order = *order;
Marc Zyngierd5df9dc2020-03-13 11:01:15 +00002415 u32 psz = baser->psz;
Shanker Donthineni3faf24e2016-06-06 18:17:32 -05002416 bool indirect = false;
2417
2418 /* No need to enable Indirection if memory requirement < (psz*2)bytes */
2419 if ((esz << ids) > (psz * 2)) {
2420 /*
2421 * Find out whether hw supports a single or two-level table by
2422 * table by reading bit at offset '62' after writing '1' to it.
2423 */
2424 its_write_baser(its, baser, val | GITS_BASER_INDIRECT);
2425 indirect = !!(baser->val & GITS_BASER_INDIRECT);
2426
2427 if (indirect) {
2428 /*
2429 * The size of the lvl2 table is equal to ITS page size
2430 * which is 'psz'. For computing lvl1 table size,
2431 * subtract ID bits that sparse lvl2 table from 'ids'
2432 * which is reported by ITS hardware times lvl1 table
2433 * entry size.
2434 */
Vladimir Murzind524eaa2016-11-02 11:54:04 +00002435 ids -= ilog2(psz / (int)esz);
Shanker Donthineni3faf24e2016-06-06 18:17:32 -05002436 esz = GITS_LVL1_ENTRY_SIZE;
2437 }
2438 }
Shanker Donthineni4b75c452016-06-06 18:17:29 -05002439
2440 /*
2441 * Allocate as many entries as required to fit the
2442 * range of device IDs that the ITS can grok... The ID
2443 * space being incredibly sparse, this results in a
Shanker Donthineni3faf24e2016-06-06 18:17:32 -05002444 * massive waste of memory if two-level device table
2445 * feature is not supported by hardware.
Shanker Donthineni4b75c452016-06-06 18:17:29 -05002446 */
2447 new_order = max_t(u32, get_order(esz << ids), new_order);
Kirill A. Shutemov5e0a7602023-12-28 17:47:04 +03002448 if (new_order > MAX_PAGE_ORDER) {
2449 new_order = MAX_PAGE_ORDER;
Vladimir Murzind524eaa2016-11-02 11:54:04 +00002450 ids = ilog2(PAGE_ORDER_TO_SIZE(new_order) / (int)esz);
Marc Zyngier576a8342019-11-08 16:58:00 +00002451 pr_warn("ITS@%pa: %s Table too large, reduce ids %llu->%u\n",
Marc Zyngier4cacac52016-12-19 18:18:34 +00002452 &its->phys_base, its_base_type_string[type],
Marc Zyngier576a8342019-11-08 16:58:00 +00002453 device_ids(its), ids);
Shanker Donthineni4b75c452016-06-06 18:17:29 -05002454 }
2455
2456 *order = new_order;
Shanker Donthineni3faf24e2016-06-06 18:17:32 -05002457
2458 return indirect;
Shanker Donthineni4b75c452016-06-06 18:17:29 -05002459}
2460
Marc Zyngier5e516842019-12-24 11:10:28 +00002461static u32 compute_common_aff(u64 val)
2462{
2463 u32 aff, clpiaff;
2464
2465 aff = FIELD_GET(GICR_TYPER_AFFINITY, val);
2466 clpiaff = FIELD_GET(GICR_TYPER_COMMON_LPI_AFF, val);
2467
2468 return aff & ~(GENMASK(31, 0) >> (clpiaff * 8));
2469}
2470
2471static u32 compute_its_aff(struct its_node *its)
2472{
2473 u64 val;
2474 u32 svpet;
2475
2476 /*
2477 * Reencode the ITS SVPET and MPIDR as a GICR_TYPER, and compute
2478 * the resulting affinity. We then use that to see if this match
2479 * our own affinity.
2480 */
2481 svpet = FIELD_GET(GITS_TYPER_SVPET, its->typer);
2482 val = FIELD_PREP(GICR_TYPER_COMMON_LPI_AFF, svpet);
2483 val |= FIELD_PREP(GICR_TYPER_AFFINITY, its->mpidr);
2484 return compute_common_aff(val);
2485}
2486
2487static struct its_node *find_sibling_its(struct its_node *cur_its)
2488{
2489 struct its_node *its;
2490 u32 aff;
2491
2492 if (!FIELD_GET(GITS_TYPER_SVPET, cur_its->typer))
2493 return NULL;
2494
2495 aff = compute_its_aff(cur_its);
2496
2497 list_for_each_entry(its, &its_nodes, entry) {
2498 u64 baser;
2499
2500 if (!is_v4_1(its) || its == cur_its)
2501 continue;
2502
2503 if (!FIELD_GET(GITS_TYPER_SVPET, its->typer))
2504 continue;
2505
2506 if (aff != compute_its_aff(its))
2507 continue;
2508
2509 /* GICv4.1 guarantees that the vPE table is GITS_BASER2 */
2510 baser = its->tables[2].val;
2511 if (!(baser & GITS_BASER_VALID))
2512 continue;
2513
2514 return its;
2515 }
2516
2517 return NULL;
2518}
2519
Marc Zyngier1ac19ca2014-11-24 14:35:14 +00002520static void its_free_tables(struct its_node *its)
2521{
2522 int i;
2523
2524 for (i = 0; i < GITS_BASER_NR_REGS; i++) {
Shanker Donthineni1a485f42016-02-01 20:19:44 -06002525 if (its->tables[i].base) {
2526 free_pages((unsigned long)its->tables[i].base,
2527 its->tables[i].order);
2528 its->tables[i].base = NULL;
Marc Zyngier1ac19ca2014-11-24 14:35:14 +00002529 }
2530 }
2531}
2532
Marc Zyngierd5df9dc2020-03-13 11:01:15 +00002533static int its_probe_baser_psz(struct its_node *its, struct its_baser *baser)
2534{
2535 u64 psz = SZ_64K;
2536
2537 while (psz) {
2538 u64 val, gpsz;
2539
2540 val = its_read_baser(its, baser);
2541 val &= ~GITS_BASER_PAGE_SIZE_MASK;
2542
2543 switch (psz) {
2544 case SZ_64K:
2545 gpsz = GITS_BASER_PAGE_SIZE_64K;
2546 break;
2547 case SZ_16K:
2548 gpsz = GITS_BASER_PAGE_SIZE_16K;
2549 break;
2550 case SZ_4K:
2551 default:
2552 gpsz = GITS_BASER_PAGE_SIZE_4K;
2553 break;
2554 }
2555
2556 gpsz >>= GITS_BASER_PAGE_SIZE_SHIFT;
2557
2558 val |= FIELD_PREP(GITS_BASER_PAGE_SIZE_MASK, gpsz);
2559 its_write_baser(its, baser, val);
2560
2561 if (FIELD_GET(GITS_BASER_PAGE_SIZE_MASK, baser->val) == gpsz)
2562 break;
2563
2564 switch (psz) {
2565 case SZ_64K:
2566 psz = SZ_16K;
2567 break;
2568 case SZ_16K:
2569 psz = SZ_4K;
2570 break;
2571 case SZ_4K:
2572 default:
2573 return -1;
2574 }
2575 }
2576
2577 baser->psz = psz;
2578 return 0;
2579}
2580
Shanker Donthineni0e0b0f62016-06-06 18:17:31 -05002581static int its_alloc_tables(struct its_node *its)
Marc Zyngier1ac19ca2014-11-24 14:35:14 +00002582{
Marc Zyngier1ac19ca2014-11-24 14:35:14 +00002583 u64 shr = GITS_BASER_InnerShareable;
Shanker Donthineni2fd632a2017-01-25 21:51:41 -06002584 u64 cache = GITS_BASER_RaWaWb;
Shanker Donthineni93473592016-06-06 18:17:30 -05002585 int err, i;
Robert Richter94100972015-09-21 22:58:38 +02002586
Ard Biesheuvelfa150012017-10-17 17:55:54 +01002587 if (its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_22375)
2588 /* erratum 24313: ignore memory access type */
2589 cache = GITS_BASER_nCnB;
Shanker Donthineni466b7d12016-03-09 22:10:49 -06002590
Fang Xiangd3badb12023-10-30 16:32:56 +08002591 if (its->flags & ITS_FLAGS_FORCE_NON_SHAREABLE) {
2592 cache = GITS_BASER_nC;
2593 shr = 0;
2594 }
2595
Marc Zyngier1ac19ca2014-11-24 14:35:14 +00002596 for (i = 0; i < GITS_BASER_NR_REGS; i++) {
Shanker Donthineni2d81d422016-06-06 18:17:28 -05002597 struct its_baser *baser = its->tables + i;
2598 u64 val = its_read_baser(its, baser);
Marc Zyngier1ac19ca2014-11-24 14:35:14 +00002599 u64 type = GITS_BASER_TYPE(val);
Shanker Donthineni3faf24e2016-06-06 18:17:32 -05002600 bool indirect = false;
Marc Zyngierd5df9dc2020-03-13 11:01:15 +00002601 u32 order;
Marc Zyngier1ac19ca2014-11-24 14:35:14 +00002602
Marc Zyngierd5df9dc2020-03-13 11:01:15 +00002603 if (type == GITS_BASER_TYPE_NONE)
Marc Zyngier1ac19ca2014-11-24 14:35:14 +00002604 continue;
2605
Marc Zyngierd5df9dc2020-03-13 11:01:15 +00002606 if (its_probe_baser_psz(its, baser)) {
2607 its_free_tables(its);
2608 return -ENXIO;
2609 }
2610
2611 order = get_order(baser->psz);
2612
2613 switch (type) {
Marc Zyngier4cacac52016-12-19 18:18:34 +00002614 case GITS_BASER_TYPE_DEVICE:
Marc Zyngierd5df9dc2020-03-13 11:01:15 +00002615 indirect = its_parse_indirect_baser(its, baser, &order,
Marc Zyngier576a8342019-11-08 16:58:00 +00002616 device_ids(its));
Zenghui Yu8d565742019-02-10 05:24:10 +00002617 break;
2618
Marc Zyngier4cacac52016-12-19 18:18:34 +00002619 case GITS_BASER_TYPE_VCPU:
Marc Zyngier5e516842019-12-24 11:10:28 +00002620 if (is_v4_1(its)) {
2621 struct its_node *sibling;
2622
2623 WARN_ON(i != 2);
2624 if ((sibling = find_sibling_its(its))) {
2625 *baser = sibling->tables[2];
2626 its_write_baser(its, baser, baser->val);
2627 continue;
2628 }
2629 }
2630
Marc Zyngierd5df9dc2020-03-13 11:01:15 +00002631 indirect = its_parse_indirect_baser(its, baser, &order,
Shanker Donthineni32bd44d2017-10-07 15:43:48 -05002632 ITS_MAX_VPEID_BITS);
Marc Zyngier4cacac52016-12-19 18:18:34 +00002633 break;
2634 }
Marc Zyngierf54b97e2015-03-06 16:37:41 +00002635
Marc Zyngierd5df9dc2020-03-13 11:01:15 +00002636 err = its_setup_baser(its, baser, cache, shr, order, indirect);
Shanker Donthineni93473592016-06-06 18:17:30 -05002637 if (err < 0) {
2638 its_free_tables(its);
2639 return err;
Robert Richter30f21362015-09-21 22:58:34 +02002640 }
2641
Shanker Donthineni93473592016-06-06 18:17:30 -05002642 /* Update settings which will be used for next BASERn */
Shanker Donthineni93473592016-06-06 18:17:30 -05002643 cache = baser->val & GITS_BASER_CACHEABILITY_MASK;
2644 shr = baser->val & GITS_BASER_SHAREABILITY_MASK;
Marc Zyngier1ac19ca2014-11-24 14:35:14 +00002645 }
2646
2647 return 0;
Marc Zyngier1ac19ca2014-11-24 14:35:14 +00002648}
2649
Marc Zyngier5e516842019-12-24 11:10:28 +00002650static u64 inherit_vpe_l1_table_from_its(void)
2651{
2652 struct its_node *its;
2653 u64 val;
2654 u32 aff;
2655
2656 val = gic_read_typer(gic_data_rdist_rd_base() + GICR_TYPER);
2657 aff = compute_common_aff(val);
2658
2659 list_for_each_entry(its, &its_nodes, entry) {
2660 u64 baser, addr;
2661
2662 if (!is_v4_1(its))
2663 continue;
2664
2665 if (!FIELD_GET(GITS_TYPER_SVPET, its->typer))
2666 continue;
2667
2668 if (aff != compute_its_aff(its))
2669 continue;
2670
2671 /* GICv4.1 guarantees that the vPE table is GITS_BASER2 */
2672 baser = its->tables[2].val;
2673 if (!(baser & GITS_BASER_VALID))
2674 continue;
2675
2676 /* We have a winner! */
Zenghui Yu8b718d42020-02-06 15:57:07 +08002677 gic_data_rdist()->vpe_l1_base = its->tables[2].base;
2678
Marc Zyngier5e516842019-12-24 11:10:28 +00002679 val = GICR_VPROPBASER_4_1_VALID;
2680 if (baser & GITS_BASER_INDIRECT)
2681 val |= GICR_VPROPBASER_4_1_INDIRECT;
2682 val |= FIELD_PREP(GICR_VPROPBASER_4_1_PAGE_SIZE,
2683 FIELD_GET(GITS_BASER_PAGE_SIZE_MASK, baser));
2684 switch (FIELD_GET(GITS_BASER_PAGE_SIZE_MASK, baser)) {
2685 case GIC_PAGE_SIZE_64K:
2686 addr = GITS_BASER_ADDR_48_to_52(baser);
2687 break;
2688 default:
2689 addr = baser & GENMASK_ULL(47, 12);
2690 break;
2691 }
2692 val |= FIELD_PREP(GICR_VPROPBASER_4_1_ADDR, addr >> 12);
Marc Zyngier846297e2024-02-13 10:12:04 +00002693 if (rdists_support_shareable()) {
2694 val |= FIELD_PREP(GICR_VPROPBASER_SHAREABILITY_MASK,
2695 FIELD_GET(GITS_BASER_SHAREABILITY_MASK, baser));
2696 val |= FIELD_PREP(GICR_VPROPBASER_INNER_CACHEABILITY_MASK,
2697 FIELD_GET(GITS_BASER_INNER_CACHEABILITY_MASK, baser));
2698 }
Marc Zyngier5e516842019-12-24 11:10:28 +00002699 val |= FIELD_PREP(GICR_VPROPBASER_4_1_SIZE, GITS_BASER_NR_PAGES(baser) - 1);
2700
2701 return val;
2702 }
2703
2704 return 0;
2705}
2706
2707static u64 inherit_vpe_l1_table_from_rd(cpumask_t **mask)
2708{
2709 u32 aff;
2710 u64 val;
2711 int cpu;
2712
2713 val = gic_read_typer(gic_data_rdist_rd_base() + GICR_TYPER);
2714 aff = compute_common_aff(val);
2715
2716 for_each_possible_cpu(cpu) {
2717 void __iomem *base = gic_data_rdist_cpu(cpu)->rd_base;
Marc Zyngier5e516842019-12-24 11:10:28 +00002718
2719 if (!base || cpu == smp_processor_id())
2720 continue;
2721
2722 val = gic_read_typer(base + GICR_TYPER);
Zenghui Yu4bccf1d2020-02-06 15:57:09 +08002723 if (aff != compute_common_aff(val))
Marc Zyngier5e516842019-12-24 11:10:28 +00002724 continue;
2725
2726 /*
2727 * At this point, we have a victim. This particular CPU
2728 * has already booted, and has an affinity that matches
2729 * ours wrt CommonLPIAff. Let's use its own VPROPBASER.
2730 * Make sure we don't write the Z bit in that case.
2731 */
Zenghui Yu5186a6c2020-02-06 15:57:11 +08002732 val = gicr_read_vpropbaser(base + SZ_128K + GICR_VPROPBASER);
Marc Zyngier5e516842019-12-24 11:10:28 +00002733 val &= ~GICR_VPROPBASER_4_1_Z;
2734
Zenghui Yu8b718d42020-02-06 15:57:07 +08002735 gic_data_rdist()->vpe_l1_base = gic_data_rdist_cpu(cpu)->vpe_l1_base;
Marc Zyngier5e516842019-12-24 11:10:28 +00002736 *mask = gic_data_rdist_cpu(cpu)->vpe_table_mask;
2737
2738 return val;
2739 }
2740
2741 return 0;
2742}
2743
Zenghui Yu4e6437f2020-02-06 15:57:08 +08002744static bool allocate_vpe_l2_table(int cpu, u32 id)
2745{
2746 void __iomem *base = gic_data_rdist_cpu(cpu)->rd_base;
Marc Zyngier490d3322020-02-09 22:48:50 +00002747 unsigned int psz, esz, idx, npg, gpsz;
2748 u64 val;
Zenghui Yu4e6437f2020-02-06 15:57:08 +08002749 struct page *page;
2750 __le64 *table;
2751
2752 if (!gic_rdists->has_rvpeid)
2753 return true;
2754
Marc Zyngier28d160d2020-03-04 20:33:09 +00002755 /* Skip non-present CPUs */
2756 if (!base)
2757 return true;
2758
Zenghui Yu5186a6c2020-02-06 15:57:11 +08002759 val = gicr_read_vpropbaser(base + SZ_128K + GICR_VPROPBASER);
Zenghui Yu4e6437f2020-02-06 15:57:08 +08002760
2761 esz = FIELD_GET(GICR_VPROPBASER_4_1_ENTRY_SIZE, val) + 1;
2762 gpsz = FIELD_GET(GICR_VPROPBASER_4_1_PAGE_SIZE, val);
2763 npg = FIELD_GET(GICR_VPROPBASER_4_1_SIZE, val) + 1;
2764
2765 switch (gpsz) {
2766 default:
2767 WARN_ON(1);
Gustavo A. R. Silvadf561f662020-08-23 17:36:59 -05002768 fallthrough;
Zenghui Yu4e6437f2020-02-06 15:57:08 +08002769 case GIC_PAGE_SIZE_4K:
2770 psz = SZ_4K;
2771 break;
2772 case GIC_PAGE_SIZE_16K:
2773 psz = SZ_16K;
2774 break;
2775 case GIC_PAGE_SIZE_64K:
2776 psz = SZ_64K;
2777 break;
2778 }
2779
2780 /* Don't allow vpe_id that exceeds single, flat table limit */
2781 if (!(val & GICR_VPROPBASER_4_1_INDIRECT))
2782 return (id < (npg * psz / (esz * SZ_8)));
2783
2784 /* Compute 1st level table index & check if that exceeds table limit */
2785 idx = id >> ilog2(psz / (esz * SZ_8));
2786 if (idx >= (npg * psz / GITS_LVL1_ENTRY_SIZE))
2787 return false;
2788
2789 table = gic_data_rdist_cpu(cpu)->vpe_l1_base;
2790
2791 /* Allocate memory for 2nd level table */
2792 if (!table[idx]) {
2793 page = alloc_pages(GFP_KERNEL | __GFP_ZERO, get_order(psz));
2794 if (!page)
2795 return false;
2796
2797 /* Flush Lvl2 table to PoC if hw doesn't support coherency */
2798 if (!(val & GICR_VPROPBASER_SHAREABILITY_MASK))
2799 gic_flush_dcache_to_poc(page_address(page), psz);
2800
2801 table[idx] = cpu_to_le64(page_to_phys(page) | GITS_BASER_VALID);
2802
2803 /* Flush Lvl1 entry to PoC if hw doesn't support coherency */
2804 if (!(val & GICR_VPROPBASER_SHAREABILITY_MASK))
2805 gic_flush_dcache_to_poc(table + idx, GITS_LVL1_ENTRY_SIZE);
2806
2807 /* Ensure updated table contents are visible to RD hardware */
2808 dsb(sy);
2809 }
2810
2811 return true;
2812}
2813
Marc Zyngier5e516842019-12-24 11:10:28 +00002814static int allocate_vpe_l1_table(void)
2815{
2816 void __iomem *vlpi_base = gic_data_rdist_vlpi_base();
2817 u64 val, gpsz, npg, pa;
2818 unsigned int psz = SZ_64K;
2819 unsigned int np, epp, esz;
2820 struct page *page;
2821
2822 if (!gic_rdists->has_rvpeid)
2823 return 0;
2824
2825 /*
2826 * if VPENDBASER.Valid is set, disable any previously programmed
2827 * VPE by setting PendingLast while clearing Valid. This has the
2828 * effect of making sure no doorbell will be generated and we can
2829 * then safely clear VPROPBASER.Valid.
2830 */
Zenghui Yu5186a6c2020-02-06 15:57:11 +08002831 if (gicr_read_vpendbaser(vlpi_base + GICR_VPENDBASER) & GICR_VPENDBASER_Valid)
2832 gicr_write_vpendbaser(GICR_VPENDBASER_PendingLast,
Marc Zyngier5e516842019-12-24 11:10:28 +00002833 vlpi_base + GICR_VPENDBASER);
2834
2835 /*
2836 * If we can inherit the configuration from another RD, let's do
2837 * so. Otherwise, we have to go through the allocation process. We
2838 * assume that all RDs have the exact same requirements, as
2839 * nothing will work otherwise.
2840 */
2841 val = inherit_vpe_l1_table_from_rd(&gic_data_rdist()->vpe_table_mask);
2842 if (val & GICR_VPROPBASER_4_1_VALID)
2843 goto out;
2844
Zenghui Yud1bd7e02020-06-30 21:37:46 +08002845 gic_data_rdist()->vpe_table_mask = kzalloc(sizeof(cpumask_t), GFP_ATOMIC);
Marc Zyngier5e516842019-12-24 11:10:28 +00002846 if (!gic_data_rdist()->vpe_table_mask)
2847 return -ENOMEM;
2848
2849 val = inherit_vpe_l1_table_from_its();
2850 if (val & GICR_VPROPBASER_4_1_VALID)
2851 goto out;
2852
2853 /* First probe the page size */
2854 val = FIELD_PREP(GICR_VPROPBASER_4_1_PAGE_SIZE, GIC_PAGE_SIZE_64K);
Zenghui Yu5186a6c2020-02-06 15:57:11 +08002855 gicr_write_vpropbaser(val, vlpi_base + GICR_VPROPBASER);
2856 val = gicr_read_vpropbaser(vlpi_base + GICR_VPROPBASER);
Marc Zyngier5e516842019-12-24 11:10:28 +00002857 gpsz = FIELD_GET(GICR_VPROPBASER_4_1_PAGE_SIZE, val);
2858 esz = FIELD_GET(GICR_VPROPBASER_4_1_ENTRY_SIZE, val);
2859
2860 switch (gpsz) {
2861 default:
2862 gpsz = GIC_PAGE_SIZE_4K;
Gustavo A. R. Silvadf561f662020-08-23 17:36:59 -05002863 fallthrough;
Marc Zyngier5e516842019-12-24 11:10:28 +00002864 case GIC_PAGE_SIZE_4K:
2865 psz = SZ_4K;
2866 break;
2867 case GIC_PAGE_SIZE_16K:
2868 psz = SZ_16K;
2869 break;
2870 case GIC_PAGE_SIZE_64K:
2871 psz = SZ_64K;
2872 break;
2873 }
2874
2875 /*
2876 * Start populating the register from scratch, including RO fields
2877 * (which we want to print in debug cases...)
2878 */
2879 val = 0;
2880 val |= FIELD_PREP(GICR_VPROPBASER_4_1_PAGE_SIZE, gpsz);
2881 val |= FIELD_PREP(GICR_VPROPBASER_4_1_ENTRY_SIZE, esz);
2882
2883 /* How many entries per GIC page? */
2884 esz++;
2885 epp = psz / (esz * SZ_8);
2886
2887 /*
2888 * If we need more than just a single L1 page, flag the table
2889 * as indirect and compute the number of required L1 pages.
2890 */
2891 if (epp < ITS_MAX_VPEID) {
2892 int nl2;
2893
2894 val |= GICR_VPROPBASER_4_1_INDIRECT;
2895
2896 /* Number of L2 pages required to cover the VPEID space */
2897 nl2 = DIV_ROUND_UP(ITS_MAX_VPEID, epp);
2898
2899 /* Number of L1 pages to point to the L2 pages */
2900 npg = DIV_ROUND_UP(nl2 * SZ_8, psz);
2901 } else {
2902 npg = 1;
2903 }
2904
Zenghui Yue88bd312020-02-06 15:57:06 +08002905 val |= FIELD_PREP(GICR_VPROPBASER_4_1_SIZE, npg - 1);
Marc Zyngier5e516842019-12-24 11:10:28 +00002906
2907 /* Right, that's the number of CPU pages we need for L1 */
2908 np = DIV_ROUND_UP(npg * psz, PAGE_SIZE);
2909
2910 pr_debug("np = %d, npg = %lld, psz = %d, epp = %d, esz = %d\n",
2911 np, npg, psz, epp, esz);
Zenghui Yud1bd7e02020-06-30 21:37:46 +08002912 page = alloc_pages(GFP_ATOMIC | __GFP_ZERO, get_order(np * PAGE_SIZE));
Marc Zyngier5e516842019-12-24 11:10:28 +00002913 if (!page)
2914 return -ENOMEM;
2915
Zenghui Yu8b718d42020-02-06 15:57:07 +08002916 gic_data_rdist()->vpe_l1_base = page_address(page);
Marc Zyngier5e516842019-12-24 11:10:28 +00002917 pa = virt_to_phys(page_address(page));
2918 WARN_ON(!IS_ALIGNED(pa, psz));
2919
2920 val |= FIELD_PREP(GICR_VPROPBASER_4_1_ADDR, pa >> 12);
Marc Zyngier846297e2024-02-13 10:12:04 +00002921 if (rdists_support_shareable()) {
2922 val |= GICR_VPROPBASER_RaWb;
2923 val |= GICR_VPROPBASER_InnerShareable;
2924 }
Marc Zyngier5e516842019-12-24 11:10:28 +00002925 val |= GICR_VPROPBASER_4_1_Z;
2926 val |= GICR_VPROPBASER_4_1_VALID;
2927
2928out:
Zenghui Yu5186a6c2020-02-06 15:57:11 +08002929 gicr_write_vpropbaser(val, vlpi_base + GICR_VPROPBASER);
Marc Zyngier5e516842019-12-24 11:10:28 +00002930 cpumask_set_cpu(smp_processor_id(), gic_data_rdist()->vpe_table_mask);
2931
2932 pr_debug("CPU%d: VPROPBASER = %llx %*pbl\n",
2933 smp_processor_id(), val,
2934 cpumask_pr_args(gic_data_rdist()->vpe_table_mask));
2935
2936 return 0;
2937}
2938
Marc Zyngier1ac19ca2014-11-24 14:35:14 +00002939static int its_alloc_collections(struct its_node *its)
2940{
Marc Zyngier83559b42018-06-22 10:52:52 +01002941 int i;
2942
Kees Cook6396bb22018-06-12 14:03:40 -07002943 its->collections = kcalloc(nr_cpu_ids, sizeof(*its->collections),
Marc Zyngier1ac19ca2014-11-24 14:35:14 +00002944 GFP_KERNEL);
2945 if (!its->collections)
2946 return -ENOMEM;
2947
Marc Zyngier83559b42018-06-22 10:52:52 +01002948 for (i = 0; i < nr_cpu_ids; i++)
2949 its->collections[i].target_address = ~0ULL;
2950
Marc Zyngier1ac19ca2014-11-24 14:35:14 +00002951 return 0;
2952}
2953
Marc Zyngier7c297a22016-12-19 18:34:38 +00002954static struct page *its_allocate_pending_table(gfp_t gfp_flags)
2955{
2956 struct page *pend_page;
Marc Zyngieradaab502018-07-17 18:06:39 +01002957
Marc Zyngier7c297a22016-12-19 18:34:38 +00002958 pend_page = alloc_pages(gfp_flags | __GFP_ZERO,
Marc Zyngieradaab502018-07-17 18:06:39 +01002959 get_order(LPI_PENDBASE_SZ));
Marc Zyngier7c297a22016-12-19 18:34:38 +00002960 if (!pend_page)
2961 return NULL;
2962
2963 /* Make sure the GIC will observe the zero-ed page */
2964 gic_flush_dcache_to_poc(page_address(pend_page), LPI_PENDBASE_SZ);
2965
2966 return pend_page;
2967}
2968
Marc Zyngier7d75bbb2016-12-20 13:55:54 +00002969static void its_free_pending_table(struct page *pt)
2970{
Marc Zyngieradaab502018-07-17 18:06:39 +01002971 free_pages((unsigned long)page_address(pt), get_order(LPI_PENDBASE_SZ));
Marc Zyngier7d75bbb2016-12-20 13:55:54 +00002972}
2973
Marc Zyngierc6e2ccb2018-06-26 11:21:11 +01002974/*
Marc Zyngier5e2c9f92018-07-27 16:23:18 +01002975 * Booting with kdump and LPIs enabled is generally fine. Any other
2976 * case is wrong in the absence of firmware/EFI support.
Marc Zyngierc6e2ccb2018-06-26 11:21:11 +01002977 */
Marc Zyngierc440a9d2018-07-27 15:40:13 +01002978static bool enabled_lpis_allowed(void)
2979{
Marc Zyngier5e2c9f92018-07-27 16:23:18 +01002980 phys_addr_t addr;
2981 u64 val;
Marc Zyngierc6e2ccb2018-06-26 11:21:11 +01002982
Marc Zyngier5e2c9f92018-07-27 16:23:18 +01002983 /* Check whether the property table is in a reserved region */
2984 val = gicr_read_propbaser(gic_data_rdist_rd_base() + GICR_PROPBASER);
2985 addr = val & GENMASK_ULL(51, 12);
2986
2987 return gic_check_reserved_range(addr, LPI_PROPBASE_SZ);
Marc Zyngierc440a9d2018-07-27 15:40:13 +01002988}
2989
Marc Zyngier11e37d32018-07-27 13:38:54 +01002990static int __init allocate_lpi_tables(void)
2991{
Marc Zyngierc440a9d2018-07-27 15:40:13 +01002992 u64 val;
Marc Zyngier11e37d32018-07-27 13:38:54 +01002993 int err, cpu;
2994
Marc Zyngierc440a9d2018-07-27 15:40:13 +01002995 /*
2996 * If LPIs are enabled while we run this from the boot CPU,
2997 * flag the RD tables as pre-allocated if the stars do align.
2998 */
2999 val = readl_relaxed(gic_data_rdist_rd_base() + GICR_CTLR);
3000 if ((val & GICR_CTLR_ENABLE_LPIS) && enabled_lpis_allowed()) {
3001 gic_rdists->flags |= (RDIST_FLAGS_RD_TABLES_PREALLOCATED |
3002 RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING);
3003 pr_info("GICv3: Using preallocated redistributor tables\n");
3004 }
3005
Marc Zyngier11e37d32018-07-27 13:38:54 +01003006 err = its_setup_lpi_prop_table();
3007 if (err)
3008 return err;
3009
3010 /*
3011 * We allocate all the pending tables anyway, as we may have a
3012 * mix of RDs that have had LPIs enabled, and some that
3013 * don't. We'll free the unused ones as each CPU comes online.
3014 */
3015 for_each_possible_cpu(cpu) {
3016 struct page *pend_page;
3017
3018 pend_page = its_allocate_pending_table(GFP_NOWAIT);
3019 if (!pend_page) {
3020 pr_err("Failed to allocate PENDBASE for CPU%d\n", cpu);
3021 return -ENOMEM;
3022 }
3023
3024 gic_data_rdist_cpu(cpu)->pend_page = pend_page;
3025 }
3026
3027 return 0;
3028}
3029
Marc Zyngieraf27e412022-03-17 09:49:02 +00003030static u64 read_vpend_dirty_clear(void __iomem *vlpi_base)
Heyi Guo64794502019-01-24 21:37:08 +08003031{
3032 u32 count = 1000000; /* 1s! */
3033 bool clean;
3034 u64 val;
3035
Heyi Guo64794502019-01-24 21:37:08 +08003036 do {
Zenghui Yu5186a6c2020-02-06 15:57:11 +08003037 val = gicr_read_vpendbaser(vlpi_base + GICR_VPENDBASER);
Heyi Guo64794502019-01-24 21:37:08 +08003038 clean = !(val & GICR_VPENDBASER_Dirty);
3039 if (!clean) {
3040 count--;
3041 cpu_relax();
3042 udelay(1);
3043 }
3044 } while (!clean && count);
3045
Marc Zyngieraf27e412022-03-17 09:49:02 +00003046 if (unlikely(!clean))
Marc Zyngiere64fab12019-12-24 11:10:35 +00003047 pr_err_ratelimited("ITS virtual pending table not cleaning\n");
Marc Zyngieraf27e412022-03-17 09:49:02 +00003048
3049 return val;
3050}
3051
3052static u64 its_clear_vpend_valid(void __iomem *vlpi_base, u64 clr, u64 set)
3053{
3054 u64 val;
3055
3056 /* Make sure we wait until the RD is done with the initial scan */
3057 val = read_vpend_dirty_clear(vlpi_base);
3058 val &= ~GICR_VPENDBASER_Valid;
3059 val &= ~clr;
3060 val |= set;
3061 gicr_write_vpendbaser(val, vlpi_base + GICR_VPENDBASER);
3062
3063 val = read_vpend_dirty_clear(vlpi_base);
3064 if (unlikely(val & GICR_VPENDBASER_Dirty))
Marc Zyngiere64fab12019-12-24 11:10:35 +00003065 val |= GICR_VPENDBASER_PendingLast;
Marc Zyngiere64fab12019-12-24 11:10:35 +00003066
Heyi Guo64794502019-01-24 21:37:08 +08003067 return val;
3068}
3069
Marc Zyngier1ac19ca2014-11-24 14:35:14 +00003070static void its_cpu_init_lpis(void)
3071{
3072 void __iomem *rbase = gic_data_rdist_rd_base();
3073 struct page *pend_page;
Marc Zyngier11e37d32018-07-27 13:38:54 +01003074 phys_addr_t paddr;
Marc Zyngier1ac19ca2014-11-24 14:35:14 +00003075 u64 val, tmp;
3076
Valentin Schneiderc0cdc8902021-10-27 16:15:04 +01003077 if (gic_data_rdist()->flags & RD_LOCAL_LPI_ENABLED)
Marc Zyngier11e37d32018-07-27 13:38:54 +01003078 return;
3079
Marc Zyngierc440a9d2018-07-27 15:40:13 +01003080 val = readl_relaxed(rbase + GICR_CTLR);
3081 if ((gic_rdists->flags & RDIST_FLAGS_RD_TABLES_PREALLOCATED) &&
3082 (val & GICR_CTLR_ENABLE_LPIS)) {
Marc Zyngierf842ca82018-07-27 16:03:31 +01003083 /*
3084 * Check that we get the same property table on all
3085 * RDs. If we don't, this is hopeless.
3086 */
3087 paddr = gicr_read_propbaser(rbase + GICR_PROPBASER);
3088 paddr &= GENMASK_ULL(51, 12);
3089 if (WARN_ON(gic_rdists->prop_table_pa != paddr))
3090 add_taint(TAINT_CRAP, LOCKDEP_STILL_OK);
3091
Marc Zyngierc440a9d2018-07-27 15:40:13 +01003092 paddr = gicr_read_pendbaser(rbase + GICR_PENDBASER);
3093 paddr &= GENMASK_ULL(51, 16);
3094
Marc Zyngier5e2c9f92018-07-27 16:23:18 +01003095 WARN_ON(!gic_check_reserved_range(paddr, LPI_PENDBASE_SZ));
Valentin Schneiderd23bc2b2021-10-27 16:15:05 +01003096 gic_data_rdist()->flags |= RD_LOCAL_PENDTABLE_PREALLOCATED;
Marc Zyngierc440a9d2018-07-27 15:40:13 +01003097
3098 goto out;
3099 }
3100
Marc Zyngier1ac19ca2014-11-24 14:35:14 +00003101 pend_page = gic_data_rdist()->pend_page;
Marc Zyngier11e37d32018-07-27 13:38:54 +01003102 paddr = page_to_phys(pend_page);
Marc Zyngier1ac19ca2014-11-24 14:35:14 +00003103
Marc Zyngier1ac19ca2014-11-24 14:35:14 +00003104 /* set PROPBASE */
Marc Zyngiere1a2e202018-07-27 14:36:00 +01003105 val = (gic_rdists->prop_table_pa |
Marc Zyngier1ac19ca2014-11-24 14:35:14 +00003106 GICR_PROPBASER_InnerShareable |
Shanker Donthineni2fd632a2017-01-25 21:51:41 -06003107 GICR_PROPBASER_RaWaWb |
Marc Zyngier1ac19ca2014-11-24 14:35:14 +00003108 ((LPI_NRBITS - 1) & GICR_PROPBASER_IDBITS_MASK));
3109
Vladimir Murzin0968a612016-11-02 11:54:06 +00003110 gicr_write_propbaser(val, rbase + GICR_PROPBASER);
3111 tmp = gicr_read_propbaser(rbase + GICR_PROPBASER);
Marc Zyngier1ac19ca2014-11-24 14:35:14 +00003112
Marc Zyngier846297e2024-02-13 10:12:04 +00003113 if (!rdists_support_shareable())
Sebastian Reichela8707f52023-04-18 16:21:08 +02003114 tmp &= ~GICR_PROPBASER_SHAREABILITY_MASK;
3115
Marc Zyngier1ac19ca2014-11-24 14:35:14 +00003116 if ((tmp ^ val) & GICR_PROPBASER_SHAREABILITY_MASK) {
Marc Zyngier241a3862015-03-27 14:15:05 +00003117 if (!(tmp & GICR_PROPBASER_SHAREABILITY_MASK)) {
3118 /*
3119 * The HW reports non-shareable, we must
3120 * remove the cacheability attributes as
3121 * well.
3122 */
3123 val &= ~(GICR_PROPBASER_SHAREABILITY_MASK |
3124 GICR_PROPBASER_CACHEABILITY_MASK);
3125 val |= GICR_PROPBASER_nC;
Vladimir Murzin0968a612016-11-02 11:54:06 +00003126 gicr_write_propbaser(val, rbase + GICR_PROPBASER);
Marc Zyngier241a3862015-03-27 14:15:05 +00003127 }
Marc Zyngier1ac19ca2014-11-24 14:35:14 +00003128 pr_info_once("GIC: using cache flushing for LPI property table\n");
3129 gic_rdists->flags |= RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING;
3130 }
3131
3132 /* set PENDBASE */
3133 val = (page_to_phys(pend_page) |
Marc Zyngier4ad3e362015-03-27 14:15:04 +00003134 GICR_PENDBASER_InnerShareable |
Shanker Donthineni2fd632a2017-01-25 21:51:41 -06003135 GICR_PENDBASER_RaWaWb);
Marc Zyngier1ac19ca2014-11-24 14:35:14 +00003136
Vladimir Murzin0968a612016-11-02 11:54:06 +00003137 gicr_write_pendbaser(val, rbase + GICR_PENDBASER);
3138 tmp = gicr_read_pendbaser(rbase + GICR_PENDBASER);
Marc Zyngier241a3862015-03-27 14:15:05 +00003139
Marc Zyngier846297e2024-02-13 10:12:04 +00003140 if (!rdists_support_shareable())
Sebastian Reichela8707f52023-04-18 16:21:08 +02003141 tmp &= ~GICR_PENDBASER_SHAREABILITY_MASK;
3142
Marc Zyngier241a3862015-03-27 14:15:05 +00003143 if (!(tmp & GICR_PENDBASER_SHAREABILITY_MASK)) {
3144 /*
3145 * The HW reports non-shareable, we must remove the
3146 * cacheability attributes as well.
3147 */
3148 val &= ~(GICR_PENDBASER_SHAREABILITY_MASK |
3149 GICR_PENDBASER_CACHEABILITY_MASK);
3150 val |= GICR_PENDBASER_nC;
Vladimir Murzin0968a612016-11-02 11:54:06 +00003151 gicr_write_pendbaser(val, rbase + GICR_PENDBASER);
Marc Zyngier241a3862015-03-27 14:15:05 +00003152 }
Marc Zyngier1ac19ca2014-11-24 14:35:14 +00003153
3154 /* Enable LPIs */
3155 val = readl_relaxed(rbase + GICR_CTLR);
3156 val |= GICR_CTLR_ENABLE_LPIS;
3157 writel_relaxed(val, rbase + GICR_CTLR);
3158
Oliver Uptonec4308e2024-02-19 18:58:06 +00003159out:
Marc Zyngier5e516842019-12-24 11:10:28 +00003160 if (gic_rdists->has_vlpis && !gic_rdists->has_rvpeid) {
Heyi Guo64794502019-01-24 21:37:08 +08003161 void __iomem *vlpi_base = gic_data_rdist_vlpi_base();
3162
3163 /*
3164 * It's possible for CPU to receive VLPIs before it is
Ingo Molnara359f752021-03-22 04:21:30 +01003165 * scheduled as a vPE, especially for the first CPU, and the
Heyi Guo64794502019-01-24 21:37:08 +08003166 * VLPI with INTID larger than 2^(IDbits+1) will be considered
3167 * as out of range and dropped by GIC.
3168 * So we initialize IDbits to known value to avoid VLPI drop.
3169 */
3170 val = (LPI_NRBITS - 1) & GICR_VPROPBASER_IDBITS_MASK;
3171 pr_debug("GICv4: CPU%d: Init IDbits to 0x%llx for GICR_VPROPBASER\n",
3172 smp_processor_id(), val);
Zenghui Yu5186a6c2020-02-06 15:57:11 +08003173 gicr_write_vpropbaser(val, vlpi_base + GICR_VPROPBASER);
Heyi Guo64794502019-01-24 21:37:08 +08003174
3175 /*
3176 * Also clear Valid bit of GICR_VPENDBASER, in case some
3177 * ancient programming gets left in and has possibility of
3178 * corrupting memory.
3179 */
Marc Zyngiere64fab12019-12-24 11:10:35 +00003180 val = its_clear_vpend_valid(vlpi_base, 0, 0);
Heyi Guo64794502019-01-24 21:37:08 +08003181 }
3182
Marc Zyngier5e516842019-12-24 11:10:28 +00003183 if (allocate_vpe_l1_table()) {
3184 /*
3185 * If the allocation has failed, we're in massive trouble.
3186 * Disable direct injection, and pray that no VM was
3187 * already running...
3188 */
3189 gic_rdists->has_rvpeid = false;
3190 gic_rdists->has_vlpis = false;
3191 }
3192
Marc Zyngier1ac19ca2014-11-24 14:35:14 +00003193 /* Make sure the GIC has seen the above */
3194 dsb(sy);
Valentin Schneiderc0cdc8902021-10-27 16:15:04 +01003195 gic_data_rdist()->flags |= RD_LOCAL_LPI_ENABLED;
Marc Zyngierc440a9d2018-07-27 15:40:13 +01003196 pr_info("GICv3: CPU%d: using %s LPI pending table @%pa\n",
Marc Zyngier11e37d32018-07-27 13:38:54 +01003197 smp_processor_id(),
Valentin Schneiderd23bc2b2021-10-27 16:15:05 +01003198 gic_data_rdist()->flags & RD_LOCAL_PENDTABLE_PREALLOCATED ?
3199 "reserved" : "allocated",
Marc Zyngier11e37d32018-07-27 13:38:54 +01003200 &paddr);
Marc Zyngier1ac19ca2014-11-24 14:35:14 +00003201}
3202
Derek Basehore920181c2018-02-28 21:48:20 -08003203static void its_cpu_init_collection(struct its_node *its)
Marc Zyngier1ac19ca2014-11-24 14:35:14 +00003204{
Derek Basehore920181c2018-02-28 21:48:20 -08003205 int cpu = smp_processor_id();
3206 u64 target;
Marc Zyngier1ac19ca2014-11-24 14:35:14 +00003207
Derek Basehore920181c2018-02-28 21:48:20 -08003208 /* avoid cross node collections and its mapping */
3209 if (its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_23144) {
3210 struct device_node *cpu_node;
Marc Zyngier1ac19ca2014-11-24 14:35:14 +00003211
Derek Basehore920181c2018-02-28 21:48:20 -08003212 cpu_node = of_get_cpu_node(cpu, NULL);
3213 if (its->numa_node != NUMA_NO_NODE &&
3214 its->numa_node != of_node_to_nid(cpu_node))
3215 return;
3216 }
Marc Zyngier1ac19ca2014-11-24 14:35:14 +00003217
Derek Basehore920181c2018-02-28 21:48:20 -08003218 /*
3219 * We now have to bind each collection to its target
3220 * redistributor.
3221 */
3222 if (gic_read_typer(its->base + GITS_TYPER) & GITS_TYPER_PTA) {
Marc Zyngier1ac19ca2014-11-24 14:35:14 +00003223 /*
Derek Basehore920181c2018-02-28 21:48:20 -08003224 * This ITS wants the physical address of the
Marc Zyngier1ac19ca2014-11-24 14:35:14 +00003225 * redistributor.
3226 */
Derek Basehore920181c2018-02-28 21:48:20 -08003227 target = gic_data_rdist()->phys_base;
3228 } else {
3229 /* This ITS wants a linear CPU number. */
3230 target = gic_read_typer(gic_data_rdist_rd_base() + GICR_TYPER);
3231 target = GICR_TYPER_CPU_NUMBER(target) << 16;
Marc Zyngier1ac19ca2014-11-24 14:35:14 +00003232 }
3233
Derek Basehore920181c2018-02-28 21:48:20 -08003234 /* Perform collection mapping */
3235 its->collections[cpu].target_address = target;
3236 its->collections[cpu].col_id = cpu;
3237
3238 its_send_mapc(its, &its->collections[cpu], 1);
3239 its_send_invall(its, &its->collections[cpu]);
3240}
3241
3242static void its_cpu_init_collections(void)
3243{
3244 struct its_node *its;
3245
Sebastian Andrzej Siewiora8db7452018-07-18 17:42:04 +02003246 raw_spin_lock(&its_lock);
Derek Basehore920181c2018-02-28 21:48:20 -08003247
3248 list_for_each_entry(its, &its_nodes, entry)
3249 its_cpu_init_collection(its);
3250
Sebastian Andrzej Siewiora8db7452018-07-18 17:42:04 +02003251 raw_spin_unlock(&its_lock);
Marc Zyngier1ac19ca2014-11-24 14:35:14 +00003252}
Marc Zyngier84a6a2e2014-11-24 14:35:15 +00003253
3254static struct its_device *its_find_device(struct its_node *its, u32 dev_id)
3255{
3256 struct its_device *its_dev = NULL, *tmp;
Marc Zyngier3e39e8f52015-03-06 16:37:43 +00003257 unsigned long flags;
Marc Zyngier84a6a2e2014-11-24 14:35:15 +00003258
Marc Zyngier3e39e8f52015-03-06 16:37:43 +00003259 raw_spin_lock_irqsave(&its->lock, flags);
Marc Zyngier84a6a2e2014-11-24 14:35:15 +00003260
3261 list_for_each_entry(tmp, &its->its_device_list, entry) {
3262 if (tmp->device_id == dev_id) {
3263 its_dev = tmp;
3264 break;
3265 }
3266 }
3267
Marc Zyngier3e39e8f52015-03-06 16:37:43 +00003268 raw_spin_unlock_irqrestore(&its->lock, flags);
Marc Zyngier84a6a2e2014-11-24 14:35:15 +00003269
3270 return its_dev;
3271}
3272
Shanker Donthineni466b7d12016-03-09 22:10:49 -06003273static struct its_baser *its_get_baser(struct its_node *its, u32 type)
3274{
3275 int i;
3276
3277 for (i = 0; i < GITS_BASER_NR_REGS; i++) {
3278 if (GITS_BASER_TYPE(its->tables[i].val) == type)
3279 return &its->tables[i];
3280 }
3281
3282 return NULL;
3283}
3284
Shanker Donthineni539d3782019-01-14 09:50:19 +00003285static bool its_alloc_table_entry(struct its_node *its,
3286 struct its_baser *baser, u32 id)
Shanker Donthineni3faf24e2016-06-06 18:17:32 -05003287{
Shanker Donthineni3faf24e2016-06-06 18:17:32 -05003288 struct page *page;
3289 u32 esz, idx;
3290 __le64 *table;
3291
Shanker Donthineni3faf24e2016-06-06 18:17:32 -05003292 /* Don't allow device id that exceeds single, flat table limit */
3293 esz = GITS_BASER_ENTRY_SIZE(baser->val);
3294 if (!(baser->val & GITS_BASER_INDIRECT))
Marc Zyngier70cc81e2016-12-19 18:53:02 +00003295 return (id < (PAGE_ORDER_TO_SIZE(baser->order) / esz));
Shanker Donthineni3faf24e2016-06-06 18:17:32 -05003296
3297 /* Compute 1st level table index & check if that exceeds table limit */
Marc Zyngier70cc81e2016-12-19 18:53:02 +00003298 idx = id >> ilog2(baser->psz / esz);
Shanker Donthineni3faf24e2016-06-06 18:17:32 -05003299 if (idx >= (PAGE_ORDER_TO_SIZE(baser->order) / GITS_LVL1_ENTRY_SIZE))
3300 return false;
3301
3302 table = baser->base;
3303
3304 /* Allocate memory for 2nd level table */
3305 if (!table[idx]) {
Shanker Donthineni539d3782019-01-14 09:50:19 +00003306 page = alloc_pages_node(its->numa_node, GFP_KERNEL | __GFP_ZERO,
3307 get_order(baser->psz));
Shanker Donthineni3faf24e2016-06-06 18:17:32 -05003308 if (!page)
3309 return false;
3310
3311 /* Flush Lvl2 table to PoC if hw doesn't support coherency */
3312 if (!(baser->val & GITS_BASER_SHAREABILITY_MASK))
Vladimir Murzin328191c2016-11-02 11:54:05 +00003313 gic_flush_dcache_to_poc(page_address(page), baser->psz);
Shanker Donthineni3faf24e2016-06-06 18:17:32 -05003314
3315 table[idx] = cpu_to_le64(page_to_phys(page) | GITS_BASER_VALID);
3316
3317 /* Flush Lvl1 entry to PoC if hw doesn't support coherency */
3318 if (!(baser->val & GITS_BASER_SHAREABILITY_MASK))
Vladimir Murzin328191c2016-11-02 11:54:05 +00003319 gic_flush_dcache_to_poc(table + idx, GITS_LVL1_ENTRY_SIZE);
Shanker Donthineni3faf24e2016-06-06 18:17:32 -05003320
3321 /* Ensure updated table contents are visible to ITS hardware */
3322 dsb(sy);
3323 }
3324
3325 return true;
3326}
3327
Marc Zyngier70cc81e2016-12-19 18:53:02 +00003328static bool its_alloc_device_table(struct its_node *its, u32 dev_id)
3329{
3330 struct its_baser *baser;
3331
3332 baser = its_get_baser(its, GITS_BASER_TYPE_DEVICE);
3333
3334 /* Don't allow device id that exceeds ITS hardware limit */
3335 if (!baser)
Marc Zyngier576a8342019-11-08 16:58:00 +00003336 return (ilog2(dev_id) < device_ids(its));
Marc Zyngier70cc81e2016-12-19 18:53:02 +00003337
Shanker Donthineni539d3782019-01-14 09:50:19 +00003338 return its_alloc_table_entry(its, baser, dev_id);
Marc Zyngier70cc81e2016-12-19 18:53:02 +00003339}
3340
Marc Zyngier7d75bbb2016-12-20 13:55:54 +00003341static bool its_alloc_vpe_table(u32 vpe_id)
3342{
3343 struct its_node *its;
Zenghui Yu4e6437f2020-02-06 15:57:08 +08003344 int cpu;
Marc Zyngier7d75bbb2016-12-20 13:55:54 +00003345
3346 /*
3347 * Make sure the L2 tables are allocated on *all* v4 ITSs. We
3348 * could try and only do it on ITSs corresponding to devices
3349 * that have interrupts targeted at this VPE, but the
3350 * complexity becomes crazy (and you have tons of memory
3351 * anyway, right?).
3352 */
3353 list_for_each_entry(its, &its_nodes, entry) {
3354 struct its_baser *baser;
3355
Marc Zyngier0dd57fe2019-11-08 16:57:58 +00003356 if (!is_v4(its))
Marc Zyngier7d75bbb2016-12-20 13:55:54 +00003357 continue;
3358
3359 baser = its_get_baser(its, GITS_BASER_TYPE_VCPU);
3360 if (!baser)
3361 return false;
3362
Shanker Donthineni539d3782019-01-14 09:50:19 +00003363 if (!its_alloc_table_entry(its, baser, vpe_id))
Marc Zyngier7d75bbb2016-12-20 13:55:54 +00003364 return false;
3365 }
3366
Zenghui Yu4e6437f2020-02-06 15:57:08 +08003367 /* Non v4.1? No need to iterate RDs and go back early. */
3368 if (!gic_rdists->has_rvpeid)
3369 return true;
3370
3371 /*
3372 * Make sure the L2 tables are allocated for all copies of
3373 * the L1 table on *all* v4.1 RDs.
3374 */
3375 for_each_possible_cpu(cpu) {
3376 if (!allocate_vpe_l2_table(cpu, vpe_id))
3377 return false;
3378 }
3379
Marc Zyngier7d75bbb2016-12-20 13:55:54 +00003380 return true;
3381}
3382
Marc Zyngier84a6a2e2014-11-24 14:35:15 +00003383static struct its_device *its_create_device(struct its_node *its, u32 dev_id,
Marc Zyngier93f94ea2017-08-04 18:37:09 +01003384 int nvecs, bool alloc_lpis)
Marc Zyngier84a6a2e2014-11-24 14:35:15 +00003385{
3386 struct its_device *dev;
Marc Zyngier93f94ea2017-08-04 18:37:09 +01003387 unsigned long *lpi_map = NULL;
Marc Zyngier3e39e8f52015-03-06 16:37:43 +00003388 unsigned long flags;
Marc Zyngier591e5be2015-07-17 10:46:42 +01003389 u16 *col_map = NULL;
Marc Zyngier84a6a2e2014-11-24 14:35:15 +00003390 void *itt;
3391 int lpi_base;
3392 int nr_lpis;
Marc Zyngierc8481262014-12-12 10:51:24 +00003393 int nr_ites;
Marc Zyngier84a6a2e2014-11-24 14:35:15 +00003394 int sz;
3395
Shanker Donthineni3faf24e2016-06-06 18:17:32 -05003396 if (!its_alloc_device_table(its, dev_id))
Shanker Donthineni466b7d12016-03-09 22:10:49 -06003397 return NULL;
3398
Marc Zyngier147c8f32018-05-27 16:39:55 +01003399 if (WARN_ON(!is_power_of_2(nvecs)))
3400 nvecs = roundup_pow_of_two(nvecs);
3401
Marc Zyngier84a6a2e2014-11-24 14:35:15 +00003402 dev = kzalloc(sizeof(*dev), GFP_KERNEL);
Marc Zyngierc8481262014-12-12 10:51:24 +00003403 /*
Marc Zyngier147c8f32018-05-27 16:39:55 +01003404 * Even if the device wants a single LPI, the ITT must be
3405 * sized as a power of two (and you need at least one bit...).
Marc Zyngierc8481262014-12-12 10:51:24 +00003406 */
Marc Zyngier147c8f32018-05-27 16:39:55 +01003407 nr_ites = max(2, nvecs);
Marc Zyngierffedbf02019-11-08 16:57:59 +00003408 sz = nr_ites * (FIELD_GET(GITS_TYPER_ITT_ENTRY_SIZE, its->typer) + 1);
Marc Zyngier84a6a2e2014-11-24 14:35:15 +00003409 sz = max(sz, ITS_ITT_ALIGN) + ITS_ITT_ALIGN - 1;
Shanker Donthineni539d3782019-01-14 09:50:19 +00003410 itt = kzalloc_node(sz, GFP_KERNEL, its->numa_node);
Marc Zyngier93f94ea2017-08-04 18:37:09 +01003411 if (alloc_lpis) {
Marc Zyngier38dd7c42018-05-27 17:03:03 +01003412 lpi_map = its_lpi_alloc(nvecs, &lpi_base, &nr_lpis);
Marc Zyngier93f94ea2017-08-04 18:37:09 +01003413 if (lpi_map)
Kees Cook6396bb22018-06-12 14:03:40 -07003414 col_map = kcalloc(nr_lpis, sizeof(*col_map),
Marc Zyngier93f94ea2017-08-04 18:37:09 +01003415 GFP_KERNEL);
3416 } else {
Kees Cook6396bb22018-06-12 14:03:40 -07003417 col_map = kcalloc(nr_ites, sizeof(*col_map), GFP_KERNEL);
Marc Zyngier93f94ea2017-08-04 18:37:09 +01003418 nr_lpis = 0;
3419 lpi_base = 0;
3420 }
Marc Zyngier84a6a2e2014-11-24 14:35:15 +00003421
Marc Zyngier93f94ea2017-08-04 18:37:09 +01003422 if (!dev || !itt || !col_map || (!lpi_map && alloc_lpis)) {
Marc Zyngier84a6a2e2014-11-24 14:35:15 +00003423 kfree(dev);
3424 kfree(itt);
Andy Shevchenkoff5fe882021-06-18 18:16:54 +03003425 bitmap_free(lpi_map);
Marc Zyngier591e5be2015-07-17 10:46:42 +01003426 kfree(col_map);
Marc Zyngier84a6a2e2014-11-24 14:35:15 +00003427 return NULL;
3428 }
3429
Vladimir Murzin328191c2016-11-02 11:54:05 +00003430 gic_flush_dcache_to_poc(itt, sz);
Marc Zyngier5a9a8912015-09-13 12:14:32 +01003431
Marc Zyngier84a6a2e2014-11-24 14:35:15 +00003432 dev->its = its;
3433 dev->itt = itt;
Marc Zyngierc8481262014-12-12 10:51:24 +00003434 dev->nr_ites = nr_ites;
Marc Zyngier591e5be2015-07-17 10:46:42 +01003435 dev->event_map.lpi_map = lpi_map;
3436 dev->event_map.col_map = col_map;
3437 dev->event_map.lpi_base = lpi_base;
3438 dev->event_map.nr_lpis = nr_lpis;
Marc Zyngier11635fa2019-11-08 16:58:05 +00003439 raw_spin_lock_init(&dev->event_map.vlpi_lock);
Marc Zyngier84a6a2e2014-11-24 14:35:15 +00003440 dev->device_id = dev_id;
3441 INIT_LIST_HEAD(&dev->entry);
3442
Marc Zyngier3e39e8f52015-03-06 16:37:43 +00003443 raw_spin_lock_irqsave(&its->lock, flags);
Marc Zyngier84a6a2e2014-11-24 14:35:15 +00003444 list_add(&dev->entry, &its->its_device_list);
Marc Zyngier3e39e8f52015-03-06 16:37:43 +00003445 raw_spin_unlock_irqrestore(&its->lock, flags);
Marc Zyngier84a6a2e2014-11-24 14:35:15 +00003446
Marc Zyngier84a6a2e2014-11-24 14:35:15 +00003447 /* Map device to its ITT */
3448 its_send_mapd(dev, 1);
3449
3450 return dev;
3451}
3452
3453static void its_free_device(struct its_device *its_dev)
3454{
Marc Zyngier3e39e8f52015-03-06 16:37:43 +00003455 unsigned long flags;
3456
3457 raw_spin_lock_irqsave(&its_dev->its->lock, flags);
Marc Zyngier84a6a2e2014-11-24 14:35:15 +00003458 list_del(&its_dev->entry);
Marc Zyngier3e39e8f52015-03-06 16:37:43 +00003459 raw_spin_unlock_irqrestore(&its_dev->its->lock, flags);
Marc Zyngier898aa5c2019-11-08 16:57:55 +00003460 kfree(its_dev->event_map.col_map);
Marc Zyngier84a6a2e2014-11-24 14:35:15 +00003461 kfree(its_dev->itt);
3462 kfree(its_dev);
3463}
Marc Zyngierb48ac832014-11-24 14:35:16 +00003464
Marc Zyngier8208d172019-01-18 14:08:59 +00003465static int its_alloc_device_irq(struct its_device *dev, int nvecs, irq_hw_number_t *hwirq)
Marc Zyngierb48ac832014-11-24 14:35:16 +00003466{
3467 int idx;
3468
Zenghui Yu342be102019-07-27 06:14:22 +00003469 /* Find a free LPI region in lpi_map and allocate them. */
Marc Zyngier8208d172019-01-18 14:08:59 +00003470 idx = bitmap_find_free_region(dev->event_map.lpi_map,
3471 dev->event_map.nr_lpis,
3472 get_count_order(nvecs));
3473 if (idx < 0)
Marc Zyngierb48ac832014-11-24 14:35:16 +00003474 return -ENOSPC;
3475
Marc Zyngier591e5be2015-07-17 10:46:42 +01003476 *hwirq = dev->event_map.lpi_base + idx;
Marc Zyngierb48ac832014-11-24 14:35:16 +00003477
Marc Zyngierb48ac832014-11-24 14:35:16 +00003478 return 0;
3479}
3480
Marc Zyngier54456db2015-07-28 14:46:21 +01003481static int its_msi_prepare(struct irq_domain *domain, struct device *dev,
3482 int nvec, msi_alloc_info_t *info)
Marc Zyngiere8137f42015-03-06 16:37:42 +00003483{
Marc Zyngierb48ac832014-11-24 14:35:16 +00003484 struct its_node *its;
Marc Zyngierb48ac832014-11-24 14:35:16 +00003485 struct its_device *its_dev;
Marc Zyngier54456db2015-07-28 14:46:21 +01003486 struct msi_domain_info *msi_info;
3487 u32 dev_id;
Marc Zyngier9791ec72019-01-29 10:02:33 +00003488 int err = 0;
Marc Zyngierb48ac832014-11-24 14:35:16 +00003489
Marc Zyngier54456db2015-07-28 14:46:21 +01003490 /*
Julien Gralla7c90f52019-04-18 16:58:14 +01003491 * We ignore "dev" entirely, and rely on the dev_id that has
Marc Zyngier54456db2015-07-28 14:46:21 +01003492 * been passed via the scratchpad. This limits this domain's
3493 * usefulness to upper layers that definitely know that they
3494 * are built on top of the ITS.
3495 */
3496 dev_id = info->scratchpad[0].ul;
3497
3498 msi_info = msi_get_domain_info(domain);
3499 its = msi_info->data;
3500
Marc Zyngier20b3d542016-12-20 15:23:22 +00003501 if (!gic_rdists->has_direct_lpi &&
3502 vpe_proxy.dev &&
3503 vpe_proxy.dev->its == its &&
3504 dev_id == vpe_proxy.dev->device_id) {
3505 /* Bad luck. Get yourself a better implementation */
3506 WARN_ONCE(1, "DevId %x clashes with GICv4 VPE proxy device\n",
3507 dev_id);
3508 return -EINVAL;
3509 }
3510
Marc Zyngier9791ec72019-01-29 10:02:33 +00003511 mutex_lock(&its->dev_alloc_lock);
Marc Zyngierf1304202015-07-28 14:46:18 +01003512 its_dev = its_find_device(its, dev_id);
Marc Zyngiere8137f42015-03-06 16:37:42 +00003513 if (its_dev) {
3514 /*
3515 * We already have seen this ID, probably through
3516 * another alias (PCI bridge of some sort). No need to
3517 * create the device.
3518 */
Marc Zyngier9791ec72019-01-29 10:02:33 +00003519 its_dev->shared = true;
Marc Zyngierf1304202015-07-28 14:46:18 +01003520 pr_debug("Reusing ITT for devID %x\n", dev_id);
Marc Zyngiere8137f42015-03-06 16:37:42 +00003521 goto out;
3522 }
Marc Zyngierb48ac832014-11-24 14:35:16 +00003523
Marc Zyngier93f94ea2017-08-04 18:37:09 +01003524 its_dev = its_create_device(its, dev_id, nvec, true);
Marc Zyngier9791ec72019-01-29 10:02:33 +00003525 if (!its_dev) {
3526 err = -ENOMEM;
3527 goto out;
3528 }
Marc Zyngierb48ac832014-11-24 14:35:16 +00003529
Marc Zyngier5fe71d22020-11-29 13:52:07 +00003530 if (info->flags & MSI_ALLOC_FLAGS_PROXY_DEVICE)
3531 its_dev->shared = true;
3532
Marc Zyngierf1304202015-07-28 14:46:18 +01003533 pr_debug("ITT %d entries, %d bits\n", nvec, ilog2(nvec));
Marc Zyngiere8137f42015-03-06 16:37:42 +00003534out:
Marc Zyngier9791ec72019-01-29 10:02:33 +00003535 mutex_unlock(&its->dev_alloc_lock);
Marc Zyngierb48ac832014-11-24 14:35:16 +00003536 info->scratchpad[0].ptr = its_dev;
Marc Zyngier9791ec72019-01-29 10:02:33 +00003537 return err;
Marc Zyngierb48ac832014-11-24 14:35:16 +00003538}
3539
Marc Zyngier54456db2015-07-28 14:46:21 +01003540static struct msi_domain_ops its_msi_domain_ops = {
3541 .msi_prepare = its_msi_prepare,
3542};
3543
Marc Zyngierb48ac832014-11-24 14:35:16 +00003544static int its_irq_gic_domain_alloc(struct irq_domain *domain,
3545 unsigned int virq,
3546 irq_hw_number_t hwirq)
3547{
Marc Zyngierf833f572015-10-13 12:51:33 +01003548 struct irq_fwspec fwspec;
Marc Zyngierb48ac832014-11-24 14:35:16 +00003549
Marc Zyngierf833f572015-10-13 12:51:33 +01003550 if (irq_domain_get_of_node(domain->parent)) {
3551 fwspec.fwnode = domain->parent->fwnode;
3552 fwspec.param_count = 3;
3553 fwspec.param[0] = GIC_IRQ_TYPE_LPI;
3554 fwspec.param[1] = hwirq;
3555 fwspec.param[2] = IRQ_TYPE_EDGE_RISING;
Tomasz Nowicki3f010cf2016-09-12 20:32:25 +02003556 } else if (is_fwnode_irqchip(domain->parent->fwnode)) {
3557 fwspec.fwnode = domain->parent->fwnode;
3558 fwspec.param_count = 2;
3559 fwspec.param[0] = hwirq;
3560 fwspec.param[1] = IRQ_TYPE_EDGE_RISING;
Marc Zyngierf833f572015-10-13 12:51:33 +01003561 } else {
3562 return -EINVAL;
3563 }
Marc Zyngierb48ac832014-11-24 14:35:16 +00003564
Marc Zyngierf833f572015-10-13 12:51:33 +01003565 return irq_domain_alloc_irqs_parent(domain, virq, 1, &fwspec);
Marc Zyngierb48ac832014-11-24 14:35:16 +00003566}
3567
3568static int its_irq_domain_alloc(struct irq_domain *domain, unsigned int virq,
3569 unsigned int nr_irqs, void *args)
3570{
3571 msi_alloc_info_t *info = args;
3572 struct its_device *its_dev = info->scratchpad[0].ptr;
Julien Grall35ae7df2019-05-01 14:58:21 +01003573 struct its_node *its = its_dev->its;
Thomas Gleixnerf0c7bac2020-07-24 22:44:41 +02003574 struct irq_data *irqd;
Marc Zyngierb48ac832014-11-24 14:35:16 +00003575 irq_hw_number_t hwirq;
3576 int err;
3577 int i;
3578
Marc Zyngier8208d172019-01-18 14:08:59 +00003579 err = its_alloc_device_irq(its_dev, nr_irqs, &hwirq);
3580 if (err)
3581 return err;
Marc Zyngierb48ac832014-11-24 14:35:16 +00003582
Julien Grall35ae7df2019-05-01 14:58:21 +01003583 err = iommu_dma_prepare_msi(info->desc, its->get_msi_base(its_dev));
3584 if (err)
3585 return err;
3586
Marc Zyngier8208d172019-01-18 14:08:59 +00003587 for (i = 0; i < nr_irqs; i++) {
3588 err = its_irq_gic_domain_alloc(domain, virq + i, hwirq + i);
Marc Zyngierb48ac832014-11-24 14:35:16 +00003589 if (err)
3590 return err;
3591
3592 irq_domain_set_hwirq_and_chip(domain, virq + i,
Marc Zyngier8208d172019-01-18 14:08:59 +00003593 hwirq + i, &its_irq_chip, its_dev);
Thomas Gleixnerf0c7bac2020-07-24 22:44:41 +02003594 irqd = irq_get_irq_data(virq + i);
3595 irqd_set_single_target(irqd);
3596 irqd_set_affinity_on_activate(irqd);
James Gowans8f4b5892023-06-08 14:00:21 +02003597 irqd_set_resend_when_in_progress(irqd);
Marc Zyngierf1304202015-07-28 14:46:18 +01003598 pr_debug("ID:%d pID:%d vID:%d\n",
Marc Zyngier8208d172019-01-18 14:08:59 +00003599 (int)(hwirq + i - its_dev->event_map.lpi_base),
3600 (int)(hwirq + i), virq + i);
Marc Zyngierb48ac832014-11-24 14:35:16 +00003601 }
3602
3603 return 0;
3604}
3605
Thomas Gleixner72491642017-09-13 23:29:10 +02003606static int its_irq_domain_activate(struct irq_domain *domain,
Thomas Gleixner702cb0a2017-12-29 16:59:06 +01003607 struct irq_data *d, bool reserve)
Marc Zyngieraca268d2014-12-12 10:51:23 +00003608{
3609 struct its_device *its_dev = irq_data_get_irq_chip_data(d);
3610 u32 event = its_get_event_id(d);
Marc Zyngier0d224d32017-08-18 09:39:18 +01003611 int cpu;
Ganapatrao Kulkarnifbf8f402016-05-25 15:29:20 +02003612
Marc Zyngierc5d60822020-05-15 17:57:52 +01003613 cpu = its_select_cpu(d, cpu_online_mask);
3614 if (cpu < 0 || cpu >= nr_cpu_ids)
3615 return -EINVAL;
Yang Yingliangc1797b12018-06-22 10:52:51 +01003616
Marc Zyngier2f13ff12020-05-15 17:57:51 +01003617 its_inc_lpi_count(d, cpu);
Marc Zyngier0d224d32017-08-18 09:39:18 +01003618 its_dev->event_map.col_map[event] = cpu;
3619 irq_data_update_effective_affinity(d, cpumask_of(cpu));
Marc Zyngier591e5be2015-07-17 10:46:42 +01003620
Marc Zyngieraca268d2014-12-12 10:51:23 +00003621 /* Map the GIC IRQ and event to the device */
Marc Zyngier6a25ad32016-12-20 15:52:26 +00003622 its_send_mapti(its_dev, d->hwirq, event);
Thomas Gleixner72491642017-09-13 23:29:10 +02003623 return 0;
Marc Zyngieraca268d2014-12-12 10:51:23 +00003624}
3625
3626static void its_irq_domain_deactivate(struct irq_domain *domain,
3627 struct irq_data *d)
3628{
3629 struct its_device *its_dev = irq_data_get_irq_chip_data(d);
3630 u32 event = its_get_event_id(d);
3631
Marc Zyngier2f13ff12020-05-15 17:57:51 +01003632 its_dec_lpi_count(d, its_dev->event_map.col_map[event]);
Marc Zyngieraca268d2014-12-12 10:51:23 +00003633 /* Stop the delivery of interrupts */
3634 its_send_discard(its_dev, event);
3635}
3636
Marc Zyngierb48ac832014-11-24 14:35:16 +00003637static void its_irq_domain_free(struct irq_domain *domain, unsigned int virq,
3638 unsigned int nr_irqs)
3639{
3640 struct irq_data *d = irq_domain_get_irq_data(domain, virq);
3641 struct its_device *its_dev = irq_data_get_irq_chip_data(d);
Marc Zyngier9791ec72019-01-29 10:02:33 +00003642 struct its_node *its = its_dev->its;
Marc Zyngierb48ac832014-11-24 14:35:16 +00003643 int i;
3644
Marc Zyngierc9c96e32019-09-05 14:56:47 +01003645 bitmap_release_region(its_dev->event_map.lpi_map,
3646 its_get_event_id(irq_domain_get_irq_data(domain, virq)),
3647 get_count_order(nr_irqs));
3648
Marc Zyngierb48ac832014-11-24 14:35:16 +00003649 for (i = 0; i < nr_irqs; i++) {
3650 struct irq_data *data = irq_domain_get_irq_data(domain,
3651 virq + i);
Marc Zyngierb48ac832014-11-24 14:35:16 +00003652 /* Nuke the entry in the domain */
Marc Zyngier2da39942014-12-12 10:51:22 +00003653 irq_domain_reset_irq_data(data);
Marc Zyngierb48ac832014-11-24 14:35:16 +00003654 }
3655
Marc Zyngier9791ec72019-01-29 10:02:33 +00003656 mutex_lock(&its->dev_alloc_lock);
3657
3658 /*
3659 * If all interrupts have been freed, start mopping the
Ingo Molnara359f752021-03-22 04:21:30 +01003660 * floor. This is conditioned on the device not being shared.
Marc Zyngier9791ec72019-01-29 10:02:33 +00003661 */
3662 if (!its_dev->shared &&
3663 bitmap_empty(its_dev->event_map.lpi_map,
Marc Zyngier591e5be2015-07-17 10:46:42 +01003664 its_dev->event_map.nr_lpis)) {
Marc Zyngier38dd7c42018-05-27 17:03:03 +01003665 its_lpi_free(its_dev->event_map.lpi_map,
3666 its_dev->event_map.lpi_base,
3667 its_dev->event_map.nr_lpis);
Marc Zyngierb48ac832014-11-24 14:35:16 +00003668
3669 /* Unmap device/itt */
3670 its_send_mapd(its_dev, 0);
3671 its_free_device(its_dev);
3672 }
3673
Marc Zyngier9791ec72019-01-29 10:02:33 +00003674 mutex_unlock(&its->dev_alloc_lock);
3675
Marc Zyngierb48ac832014-11-24 14:35:16 +00003676 irq_domain_free_irqs_parent(domain, virq, nr_irqs);
3677}
3678
3679static const struct irq_domain_ops its_domain_ops = {
Thomas Gleixner48f71d52024-06-23 17:18:36 +02003680 .select = msi_lib_irq_domain_select,
Marc Zyngierb48ac832014-11-24 14:35:16 +00003681 .alloc = its_irq_domain_alloc,
3682 .free = its_irq_domain_free,
Marc Zyngieraca268d2014-12-12 10:51:23 +00003683 .activate = its_irq_domain_activate,
3684 .deactivate = its_irq_domain_deactivate,
Marc Zyngierb48ac832014-11-24 14:35:16 +00003685};
Marc Zyngier4c21f3c2014-11-24 14:35:17 +00003686
Marc Zyngier20b3d542016-12-20 15:23:22 +00003687/*
3688 * This is insane.
3689 *
Marc Zyngier0684c702019-12-24 11:10:30 +00003690 * If a GICv4.0 doesn't implement Direct LPIs (which is extremely
Marc Zyngier20b3d542016-12-20 15:23:22 +00003691 * likely), the only way to perform an invalidate is to use a fake
3692 * device to issue an INV command, implying that the LPI has first
3693 * been mapped to some event on that device. Since this is not exactly
3694 * cheap, we try to keep that mapping around as long as possible, and
3695 * only issue an UNMAP if we're short on available slots.
3696 *
3697 * Broken by design(tm).
Marc Zyngier0684c702019-12-24 11:10:30 +00003698 *
3699 * GICv4.1, on the other hand, mandates that we're able to invalidate
3700 * by writing to a MMIO register. It doesn't implement the whole of
3701 * DirectLPI, but that's good enough. And most of the time, we don't
3702 * even have to invalidate anything, as the redistributor can be told
3703 * whether to generate a doorbell or not (we thus leave it enabled,
3704 * always).
Marc Zyngier20b3d542016-12-20 15:23:22 +00003705 */
3706static void its_vpe_db_proxy_unmap_locked(struct its_vpe *vpe)
3707{
Marc Zyngier0684c702019-12-24 11:10:30 +00003708 /* GICv4.1 doesn't use a proxy, so nothing to do here */
3709 if (gic_rdists->has_rvpeid)
3710 return;
3711
Marc Zyngier20b3d542016-12-20 15:23:22 +00003712 /* Already unmapped? */
3713 if (vpe->vpe_proxy_event == -1)
3714 return;
3715
3716 its_send_discard(vpe_proxy.dev, vpe->vpe_proxy_event);
3717 vpe_proxy.vpes[vpe->vpe_proxy_event] = NULL;
3718
3719 /*
3720 * We don't track empty slots at all, so let's move the
3721 * next_victim pointer if we can quickly reuse that slot
3722 * instead of nuking an existing entry. Not clear that this is
3723 * always a win though, and this might just generate a ripple
3724 * effect... Let's just hope VPEs don't migrate too often.
3725 */
3726 if (vpe_proxy.vpes[vpe_proxy.next_victim])
3727 vpe_proxy.next_victim = vpe->vpe_proxy_event;
3728
3729 vpe->vpe_proxy_event = -1;
3730}
3731
3732static void its_vpe_db_proxy_unmap(struct its_vpe *vpe)
3733{
Marc Zyngier0684c702019-12-24 11:10:30 +00003734 /* GICv4.1 doesn't use a proxy, so nothing to do here */
3735 if (gic_rdists->has_rvpeid)
3736 return;
3737
Marc Zyngier20b3d542016-12-20 15:23:22 +00003738 if (!gic_rdists->has_direct_lpi) {
3739 unsigned long flags;
3740
3741 raw_spin_lock_irqsave(&vpe_proxy.lock, flags);
3742 its_vpe_db_proxy_unmap_locked(vpe);
3743 raw_spin_unlock_irqrestore(&vpe_proxy.lock, flags);
3744 }
3745}
3746
3747static void its_vpe_db_proxy_map_locked(struct its_vpe *vpe)
3748{
Marc Zyngier0684c702019-12-24 11:10:30 +00003749 /* GICv4.1 doesn't use a proxy, so nothing to do here */
3750 if (gic_rdists->has_rvpeid)
3751 return;
3752
Marc Zyngier20b3d542016-12-20 15:23:22 +00003753 /* Already mapped? */
3754 if (vpe->vpe_proxy_event != -1)
3755 return;
3756
3757 /* This slot was already allocated. Kick the other VPE out. */
3758 if (vpe_proxy.vpes[vpe_proxy.next_victim])
3759 its_vpe_db_proxy_unmap_locked(vpe_proxy.vpes[vpe_proxy.next_victim]);
3760
3761 /* Map the new VPE instead */
3762 vpe_proxy.vpes[vpe_proxy.next_victim] = vpe;
3763 vpe->vpe_proxy_event = vpe_proxy.next_victim;
3764 vpe_proxy.next_victim = (vpe_proxy.next_victim + 1) % vpe_proxy.dev->nr_ites;
3765
3766 vpe_proxy.dev->event_map.col_map[vpe->vpe_proxy_event] = vpe->col_idx;
3767 its_send_mapti(vpe_proxy.dev, vpe->vpe_db_lpi, vpe->vpe_proxy_event);
3768}
3769
Marc Zyngier958b90d2017-08-18 16:14:17 +01003770static void its_vpe_db_proxy_move(struct its_vpe *vpe, int from, int to)
3771{
3772 unsigned long flags;
3773 struct its_collection *target_col;
3774
Marc Zyngier0684c702019-12-24 11:10:30 +00003775 /* GICv4.1 doesn't use a proxy, so nothing to do here */
3776 if (gic_rdists->has_rvpeid)
3777 return;
3778
Marc Zyngier958b90d2017-08-18 16:14:17 +01003779 if (gic_rdists->has_direct_lpi) {
3780 void __iomem *rdbase;
3781
3782 rdbase = per_cpu_ptr(gic_rdists->rdist, from)->rd_base;
3783 gic_write_lpir(vpe->vpe_db_lpi, rdbase + GICR_CLRLPIR);
Marc Zyngier2f4f0642019-11-08 16:57:56 +00003784 wait_for_syncr(rdbase);
Marc Zyngier958b90d2017-08-18 16:14:17 +01003785
3786 return;
3787 }
3788
3789 raw_spin_lock_irqsave(&vpe_proxy.lock, flags);
3790
3791 its_vpe_db_proxy_map_locked(vpe);
3792
3793 target_col = &vpe_proxy.dev->its->collections[to];
3794 its_send_movi(vpe_proxy.dev, target_col, vpe->vpe_proxy_event);
3795 vpe_proxy.dev->event_map.col_map[vpe->vpe_proxy_event] = to;
3796
3797 raw_spin_unlock_irqrestore(&vpe_proxy.lock, flags);
3798}
3799
Marc Zyngier3171a472016-12-20 15:17:28 +00003800static int its_vpe_set_affinity(struct irq_data *d,
3801 const struct cpumask *mask_val,
3802 bool force)
3803{
3804 struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
Dawei Lifcb8af42024-04-16 16:54:50 +08003805 unsigned int from, cpu = nr_cpu_ids;
3806 struct cpumask *table_mask;
Marc Zyngierf3a059212020-03-04 20:33:10 +00003807 unsigned long flags;
Marc Zyngier3171a472016-12-20 15:17:28 +00003808
3809 /*
3810 * Changing affinity is mega expensive, so let's be as lazy as
Marc Zyngier20b3d542016-12-20 15:23:22 +00003811 * we can and only do it if we really have to. Also, if mapped
Marc Zyngier958b90d2017-08-18 16:14:17 +01003812 * into the proxy device, we need to move the doorbell
3813 * interrupt to its new location.
Marc Zyngierf3a059212020-03-04 20:33:10 +00003814 *
3815 * Another thing is that changing the affinity of a vPE affects
3816 * *other interrupts* such as all the vLPIs that are routed to
3817 * this vPE. This means that the irq_desc lock is not enough to
3818 * protect us, and that we must ensure nobody samples vpe->col_idx
3819 * during the update, hence the lock below which must also be
3820 * taken on any vLPI handling path that evaluates vpe->col_idx.
Marc Zyngierf97fd452024-08-18 18:16:25 +01003821 *
3822 * Finally, we must protect ourselves against concurrent updates of
3823 * the mapping state on this VM should the ITS list be in use (see
3824 * the shortcut in its_send_vmovp() otherewise).
Marc Zyngier3171a472016-12-20 15:17:28 +00003825 */
Marc Zyngierf97fd452024-08-18 18:16:25 +01003826 if (its_list_map)
3827 raw_spin_lock(&vpe->its_vm->vmapp_lock);
3828
Marc Zyngierf3a059212020-03-04 20:33:10 +00003829 from = vpe_to_cpuid_lock(vpe, &flags);
Marc Zyngieraf9acbf2024-02-13 10:12:06 +00003830 table_mask = gic_data_rdist_cpu(from)->vpe_table_mask;
3831
3832 /*
3833 * If we are offered another CPU in the same GICv4.1 ITS
3834 * affinity, pick this one. Otherwise, any CPU will do.
3835 */
Dawei Lifcb8af42024-04-16 16:54:50 +08003836 if (table_mask)
3837 cpu = cpumask_any_and(mask_val, table_mask);
3838 if (cpu < nr_cpu_ids) {
3839 if (cpumask_test_cpu(from, mask_val) &&
3840 cpumask_test_cpu(from, table_mask))
3841 cpu = from;
3842 } else {
Marc Zyngieraf9acbf2024-02-13 10:12:06 +00003843 cpu = cpumask_first(mask_val);
Dawei Lifcb8af42024-04-16 16:54:50 +08003844 }
Marc Zyngieraf9acbf2024-02-13 10:12:06 +00003845
Marc Zyngierf3a059212020-03-04 20:33:10 +00003846 if (from == cpu)
Marc Zyngierdd3f0502019-12-24 11:10:31 +00003847 goto out;
Marc Zyngier958b90d2017-08-18 16:14:17 +01003848
Marc Zyngierdd3f0502019-12-24 11:10:31 +00003849 vpe->col_idx = cpu;
Marc Zyngier3171a472016-12-20 15:17:28 +00003850
Marc Zyngierdd3f0502019-12-24 11:10:31 +00003851 its_send_vmovp(vpe);
3852 its_vpe_db_proxy_move(vpe, from, cpu);
3853
3854out:
Marc Zyngier44c4c252017-10-19 10:11:34 +01003855 irq_data_update_effective_affinity(d, cpumask_of(cpu));
Marc Zyngierf3a059212020-03-04 20:33:10 +00003856 vpe_to_cpuid_unlock(vpe, flags);
Marc Zyngier44c4c252017-10-19 10:11:34 +01003857
Marc Zyngierf97fd452024-08-18 18:16:25 +01003858 if (its_list_map)
3859 raw_spin_unlock(&vpe->its_vm->vmapp_lock);
3860
Marc Zyngier3171a472016-12-20 15:17:28 +00003861 return IRQ_SET_MASK_OK_DONE;
3862}
3863
Marc Zyngier96806222020-04-10 11:13:26 +01003864static void its_wait_vpt_parse_complete(void)
3865{
3866 void __iomem *vlpi_base = gic_data_rdist_vlpi_base();
3867 u64 val;
3868
3869 if (!gic_rdists->has_vpend_valid_dirty)
3870 return;
3871
Zenghui Yu31dbb6b2020-06-05 13:23:45 +08003872 WARN_ON_ONCE(readq_relaxed_poll_timeout_atomic(vlpi_base + GICR_VPENDBASER,
3873 val,
3874 !(val & GICR_VPENDBASER_Dirty),
Shenming Lu0b3949822020-11-28 22:18:56 +08003875 1, 500));
Marc Zyngier96806222020-04-10 11:13:26 +01003876}
3877
Marc Zyngiere643d802016-12-20 15:09:31 +00003878static void its_vpe_schedule(struct its_vpe *vpe)
3879{
Robin Murphy50c33092018-02-16 16:57:56 +00003880 void __iomem *vlpi_base = gic_data_rdist_vlpi_base();
Marc Zyngiere643d802016-12-20 15:09:31 +00003881 u64 val;
3882
3883 /* Schedule the VPE */
3884 val = virt_to_phys(page_address(vpe->its_vm->vprop_page)) &
3885 GENMASK_ULL(51, 12);
3886 val |= (LPI_NRBITS - 1) & GICR_VPROPBASER_IDBITS_MASK;
Marc Zyngier846297e2024-02-13 10:12:04 +00003887 if (rdists_support_shareable()) {
3888 val |= GICR_VPROPBASER_RaWb;
3889 val |= GICR_VPROPBASER_InnerShareable;
3890 }
Zenghui Yu5186a6c2020-02-06 15:57:11 +08003891 gicr_write_vpropbaser(val, vlpi_base + GICR_VPROPBASER);
Marc Zyngiere643d802016-12-20 15:09:31 +00003892
3893 val = virt_to_phys(page_address(vpe->vpt_page)) &
3894 GENMASK_ULL(51, 16);
Marc Zyngier846297e2024-02-13 10:12:04 +00003895 if (rdists_support_shareable()) {
3896 val |= GICR_VPENDBASER_RaWaWb;
3897 val |= GICR_VPENDBASER_InnerShareable;
3898 }
Marc Zyngiere643d802016-12-20 15:09:31 +00003899 /*
3900 * There is no good way of finding out if the pending table is
3901 * empty as we can race against the doorbell interrupt very
3902 * easily. So in the end, vpe->pending_last is only an
3903 * indication that the vcpu has something pending, not one
3904 * that the pending table is empty. A good implementation
3905 * would be able to read its coarse map pretty quickly anyway,
3906 * making this a tolerable issue.
3907 */
3908 val |= GICR_VPENDBASER_PendingLast;
3909 val |= vpe->idai ? GICR_VPENDBASER_IDAI : 0;
3910 val |= GICR_VPENDBASER_Valid;
Zenghui Yu5186a6c2020-02-06 15:57:11 +08003911 gicr_write_vpendbaser(val, vlpi_base + GICR_VPENDBASER);
Marc Zyngiere643d802016-12-20 15:09:31 +00003912}
3913
3914static void its_vpe_deschedule(struct its_vpe *vpe)
3915{
Robin Murphy50c33092018-02-16 16:57:56 +00003916 void __iomem *vlpi_base = gic_data_rdist_vlpi_base();
Marc Zyngiere643d802016-12-20 15:09:31 +00003917 u64 val;
3918
Marc Zyngiere64fab12019-12-24 11:10:35 +00003919 val = its_clear_vpend_valid(vlpi_base, 0, 0);
Marc Zyngiere643d802016-12-20 15:09:31 +00003920
Marc Zyngiere64fab12019-12-24 11:10:35 +00003921 vpe->idai = !!(val & GICR_VPENDBASER_IDAI);
3922 vpe->pending_last = !!(val & GICR_VPENDBASER_PendingLast);
Marc Zyngiere643d802016-12-20 15:09:31 +00003923}
3924
Marc Zyngier40619a22017-10-08 15:16:09 +01003925static void its_vpe_invall(struct its_vpe *vpe)
3926{
3927 struct its_node *its;
3928
Marc Zyngierf0eb1542024-07-05 10:31:54 +01003929 guard(raw_spinlock_irqsave)(&vpe->its_vm->vmapp_lock);
3930
Marc Zyngier40619a22017-10-08 15:16:09 +01003931 list_for_each_entry(its, &its_nodes, entry) {
Marc Zyngier0dd57fe2019-11-08 16:57:58 +00003932 if (!is_v4(its))
Marc Zyngier40619a22017-10-08 15:16:09 +01003933 continue;
3934
Marc Zyngier2247e1b2017-10-08 18:50:36 +01003935 if (its_list_map && !vpe->its_vm->vlpi_count[its->list_nr])
3936 continue;
3937
Marc Zyngier3c1ccee2017-10-09 13:17:43 +01003938 /*
3939 * Sending a VINVALL to a single ITS is enough, as all
3940 * we need is to reach the redistributors.
3941 */
Marc Zyngier40619a22017-10-08 15:16:09 +01003942 its_send_vinvall(its, vpe);
Marc Zyngier3c1ccee2017-10-09 13:17:43 +01003943 return;
Marc Zyngier40619a22017-10-08 15:16:09 +01003944 }
3945}
3946
Marc Zyngiere643d802016-12-20 15:09:31 +00003947static int its_vpe_set_vcpu_affinity(struct irq_data *d, void *vcpu_info)
3948{
3949 struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
3950 struct its_cmd_info *info = vcpu_info;
3951
3952 switch (info->cmd_type) {
3953 case SCHEDULE_VPE:
3954 its_vpe_schedule(vpe);
3955 return 0;
3956
3957 case DESCHEDULE_VPE:
3958 its_vpe_deschedule(vpe);
3959 return 0;
3960
Shenming Lu57e3ceb2020-11-28 22:18:57 +08003961 case COMMIT_VPE:
3962 its_wait_vpt_parse_complete();
3963 return 0;
3964
Marc Zyngier5e2f7642016-12-20 15:10:50 +00003965 case INVALL_VPE:
Marc Zyngier40619a22017-10-08 15:16:09 +01003966 its_vpe_invall(vpe);
Marc Zyngier5e2f7642016-12-20 15:10:50 +00003967 return 0;
3968
Marc Zyngiere643d802016-12-20 15:09:31 +00003969 default:
3970 return -EINVAL;
3971 }
3972}
3973
Marc Zyngier20b3d542016-12-20 15:23:22 +00003974static void its_vpe_send_cmd(struct its_vpe *vpe,
3975 void (*cmd)(struct its_device *, u32))
3976{
3977 unsigned long flags;
3978
3979 raw_spin_lock_irqsave(&vpe_proxy.lock, flags);
3980
3981 its_vpe_db_proxy_map_locked(vpe);
3982 cmd(vpe_proxy.dev, vpe->vpe_proxy_event);
3983
3984 raw_spin_unlock_irqrestore(&vpe_proxy.lock, flags);
3985}
3986
Marc Zyngierf6a91da2016-12-20 15:20:38 +00003987static void its_vpe_send_inv(struct irq_data *d)
3988{
3989 struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
Marc Zyngierf6a91da2016-12-20 15:20:38 +00003990
Marc Zyngier926846a2023-06-17 08:32:42 +01003991 if (gic_rdists->has_direct_lpi)
3992 __direct_lpi_inv(d, d->parent_data->hwirq);
3993 else
Marc Zyngier20b3d542016-12-20 15:23:22 +00003994 its_vpe_send_cmd(vpe, its_send_inv);
Marc Zyngierf6a91da2016-12-20 15:20:38 +00003995}
3996
3997static void its_vpe_mask_irq(struct irq_data *d)
3998{
3999 /*
4000 * We need to unmask the LPI, which is described by the parent
4001 * irq_data. Instead of calling into the parent (which won't
4002 * exactly do the right thing, let's simply use the
4003 * parent_data pointer. Yes, I'm naughty.
4004 */
4005 lpi_write_config(d->parent_data, LPI_PROP_ENABLED, 0);
4006 its_vpe_send_inv(d);
4007}
4008
4009static void its_vpe_unmask_irq(struct irq_data *d)
4010{
4011 /* Same hack as above... */
4012 lpi_write_config(d->parent_data, 0, LPI_PROP_ENABLED);
4013 its_vpe_send_inv(d);
4014}
4015
Marc Zyngiere57a3e282017-07-31 14:47:24 +01004016static int its_vpe_set_irqchip_state(struct irq_data *d,
4017 enum irqchip_irq_state which,
4018 bool state)
4019{
4020 struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4021
4022 if (which != IRQCHIP_STATE_PENDING)
4023 return -EINVAL;
4024
4025 if (gic_rdists->has_direct_lpi) {
4026 void __iomem *rdbase;
4027
4028 rdbase = per_cpu_ptr(gic_rdists->rdist, vpe->col_idx)->rd_base;
4029 if (state) {
4030 gic_write_lpir(vpe->vpe_db_lpi, rdbase + GICR_SETLPIR);
4031 } else {
4032 gic_write_lpir(vpe->vpe_db_lpi, rdbase + GICR_CLRLPIR);
Marc Zyngier2f4f0642019-11-08 16:57:56 +00004033 wait_for_syncr(rdbase);
Marc Zyngiere57a3e282017-07-31 14:47:24 +01004034 }
4035 } else {
4036 if (state)
4037 its_vpe_send_cmd(vpe, its_send_int);
4038 else
4039 its_vpe_send_cmd(vpe, its_send_clear);
4040 }
4041
4042 return 0;
4043}
4044
Marc Zyngier7809f702020-03-10 18:49:21 +00004045static int its_vpe_retrigger(struct irq_data *d)
4046{
4047 return !its_vpe_set_irqchip_state(d, IRQCHIP_STATE_PENDING, true);
4048}
4049
Marc Zyngier8fff27a2016-12-20 13:41:55 +00004050static struct irq_chip its_vpe_irq_chip = {
4051 .name = "GICv4-vpe",
Marc Zyngierf6a91da2016-12-20 15:20:38 +00004052 .irq_mask = its_vpe_mask_irq,
4053 .irq_unmask = its_vpe_unmask_irq,
4054 .irq_eoi = irq_chip_eoi_parent,
Marc Zyngier3171a472016-12-20 15:17:28 +00004055 .irq_set_affinity = its_vpe_set_affinity,
Marc Zyngier7809f702020-03-10 18:49:21 +00004056 .irq_retrigger = its_vpe_retrigger,
Marc Zyngiere57a3e282017-07-31 14:47:24 +01004057 .irq_set_irqchip_state = its_vpe_set_irqchip_state,
Marc Zyngiere643d802016-12-20 15:09:31 +00004058 .irq_set_vcpu_affinity = its_vpe_set_vcpu_affinity,
Marc Zyngier8fff27a2016-12-20 13:41:55 +00004059};
4060
Marc Zyngierd97c97b2019-12-24 11:10:33 +00004061static struct its_node *find_4_1_its(void)
4062{
4063 static struct its_node *its = NULL;
4064
4065 if (!its) {
4066 list_for_each_entry(its, &its_nodes, entry) {
4067 if (is_v4_1(its))
4068 return its;
4069 }
4070
4071 /* Oops? */
4072 its = NULL;
4073 }
4074
4075 return its;
4076}
4077
4078static void its_vpe_4_1_send_inv(struct irq_data *d)
4079{
4080 struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4081 struct its_node *its;
4082
4083 /*
4084 * GICv4.1 wants doorbells to be invalidated using the
4085 * INVDB command in order to be broadcast to all RDs. Send
4086 * it to the first valid ITS, and let the HW do its magic.
4087 */
4088 its = find_4_1_its();
4089 if (its)
4090 its_send_invdb(its, vpe);
4091}
4092
4093static void its_vpe_4_1_mask_irq(struct irq_data *d)
4094{
4095 lpi_write_config(d->parent_data, LPI_PROP_ENABLED, 0);
4096 its_vpe_4_1_send_inv(d);
4097}
4098
4099static void its_vpe_4_1_unmask_irq(struct irq_data *d)
4100{
4101 lpi_write_config(d->parent_data, 0, LPI_PROP_ENABLED);
4102 its_vpe_4_1_send_inv(d);
4103}
4104
Marc Zyngier91bf6392019-12-24 11:10:34 +00004105static void its_vpe_4_1_schedule(struct its_vpe *vpe,
4106 struct its_cmd_info *info)
4107{
4108 void __iomem *vlpi_base = gic_data_rdist_vlpi_base();
4109 u64 val = 0;
4110
4111 /* Schedule the VPE */
4112 val |= GICR_VPENDBASER_Valid;
4113 val |= info->g0en ? GICR_VPENDBASER_4_1_VGRP0EN : 0;
4114 val |= info->g1en ? GICR_VPENDBASER_4_1_VGRP1EN : 0;
4115 val |= FIELD_PREP(GICR_VPENDBASER_4_1_VPEID, vpe->vpe_id);
4116
Zenghui Yu5186a6c2020-02-06 15:57:11 +08004117 gicr_write_vpendbaser(val, vlpi_base + GICR_VPENDBASER);
Marc Zyngier91bf6392019-12-24 11:10:34 +00004118}
4119
Marc Zyngiere64fab12019-12-24 11:10:35 +00004120static void its_vpe_4_1_deschedule(struct its_vpe *vpe,
4121 struct its_cmd_info *info)
4122{
4123 void __iomem *vlpi_base = gic_data_rdist_vlpi_base();
4124 u64 val;
4125
4126 if (info->req_db) {
Marc Zyngiera3f574c2020-06-23 10:44:08 +01004127 unsigned long flags;
4128
Marc Zyngiere64fab12019-12-24 11:10:35 +00004129 /*
4130 * vPE is going to block: make the vPE non-resident with
4131 * PendingLast clear and DB set. The GIC guarantees that if
4132 * we read-back PendingLast clear, then a doorbell will be
4133 * delivered when an interrupt comes.
Marc Zyngiera3f574c2020-06-23 10:44:08 +01004134 *
4135 * Note the locking to deal with the concurrent update of
4136 * pending_last from the doorbell interrupt handler that can
4137 * run concurrently.
Marc Zyngiere64fab12019-12-24 11:10:35 +00004138 */
Marc Zyngiera3f574c2020-06-23 10:44:08 +01004139 raw_spin_lock_irqsave(&vpe->vpe_lock, flags);
Marc Zyngiere64fab12019-12-24 11:10:35 +00004140 val = its_clear_vpend_valid(vlpi_base,
4141 GICR_VPENDBASER_PendingLast,
4142 GICR_VPENDBASER_4_1_DB);
4143 vpe->pending_last = !!(val & GICR_VPENDBASER_PendingLast);
Marc Zyngiera3f574c2020-06-23 10:44:08 +01004144 raw_spin_unlock_irqrestore(&vpe->vpe_lock, flags);
Marc Zyngiere64fab12019-12-24 11:10:35 +00004145 } else {
4146 /*
4147 * We're not blocking, so just make the vPE non-resident
4148 * with PendingLast set, indicating that we'll be back.
4149 */
4150 val = its_clear_vpend_valid(vlpi_base,
4151 0,
4152 GICR_VPENDBASER_PendingLast);
4153 vpe->pending_last = true;
4154 }
4155}
4156
Marc Zyngierb4a4bd02019-12-24 11:10:36 +00004157static void its_vpe_4_1_invall(struct its_vpe *vpe)
4158{
4159 void __iomem *rdbase;
Zenghui Yu3af95712020-07-20 17:23:28 +08004160 unsigned long flags;
Marc Zyngierb4a4bd02019-12-24 11:10:36 +00004161 u64 val;
Zenghui Yu3af95712020-07-20 17:23:28 +08004162 int cpu;
Marc Zyngierb4a4bd02019-12-24 11:10:36 +00004163
4164 val = GICR_INVALLR_V;
4165 val |= FIELD_PREP(GICR_INVALLR_VPEID, vpe->vpe_id);
4166
4167 /* Target the redistributor this vPE is currently known on */
Zenghui Yu3af95712020-07-20 17:23:28 +08004168 cpu = vpe_to_cpuid_lock(vpe, &flags);
4169 raw_spin_lock(&gic_data_rdist_cpu(cpu)->rd_lock);
4170 rdbase = per_cpu_ptr(gic_rdists->rdist, cpu)->rd_base;
Marc Zyngierb4a4bd02019-12-24 11:10:36 +00004171 gic_write_lpir(val, rdbase + GICR_INVALLR);
Zenghui Yub978c252020-03-04 20:33:11 +00004172
4173 wait_for_syncr(rdbase);
Zenghui Yu3af95712020-07-20 17:23:28 +08004174 raw_spin_unlock(&gic_data_rdist_cpu(cpu)->rd_lock);
4175 vpe_to_cpuid_unlock(vpe, flags);
Marc Zyngierb4a4bd02019-12-24 11:10:36 +00004176}
4177
Marc Zyngier29c647f2019-12-24 11:10:32 +00004178static int its_vpe_4_1_set_vcpu_affinity(struct irq_data *d, void *vcpu_info)
4179{
Marc Zyngier91bf6392019-12-24 11:10:34 +00004180 struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
Marc Zyngier29c647f2019-12-24 11:10:32 +00004181 struct its_cmd_info *info = vcpu_info;
4182
4183 switch (info->cmd_type) {
4184 case SCHEDULE_VPE:
Marc Zyngier91bf6392019-12-24 11:10:34 +00004185 its_vpe_4_1_schedule(vpe, info);
Marc Zyngier29c647f2019-12-24 11:10:32 +00004186 return 0;
4187
4188 case DESCHEDULE_VPE:
Marc Zyngiere64fab12019-12-24 11:10:35 +00004189 its_vpe_4_1_deschedule(vpe, info);
Marc Zyngier29c647f2019-12-24 11:10:32 +00004190 return 0;
4191
Shenming Lu57e3ceb2020-11-28 22:18:57 +08004192 case COMMIT_VPE:
4193 its_wait_vpt_parse_complete();
4194 return 0;
4195
Marc Zyngier29c647f2019-12-24 11:10:32 +00004196 case INVALL_VPE:
Marc Zyngierb4a4bd02019-12-24 11:10:36 +00004197 its_vpe_4_1_invall(vpe);
Marc Zyngier29c647f2019-12-24 11:10:32 +00004198 return 0;
4199
4200 default:
4201 return -EINVAL;
4202 }
4203}
4204
4205static struct irq_chip its_vpe_4_1_irq_chip = {
4206 .name = "GICv4.1-vpe",
Marc Zyngierd97c97b2019-12-24 11:10:33 +00004207 .irq_mask = its_vpe_4_1_mask_irq,
4208 .irq_unmask = its_vpe_4_1_unmask_irq,
Marc Zyngier29c647f2019-12-24 11:10:32 +00004209 .irq_eoi = irq_chip_eoi_parent,
4210 .irq_set_affinity = its_vpe_set_affinity,
4211 .irq_set_vcpu_affinity = its_vpe_4_1_set_vcpu_affinity,
4212};
4213
Marc Zyngiere252cf82020-03-04 20:33:16 +00004214static void its_configure_sgi(struct irq_data *d, bool clear)
4215{
4216 struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4217 struct its_cmd_desc desc;
4218
4219 desc.its_vsgi_cmd.vpe = vpe;
4220 desc.its_vsgi_cmd.sgi = d->hwirq;
4221 desc.its_vsgi_cmd.priority = vpe->sgi_config[d->hwirq].priority;
4222 desc.its_vsgi_cmd.enable = vpe->sgi_config[d->hwirq].enabled;
4223 desc.its_vsgi_cmd.group = vpe->sgi_config[d->hwirq].group;
4224 desc.its_vsgi_cmd.clear = clear;
4225
4226 /*
4227 * GICv4.1 allows us to send VSGI commands to any ITS as long as the
4228 * destination VPE is mapped there. Since we map them eagerly at
4229 * activation time, we're pretty sure the first GICv4.1 ITS will do.
4230 */
4231 its_send_single_vcommand(find_4_1_its(), its_build_vsgi_cmd, &desc);
4232}
4233
Marc Zyngierb4e8d642020-03-04 20:33:17 +00004234static void its_sgi_mask_irq(struct irq_data *d)
4235{
4236 struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4237
4238 vpe->sgi_config[d->hwirq].enabled = false;
4239 its_configure_sgi(d, false);
4240}
4241
4242static void its_sgi_unmask_irq(struct irq_data *d)
4243{
4244 struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4245
4246 vpe->sgi_config[d->hwirq].enabled = true;
4247 its_configure_sgi(d, false);
4248}
4249
Marc Zyngier166cba72020-03-04 20:33:15 +00004250static int its_sgi_set_affinity(struct irq_data *d,
4251 const struct cpumask *mask_val,
4252 bool force)
4253{
4254 /*
4255 * There is no notion of affinity for virtual SGIs, at least
Ingo Molnara359f752021-03-22 04:21:30 +01004256 * not on the host (since they can only be targeting a vPE).
Marc Zyngier166cba72020-03-04 20:33:15 +00004257 * Tell the kernel we've done whatever it asked for.
4258 */
Marc Zyngier4b2dfe12020-04-10 12:11:39 +01004259 irq_data_update_effective_affinity(d, mask_val);
Marc Zyngier166cba72020-03-04 20:33:15 +00004260 return IRQ_SET_MASK_OK;
4261}
4262
Marc Zyngier7017ff02020-03-04 20:33:18 +00004263static int its_sgi_set_irqchip_state(struct irq_data *d,
4264 enum irqchip_irq_state which,
4265 bool state)
4266{
4267 if (which != IRQCHIP_STATE_PENDING)
4268 return -EINVAL;
4269
4270 if (state) {
4271 struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4272 struct its_node *its = find_4_1_its();
4273 u64 val;
4274
4275 val = FIELD_PREP(GITS_SGIR_VPEID, vpe->vpe_id);
4276 val |= FIELD_PREP(GITS_SGIR_VINTID, d->hwirq);
4277 writeq_relaxed(val, its->sgir_base + GITS_SGIR - SZ_128K);
4278 } else {
4279 its_configure_sgi(d, true);
4280 }
4281
4282 return 0;
4283}
4284
4285static int its_sgi_get_irqchip_state(struct irq_data *d,
4286 enum irqchip_irq_state which, bool *val)
4287{
4288 struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4289 void __iomem *base;
4290 unsigned long flags;
4291 u32 count = 1000000; /* 1s! */
4292 u32 status;
4293 int cpu;
4294
4295 if (which != IRQCHIP_STATE_PENDING)
4296 return -EINVAL;
4297
4298 /*
4299 * Locking galore! We can race against two different events:
4300 *
Ingo Molnara359f752021-03-22 04:21:30 +01004301 * - Concurrent vPE affinity change: we must make sure it cannot
Marc Zyngier7017ff02020-03-04 20:33:18 +00004302 * happen, or we'll talk to the wrong redistributor. This is
4303 * identical to what happens with vLPIs.
4304 *
4305 * - Concurrent VSGIPENDR access: As it involves accessing two
4306 * MMIO registers, this must be made atomic one way or another.
4307 */
4308 cpu = vpe_to_cpuid_lock(vpe, &flags);
4309 raw_spin_lock(&gic_data_rdist_cpu(cpu)->rd_lock);
4310 base = gic_data_rdist_cpu(cpu)->rd_base + SZ_128K;
4311 writel_relaxed(vpe->vpe_id, base + GICR_VSGIR);
4312 do {
4313 status = readl_relaxed(base + GICR_VSGIPENDR);
4314 if (!(status & GICR_VSGIPENDR_BUSY))
4315 goto out;
4316
4317 count--;
4318 if (!count) {
4319 pr_err_ratelimited("Unable to get SGI status\n");
4320 goto out;
4321 }
4322 cpu_relax();
4323 udelay(1);
4324 } while (count);
4325
4326out:
4327 raw_spin_unlock(&gic_data_rdist_cpu(cpu)->rd_lock);
4328 vpe_to_cpuid_unlock(vpe, flags);
4329
4330 if (!count)
4331 return -ENXIO;
4332
4333 *val = !!(status & (1 << d->hwirq));
4334
4335 return 0;
4336}
4337
Marc Zyngier05d32df2020-03-04 20:33:19 +00004338static int its_sgi_set_vcpu_affinity(struct irq_data *d, void *vcpu_info)
4339{
4340 struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4341 struct its_cmd_info *info = vcpu_info;
4342
4343 switch (info->cmd_type) {
4344 case PROP_UPDATE_VSGI:
4345 vpe->sgi_config[d->hwirq].priority = info->priority;
4346 vpe->sgi_config[d->hwirq].group = info->group;
4347 its_configure_sgi(d, false);
4348 return 0;
4349
4350 default:
4351 return -EINVAL;
4352 }
4353}
4354
Marc Zyngier166cba72020-03-04 20:33:15 +00004355static struct irq_chip its_sgi_irq_chip = {
4356 .name = "GICv4.1-sgi",
Marc Zyngierb4e8d642020-03-04 20:33:17 +00004357 .irq_mask = its_sgi_mask_irq,
4358 .irq_unmask = its_sgi_unmask_irq,
Marc Zyngier166cba72020-03-04 20:33:15 +00004359 .irq_set_affinity = its_sgi_set_affinity,
Marc Zyngier7017ff02020-03-04 20:33:18 +00004360 .irq_set_irqchip_state = its_sgi_set_irqchip_state,
4361 .irq_get_irqchip_state = its_sgi_get_irqchip_state,
Marc Zyngier05d32df2020-03-04 20:33:19 +00004362 .irq_set_vcpu_affinity = its_sgi_set_vcpu_affinity,
Marc Zyngier166cba72020-03-04 20:33:15 +00004363};
4364
4365static int its_sgi_irq_domain_alloc(struct irq_domain *domain,
4366 unsigned int virq, unsigned int nr_irqs,
4367 void *args)
4368{
4369 struct its_vpe *vpe = args;
4370 int i;
4371
4372 /* Yes, we do want 16 SGIs */
4373 WARN_ON(nr_irqs != 16);
4374
4375 for (i = 0; i < 16; i++) {
4376 vpe->sgi_config[i].priority = 0;
4377 vpe->sgi_config[i].enabled = false;
4378 vpe->sgi_config[i].group = false;
4379
4380 irq_domain_set_hwirq_and_chip(domain, virq + i, i,
4381 &its_sgi_irq_chip, vpe);
4382 irq_set_status_flags(virq + i, IRQ_DISABLE_UNLAZY);
4383 }
4384
4385 return 0;
4386}
4387
4388static void its_sgi_irq_domain_free(struct irq_domain *domain,
4389 unsigned int virq,
4390 unsigned int nr_irqs)
4391{
4392 /* Nothing to do */
4393}
4394
4395static int its_sgi_irq_domain_activate(struct irq_domain *domain,
4396 struct irq_data *d, bool reserve)
4397{
Marc Zyngiere252cf82020-03-04 20:33:16 +00004398 /* Write out the initial SGI configuration */
4399 its_configure_sgi(d, false);
Marc Zyngier166cba72020-03-04 20:33:15 +00004400 return 0;
4401}
4402
4403static void its_sgi_irq_domain_deactivate(struct irq_domain *domain,
4404 struct irq_data *d)
4405{
Marc Zyngiere252cf82020-03-04 20:33:16 +00004406 struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4407
4408 /*
4409 * The VSGI command is awkward:
4410 *
4411 * - To change the configuration, CLEAR must be set to false,
4412 * leaving the pending bit unchanged.
4413 * - To clear the pending bit, CLEAR must be set to true, leaving
4414 * the configuration unchanged.
4415 *
4416 * You just can't do both at once, hence the two commands below.
4417 */
4418 vpe->sgi_config[d->hwirq].enabled = false;
4419 its_configure_sgi(d, false);
4420 its_configure_sgi(d, true);
Marc Zyngier166cba72020-03-04 20:33:15 +00004421}
4422
4423static const struct irq_domain_ops its_sgi_domain_ops = {
4424 .alloc = its_sgi_irq_domain_alloc,
4425 .free = its_sgi_irq_domain_free,
4426 .activate = its_sgi_irq_domain_activate,
4427 .deactivate = its_sgi_irq_domain_deactivate,
4428};
4429
Marc Zyngier7d75bbb2016-12-20 13:55:54 +00004430static int its_vpe_id_alloc(void)
4431{
Christophe JAILLETee4c1592024-01-14 09:52:39 +01004432 return ida_alloc_max(&its_vpeid_ida, ITS_MAX_VPEID - 1, GFP_KERNEL);
Marc Zyngier7d75bbb2016-12-20 13:55:54 +00004433}
4434
4435static void its_vpe_id_free(u16 id)
4436{
Christophe JAILLETee4c1592024-01-14 09:52:39 +01004437 ida_free(&its_vpeid_ida, id);
Marc Zyngier7d75bbb2016-12-20 13:55:54 +00004438}
4439
4440static int its_vpe_init(struct its_vpe *vpe)
4441{
4442 struct page *vpt_page;
4443 int vpe_id;
4444
4445 /* Allocate vpe_id */
4446 vpe_id = its_vpe_id_alloc();
4447 if (vpe_id < 0)
4448 return vpe_id;
4449
4450 /* Allocate VPT */
4451 vpt_page = its_allocate_pending_table(GFP_KERNEL);
4452 if (!vpt_page) {
4453 its_vpe_id_free(vpe_id);
4454 return -ENOMEM;
4455 }
4456
4457 if (!its_alloc_vpe_table(vpe_id)) {
4458 its_vpe_id_free(vpe_id);
Nianyao Tang34f8eb92019-07-26 17:32:57 +08004459 its_free_pending_table(vpt_page);
Marc Zyngier7d75bbb2016-12-20 13:55:54 +00004460 return -ENOMEM;
4461 }
4462
Marc Zyngierf3a059212020-03-04 20:33:10 +00004463 raw_spin_lock_init(&vpe->vpe_lock);
Marc Zyngier7d75bbb2016-12-20 13:55:54 +00004464 vpe->vpe_id = vpe_id;
4465 vpe->vpt_page = vpt_page;
Marc Zyngier64edfaa2019-12-24 11:10:29 +00004466 if (gic_rdists->has_rvpeid)
4467 atomic_set(&vpe->vmapp_count, 0);
4468 else
4469 vpe->vpe_proxy_event = -1;
Marc Zyngier7d75bbb2016-12-20 13:55:54 +00004470
4471 return 0;
4472}
4473
4474static void its_vpe_teardown(struct its_vpe *vpe)
4475{
Marc Zyngier20b3d542016-12-20 15:23:22 +00004476 its_vpe_db_proxy_unmap(vpe);
Marc Zyngier7d75bbb2016-12-20 13:55:54 +00004477 its_vpe_id_free(vpe->vpe_id);
4478 its_free_pending_table(vpe->vpt_page);
4479}
4480
4481static void its_vpe_irq_domain_free(struct irq_domain *domain,
4482 unsigned int virq,
4483 unsigned int nr_irqs)
4484{
4485 struct its_vm *vm = domain->host_data;
4486 int i;
4487
4488 irq_domain_free_irqs_parent(domain, virq, nr_irqs);
4489
4490 for (i = 0; i < nr_irqs; i++) {
4491 struct irq_data *data = irq_domain_get_irq_data(domain,
4492 virq + i);
4493 struct its_vpe *vpe = irq_data_get_irq_chip_data(data);
4494
4495 BUG_ON(vm != vpe->its_vm);
4496
4497 clear_bit(data->hwirq, vm->db_bitmap);
4498 its_vpe_teardown(vpe);
4499 irq_domain_reset_irq_data(data);
4500 }
4501
4502 if (bitmap_empty(vm->db_bitmap, vm->nr_db_lpis)) {
Marc Zyngier38dd7c42018-05-27 17:03:03 +01004503 its_lpi_free(vm->db_bitmap, vm->db_lpi_base, vm->nr_db_lpis);
Marc Zyngier7d75bbb2016-12-20 13:55:54 +00004504 its_free_prop_table(vm->vprop_page);
4505 }
4506}
4507
4508static int its_vpe_irq_domain_alloc(struct irq_domain *domain, unsigned int virq,
4509 unsigned int nr_irqs, void *args)
4510{
Marc Zyngier29c647f2019-12-24 11:10:32 +00004511 struct irq_chip *irqchip = &its_vpe_irq_chip;
Marc Zyngier7d75bbb2016-12-20 13:55:54 +00004512 struct its_vm *vm = args;
4513 unsigned long *bitmap;
4514 struct page *vprop_page;
4515 int base, nr_ids, i, err = 0;
4516
Marc Zyngier38dd7c42018-05-27 17:03:03 +01004517 bitmap = its_lpi_alloc(roundup_pow_of_two(nr_irqs), &base, &nr_ids);
Marc Zyngier7d75bbb2016-12-20 13:55:54 +00004518 if (!bitmap)
4519 return -ENOMEM;
4520
4521 if (nr_ids < nr_irqs) {
Marc Zyngier38dd7c42018-05-27 17:03:03 +01004522 its_lpi_free(bitmap, base, nr_ids);
Marc Zyngier7d75bbb2016-12-20 13:55:54 +00004523 return -ENOMEM;
4524 }
4525
4526 vprop_page = its_allocate_prop_table(GFP_KERNEL);
4527 if (!vprop_page) {
Marc Zyngier38dd7c42018-05-27 17:03:03 +01004528 its_lpi_free(bitmap, base, nr_ids);
Marc Zyngier7d75bbb2016-12-20 13:55:54 +00004529 return -ENOMEM;
4530 }
4531
4532 vm->db_bitmap = bitmap;
4533 vm->db_lpi_base = base;
4534 vm->nr_db_lpis = nr_ids;
4535 vm->vprop_page = vprop_page;
Marc Zyngierf0eb1542024-07-05 10:31:54 +01004536 raw_spin_lock_init(&vm->vmapp_lock);
Marc Zyngier7d75bbb2016-12-20 13:55:54 +00004537
Marc Zyngier29c647f2019-12-24 11:10:32 +00004538 if (gic_rdists->has_rvpeid)
4539 irqchip = &its_vpe_4_1_irq_chip;
4540
Marc Zyngier7d75bbb2016-12-20 13:55:54 +00004541 for (i = 0; i < nr_irqs; i++) {
4542 vm->vpes[i]->vpe_db_lpi = base + i;
4543 err = its_vpe_init(vm->vpes[i]);
4544 if (err)
4545 break;
4546 err = its_irq_gic_domain_alloc(domain, virq + i,
4547 vm->vpes[i]->vpe_db_lpi);
4548 if (err)
4549 break;
4550 irq_domain_set_hwirq_and_chip(domain, virq + i, i,
Marc Zyngier29c647f2019-12-24 11:10:32 +00004551 irqchip, vm->vpes[i]);
Marc Zyngier7d75bbb2016-12-20 13:55:54 +00004552 set_bit(i, bitmap);
James Gowans8f4b5892023-06-08 14:00:21 +02004553 irqd_set_resend_when_in_progress(irq_get_irq_data(virq + i));
Marc Zyngier7d75bbb2016-12-20 13:55:54 +00004554 }
4555
Guanrui Huangc26591a2024-04-18 14:10:52 +08004556 if (err)
4557 its_vpe_irq_domain_free(domain, virq, i);
Marc Zyngier7d75bbb2016-12-20 13:55:54 +00004558
4559 return err;
4560}
4561
Thomas Gleixner72491642017-09-13 23:29:10 +02004562static int its_vpe_irq_domain_activate(struct irq_domain *domain,
Thomas Gleixner702cb0a2017-12-29 16:59:06 +01004563 struct irq_data *d, bool reserve)
Marc Zyngiereb781922016-12-20 14:47:05 +00004564{
4565 struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
Marc Zyngier40619a22017-10-08 15:16:09 +01004566 struct its_node *its;
Marc Zyngiereb781922016-12-20 14:47:05 +00004567
Marc Zyngier7d2c2042024-07-05 10:31:53 +01004568 /* Map the VPE to the first possible CPU */
4569 vpe->col_idx = cpumask_first(cpu_online_mask);
4570 irq_data_update_effective_affinity(d, cpumask_of(vpe->col_idx));
4571
Marc Zyngier009384b2020-03-04 20:33:23 +00004572 /*
4573 * If we use the list map, we issue VMAPP on demand... Unless
4574 * we're on a GICv4.1 and we eagerly map the VPE on all ITSs
4575 * so that VSGIs can work.
4576 */
4577 if (!gic_requires_eager_mapping())
Marc Zyngier6ef930f2017-11-07 10:04:38 +00004578 return 0;
Marc Zyngiereb781922016-12-20 14:47:05 +00004579
Marc Zyngier40619a22017-10-08 15:16:09 +01004580 list_for_each_entry(its, &its_nodes, entry) {
Marc Zyngier0dd57fe2019-11-08 16:57:58 +00004581 if (!is_v4(its))
Marc Zyngier40619a22017-10-08 15:16:09 +01004582 continue;
4583
Marc Zyngier75fd9512017-10-08 18:46:39 +01004584 its_send_vmapp(its, vpe, true);
Marc Zyngier40619a22017-10-08 15:16:09 +01004585 its_send_vinvall(its, vpe);
4586 }
4587
Thomas Gleixner72491642017-09-13 23:29:10 +02004588 return 0;
Marc Zyngiereb781922016-12-20 14:47:05 +00004589}
4590
4591static void its_vpe_irq_domain_deactivate(struct irq_domain *domain,
4592 struct irq_data *d)
4593{
4594 struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
Marc Zyngier75fd9512017-10-08 18:46:39 +01004595 struct its_node *its;
Marc Zyngiereb781922016-12-20 14:47:05 +00004596
Marc Zyngier2247e1b2017-10-08 18:50:36 +01004597 /*
Marc Zyngier009384b2020-03-04 20:33:23 +00004598 * If we use the list map on GICv4.0, we unmap the VPE once no
4599 * VLPIs are associated with the VM.
Marc Zyngier2247e1b2017-10-08 18:50:36 +01004600 */
Marc Zyngier009384b2020-03-04 20:33:23 +00004601 if (!gic_requires_eager_mapping())
Marc Zyngier2247e1b2017-10-08 18:50:36 +01004602 return;
4603
Marc Zyngier75fd9512017-10-08 18:46:39 +01004604 list_for_each_entry(its, &its_nodes, entry) {
Marc Zyngier0dd57fe2019-11-08 16:57:58 +00004605 if (!is_v4(its))
Marc Zyngier75fd9512017-10-08 18:46:39 +01004606 continue;
4607
4608 its_send_vmapp(its, vpe, false);
4609 }
Marc Zyngier301beaf2021-03-22 14:01:53 +08004610
4611 /*
4612 * There may be a direct read to the VPT after unmapping the
4613 * vPE, to guarantee the validity of this, we make the VPT
4614 * memory coherent with the CPU caches here.
4615 */
4616 if (find_4_1_its() && !atomic_read(&vpe->vmapp_count))
4617 gic_flush_dcache_to_poc(page_address(vpe->vpt_page),
4618 LPI_PENDBASE_SZ);
Marc Zyngiereb781922016-12-20 14:47:05 +00004619}
4620
Marc Zyngier8fff27a2016-12-20 13:41:55 +00004621static const struct irq_domain_ops its_vpe_domain_ops = {
Marc Zyngier7d75bbb2016-12-20 13:55:54 +00004622 .alloc = its_vpe_irq_domain_alloc,
4623 .free = its_vpe_irq_domain_free,
Marc Zyngiereb781922016-12-20 14:47:05 +00004624 .activate = its_vpe_irq_domain_activate,
4625 .deactivate = its_vpe_irq_domain_deactivate,
Marc Zyngier8fff27a2016-12-20 13:41:55 +00004626};
4627
Yun Wu4559fbb2015-03-06 16:37:50 +00004628static int its_force_quiescent(void __iomem *base)
4629{
4630 u32 count = 1000000; /* 1s */
4631 u32 val;
4632
4633 val = readl_relaxed(base + GITS_CTLR);
David Daney7611da82016-08-18 15:41:58 -07004634 /*
4635 * GIC architecture specification requires the ITS to be both
4636 * disabled and quiescent for writes to GITS_BASER<n> or
4637 * GITS_CBASER to not have UNPREDICTABLE results.
4638 */
4639 if ((val & GITS_CTLR_QUIESCENT) && !(val & GITS_CTLR_ENABLE))
Yun Wu4559fbb2015-03-06 16:37:50 +00004640 return 0;
4641
4642 /* Disable the generation of all interrupts to this ITS */
Marc Zyngierd51c4b42017-06-27 21:24:25 +01004643 val &= ~(GITS_CTLR_ENABLE | GITS_CTLR_ImDe);
Yun Wu4559fbb2015-03-06 16:37:50 +00004644 writel_relaxed(val, base + GITS_CTLR);
4645
4646 /* Poll GITS_CTLR and wait until ITS becomes quiescent */
4647 while (1) {
4648 val = readl_relaxed(base + GITS_CTLR);
4649 if (val & GITS_CTLR_QUIESCENT)
4650 return 0;
4651
4652 count--;
4653 if (!count)
4654 return -EBUSY;
4655
4656 cpu_relax();
4657 udelay(1);
4658 }
4659}
4660
Ard Biesheuvel9d111d42017-10-17 17:55:55 +01004661static bool __maybe_unused its_enable_quirk_cavium_22375(void *data)
Robert Richter94100972015-09-21 22:58:38 +02004662{
4663 struct its_node *its = data;
4664
Marc Zyngier576a8342019-11-08 16:58:00 +00004665 /* erratum 22375: only alloc 8MB table size (20 bits) */
4666 its->typer &= ~GITS_TYPER_DEVBITS;
4667 its->typer |= FIELD_PREP(GITS_TYPER_DEVBITS, 20 - 1);
Robert Richter94100972015-09-21 22:58:38 +02004668 its->flags |= ITS_FLAGS_WORKAROUND_CAVIUM_22375;
Ard Biesheuvel9d111d42017-10-17 17:55:55 +01004669
4670 return true;
Robert Richter94100972015-09-21 22:58:38 +02004671}
4672
Ard Biesheuvel9d111d42017-10-17 17:55:55 +01004673static bool __maybe_unused its_enable_quirk_cavium_23144(void *data)
Ganapatrao Kulkarnifbf8f402016-05-25 15:29:20 +02004674{
4675 struct its_node *its = data;
4676
4677 its->flags |= ITS_FLAGS_WORKAROUND_CAVIUM_23144;
Ard Biesheuvel9d111d42017-10-17 17:55:55 +01004678
4679 return true;
Ganapatrao Kulkarnifbf8f402016-05-25 15:29:20 +02004680}
4681
Ard Biesheuvel9d111d42017-10-17 17:55:55 +01004682static bool __maybe_unused its_enable_quirk_qdf2400_e0065(void *data)
Shanker Donthineni90922a22017-03-07 08:20:38 -06004683{
4684 struct its_node *its = data;
4685
4686 /* On QDF2400, the size of the ITE is 16Bytes */
Marc Zyngierffedbf02019-11-08 16:57:59 +00004687 its->typer &= ~GITS_TYPER_ITT_ENTRY_SIZE;
4688 its->typer |= FIELD_PREP(GITS_TYPER_ITT_ENTRY_SIZE, 16 - 1);
Ard Biesheuvel9d111d42017-10-17 17:55:55 +01004689
4690 return true;
Shanker Donthineni90922a22017-03-07 08:20:38 -06004691}
4692
Ard Biesheuvel558b0162017-10-17 17:55:56 +01004693static u64 its_irq_get_msi_base_pre_its(struct its_device *its_dev)
4694{
4695 struct its_node *its = its_dev->its;
4696
4697 /*
4698 * The Socionext Synquacer SoC has a so-called 'pre-ITS',
4699 * which maps 32-bit writes targeted at a separate window of
4700 * size '4 << device_id_bits' onto writes to GITS_TRANSLATER
4701 * with device ID taken from bits [device_id_bits + 1:2] of
4702 * the window offset.
4703 */
4704 return its->pre_its_base + (its_dev->device_id << 2);
4705}
4706
4707static bool __maybe_unused its_enable_quirk_socionext_synquacer(void *data)
4708{
4709 struct its_node *its = data;
4710 u32 pre_its_window[2];
4711 u32 ids;
4712
4713 if (!fwnode_property_read_u32_array(its->fwnode_handle,
4714 "socionext,synquacer-pre-its",
4715 pre_its_window,
4716 ARRAY_SIZE(pre_its_window))) {
4717
4718 its->pre_its_base = pre_its_window[0];
4719 its->get_msi_base = its_irq_get_msi_base_pre_its;
4720
4721 ids = ilog2(pre_its_window[1]) - 2;
Marc Zyngier576a8342019-11-08 16:58:00 +00004722 if (device_ids(its) > ids) {
4723 its->typer &= ~GITS_TYPER_DEVBITS;
4724 its->typer |= FIELD_PREP(GITS_TYPER_DEVBITS, ids - 1);
4725 }
Ard Biesheuvel558b0162017-10-17 17:55:56 +01004726
4727 /* the pre-ITS breaks isolation, so disable MSI remapping */
Jason Gunthorpedcb83f62022-11-28 20:12:43 -04004728 its->msi_domain_flags &= ~IRQ_DOMAIN_FLAG_ISOLATED_MSI;
Ard Biesheuvel558b0162017-10-17 17:55:56 +01004729 return true;
4730 }
4731 return false;
4732}
4733
Marc Zyngier5c9a8822017-07-28 21:20:37 +01004734static bool __maybe_unused its_enable_quirk_hip07_161600802(void *data)
4735{
4736 struct its_node *its = data;
4737
4738 /*
4739 * Hip07 insists on using the wrong address for the VLPI
4740 * page. Trick it into doing the right thing...
4741 */
4742 its->vlpi_redist_offset = SZ_128K;
4743 return true;
Marc Zyngiercc2d3212014-11-24 14:35:11 +00004744}
Marc Zyngier4c21f3c2014-11-24 14:35:17 +00004745
Sebastian Reichela8707f52023-04-18 16:21:08 +02004746static bool __maybe_unused its_enable_rk3588001(void *data)
4747{
4748 struct its_node *its = data;
4749
Sebastian Reichel567f67a2023-07-03 18:41:29 +02004750 if (!of_machine_is_compatible("rockchip,rk3588") &&
4751 !of_machine_is_compatible("rockchip,rk3588s"))
Sebastian Reichela8707f52023-04-18 16:21:08 +02004752 return false;
4753
4754 its->flags |= ITS_FLAGS_FORCE_NON_SHAREABLE;
4755 gic_rdists->flags |= RDIST_FLAGS_FORCE_NON_SHAREABLE;
4756
4757 return true;
4758}
4759
Lorenzo Pieralisi3a0fff02023-10-06 14:59:26 +02004760static bool its_set_non_coherent(void *data)
4761{
4762 struct its_node *its = data;
4763
4764 its->flags |= ITS_FLAGS_FORCE_NON_SHAREABLE;
4765 return true;
4766}
4767
Robert Richter67510cc2015-09-21 22:58:37 +02004768static const struct gic_quirk its_quirks[] = {
Robert Richter94100972015-09-21 22:58:38 +02004769#ifdef CONFIG_CAVIUM_ERRATUM_22375
4770 {
4771 .desc = "ITS: Cavium errata 22375, 24313",
4772 .iidr = 0xa100034c, /* ThunderX pass 1.x */
4773 .mask = 0xffff0fff,
4774 .init = its_enable_quirk_cavium_22375,
4775 },
4776#endif
Ganapatrao Kulkarnifbf8f402016-05-25 15:29:20 +02004777#ifdef CONFIG_CAVIUM_ERRATUM_23144
4778 {
4779 .desc = "ITS: Cavium erratum 23144",
4780 .iidr = 0xa100034c, /* ThunderX pass 1.x */
4781 .mask = 0xffff0fff,
4782 .init = its_enable_quirk_cavium_23144,
4783 },
4784#endif
Shanker Donthineni90922a22017-03-07 08:20:38 -06004785#ifdef CONFIG_QCOM_QDF2400_ERRATUM_0065
4786 {
4787 .desc = "ITS: QDF2400 erratum 0065",
4788 .iidr = 0x00001070, /* QDF2400 ITS rev 1.x */
4789 .mask = 0xffffffff,
4790 .init = its_enable_quirk_qdf2400_e0065,
4791 },
4792#endif
Ard Biesheuvel558b0162017-10-17 17:55:56 +01004793#ifdef CONFIG_SOCIONEXT_SYNQUACER_PREITS
4794 {
4795 /*
4796 * The Socionext Synquacer SoC incorporates ARM's own GIC-500
4797 * implementation, but with a 'pre-ITS' added that requires
4798 * special handling in software.
4799 */
4800 .desc = "ITS: Socionext Synquacer pre-ITS",
4801 .iidr = 0x0001143b,
4802 .mask = 0xffffffff,
4803 .init = its_enable_quirk_socionext_synquacer,
4804 },
4805#endif
Marc Zyngier5c9a8822017-07-28 21:20:37 +01004806#ifdef CONFIG_HISILICON_ERRATUM_161600802
4807 {
4808 .desc = "ITS: Hip07 erratum 161600802",
4809 .iidr = 0x00000004,
4810 .mask = 0xffffffff,
4811 .init = its_enable_quirk_hip07_161600802,
4812 },
4813#endif
Sebastian Reichela8707f52023-04-18 16:21:08 +02004814#ifdef CONFIG_ROCKCHIP_ERRATUM_3588001
4815 {
4816 .desc = "ITS: Rockchip erratum RK3588001",
4817 .iidr = 0x0201743b,
4818 .mask = 0xffffffff,
4819 .init = its_enable_rk3588001,
4820 },
4821#endif
Robert Richter67510cc2015-09-21 22:58:37 +02004822 {
Lorenzo Pieralisi3a0fff02023-10-06 14:59:26 +02004823 .desc = "ITS: non-coherent attribute",
4824 .property = "dma-noncoherent",
4825 .init = its_set_non_coherent,
4826 },
4827 {
Robert Richter67510cc2015-09-21 22:58:37 +02004828 }
4829};
4830
4831static void its_enable_quirks(struct its_node *its)
4832{
4833 u32 iidr = readl_relaxed(its->base + GITS_IIDR);
4834
4835 gic_enable_quirks(iidr, its_quirks, its);
Lorenzo Pieralisi3a0fff02023-10-06 14:59:26 +02004836
4837 if (is_of_node(its->fwnode_handle))
4838 gic_enable_of_quirks(to_of_node(its->fwnode_handle),
4839 its_quirks, its);
Robert Richter67510cc2015-09-21 22:58:37 +02004840}
4841
Derek Basehoredba0bc72018-02-28 21:48:18 -08004842static int its_save_disable(void)
4843{
4844 struct its_node *its;
4845 int err = 0;
4846
Sebastian Andrzej Siewiora8db7452018-07-18 17:42:04 +02004847 raw_spin_lock(&its_lock);
Derek Basehoredba0bc72018-02-28 21:48:18 -08004848 list_for_each_entry(its, &its_nodes, entry) {
4849 void __iomem *base;
4850
Derek Basehoredba0bc72018-02-28 21:48:18 -08004851 base = its->base;
4852 its->ctlr_save = readl_relaxed(base + GITS_CTLR);
4853 err = its_force_quiescent(base);
4854 if (err) {
4855 pr_err("ITS@%pa: failed to quiesce: %d\n",
4856 &its->phys_base, err);
4857 writel_relaxed(its->ctlr_save, base + GITS_CTLR);
4858 goto err;
4859 }
4860
4861 its->cbaser_save = gits_read_cbaser(base + GITS_CBASER);
4862 }
4863
4864err:
4865 if (err) {
4866 list_for_each_entry_continue_reverse(its, &its_nodes, entry) {
4867 void __iomem *base;
4868
Derek Basehoredba0bc72018-02-28 21:48:18 -08004869 base = its->base;
4870 writel_relaxed(its->ctlr_save, base + GITS_CTLR);
4871 }
4872 }
Sebastian Andrzej Siewiora8db7452018-07-18 17:42:04 +02004873 raw_spin_unlock(&its_lock);
Derek Basehoredba0bc72018-02-28 21:48:18 -08004874
4875 return err;
4876}
4877
4878static void its_restore_enable(void)
4879{
4880 struct its_node *its;
4881 int ret;
4882
Sebastian Andrzej Siewiora8db7452018-07-18 17:42:04 +02004883 raw_spin_lock(&its_lock);
Derek Basehoredba0bc72018-02-28 21:48:18 -08004884 list_for_each_entry(its, &its_nodes, entry) {
4885 void __iomem *base;
4886 int i;
4887
Derek Basehoredba0bc72018-02-28 21:48:18 -08004888 base = its->base;
4889
4890 /*
4891 * Make sure that the ITS is disabled. If it fails to quiesce,
4892 * don't restore it since writing to CBASER or BASER<n>
4893 * registers is undefined according to the GIC v3 ITS
4894 * Specification.
Xu Qiang74cde1a2020-11-07 10:42:26 +00004895 *
4896 * Firmware resuming with the ITS enabled is terminally broken.
Derek Basehoredba0bc72018-02-28 21:48:18 -08004897 */
Xu Qiang74cde1a2020-11-07 10:42:26 +00004898 WARN_ON(readl_relaxed(base + GITS_CTLR) & GITS_CTLR_ENABLE);
Derek Basehoredba0bc72018-02-28 21:48:18 -08004899 ret = its_force_quiescent(base);
4900 if (ret) {
4901 pr_err("ITS@%pa: failed to quiesce on resume: %d\n",
4902 &its->phys_base, ret);
4903 continue;
4904 }
4905
4906 gits_write_cbaser(its->cbaser_save, base + GITS_CBASER);
4907
4908 /*
4909 * Writing CBASER resets CREADR to 0, so make CWRITER and
4910 * cmd_write line up with it.
4911 */
4912 its->cmd_write = its->cmd_base;
4913 gits_write_cwriter(0, base + GITS_CWRITER);
4914
4915 /* Restore GITS_BASER from the value cache. */
4916 for (i = 0; i < GITS_BASER_NR_REGS; i++) {
4917 struct its_baser *baser = &its->tables[i];
4918
4919 if (!(baser->val & GITS_BASER_VALID))
4920 continue;
4921
4922 its_write_baser(its, baser, baser->val);
4923 }
4924 writel_relaxed(its->ctlr_save, base + GITS_CTLR);
Derek Basehore920181c2018-02-28 21:48:20 -08004925
4926 /*
4927 * Reinit the collection if it's stored in the ITS. This is
4928 * indicated by the col_id being less than the HCC field.
4929 * CID < HCC as specified in the GIC v3 Documentation.
4930 */
4931 if (its->collections[smp_processor_id()].col_id <
4932 GITS_TYPER_HCC(gic_read_typer(base + GITS_TYPER)))
4933 its_cpu_init_collection(its);
Derek Basehoredba0bc72018-02-28 21:48:18 -08004934 }
Sebastian Andrzej Siewiora8db7452018-07-18 17:42:04 +02004935 raw_spin_unlock(&its_lock);
Derek Basehoredba0bc72018-02-28 21:48:18 -08004936}
4937
4938static struct syscore_ops its_syscore_ops = {
4939 .suspend = its_save_disable,
4940 .resume = its_restore_enable,
4941};
4942
Marc Zyngierc733ebb2022-01-24 13:38:09 +00004943static void __init __iomem *its_map_one(struct resource *res, int *err)
4944{
4945 void __iomem *its_base;
4946 u32 val;
4947
4948 its_base = ioremap(res->start, SZ_64K);
4949 if (!its_base) {
4950 pr_warn("ITS@%pa: Unable to map ITS registers\n", &res->start);
4951 *err = -ENOMEM;
4952 return NULL;
4953 }
4954
4955 val = readl_relaxed(its_base + GITS_PIDR2) & GIC_PIDR2_ARCH_MASK;
4956 if (val != 0x30 && val != 0x40) {
4957 pr_warn("ITS@%pa: No ITS detected, giving up\n", &res->start);
4958 *err = -ENODEV;
4959 goto out_unmap;
4960 }
4961
4962 *err = its_force_quiescent(its_base);
4963 if (*err) {
4964 pr_warn("ITS@%pa: Failed to quiesce, giving up\n", &res->start);
4965 goto out_unmap;
4966 }
4967
4968 return its_base;
4969
4970out_unmap:
4971 iounmap(its_base);
4972 return NULL;
4973}
4974
Marc Zyngier9585a492023-10-06 14:59:27 +02004975static int its_init_domain(struct its_node *its)
Tomasz Nowickid14ae5e2016-09-12 20:32:23 +02004976{
4977 struct irq_domain *inner_domain;
4978 struct msi_domain_info *info;
4979
4980 info = kzalloc(sizeof(*info), GFP_KERNEL);
4981 if (!info)
4982 return -ENOMEM;
4983
Johan Hovold1e46e042023-02-13 11:42:58 +01004984 info->ops = &its_msi_domain_ops;
4985 info->data = its;
4986
4987 inner_domain = irq_domain_create_hierarchy(its_parent,
4988 its->msi_domain_flags, 0,
Marc Zyngier9585a492023-10-06 14:59:27 +02004989 its->fwnode_handle, &its_domain_ops,
Johan Hovold1e46e042023-02-13 11:42:58 +01004990 info);
Tomasz Nowickid14ae5e2016-09-12 20:32:23 +02004991 if (!inner_domain) {
4992 kfree(info);
4993 return -ENOMEM;
4994 }
4995
Marc Zyngier96f0d932017-06-22 11:42:50 +01004996 irq_domain_update_bus_token(inner_domain, DOMAIN_BUS_NEXUS);
Tomasz Nowickid14ae5e2016-09-12 20:32:23 +02004997
Thomas Gleixner48f71d52024-06-23 17:18:36 +02004998 inner_domain->msi_parent_ops = &gic_v3_its_msi_parent_ops;
4999 inner_domain->flags |= IRQ_DOMAIN_FLAG_MSI_PARENT;
5000
Tomasz Nowickid14ae5e2016-09-12 20:32:23 +02005001 return 0;
5002}
5003
Marc Zyngier8fff27a2016-12-20 13:41:55 +00005004static int its_init_vpe_domain(void)
5005{
Marc Zyngier20b3d542016-12-20 15:23:22 +00005006 struct its_node *its;
5007 u32 devid;
5008 int entries;
5009
5010 if (gic_rdists->has_direct_lpi) {
5011 pr_info("ITS: Using DirectLPI for VPE invalidation\n");
5012 return 0;
5013 }
5014
5015 /* Any ITS will do, even if not v4 */
5016 its = list_first_entry(&its_nodes, struct its_node, entry);
5017
5018 entries = roundup_pow_of_two(nr_cpu_ids);
Kees Cook6396bb22018-06-12 14:03:40 -07005019 vpe_proxy.vpes = kcalloc(entries, sizeof(*vpe_proxy.vpes),
Marc Zyngier20b3d542016-12-20 15:23:22 +00005020 GFP_KERNEL);
Zhen Lei944a1a12021-06-09 22:06:42 +08005021 if (!vpe_proxy.vpes)
Marc Zyngier20b3d542016-12-20 15:23:22 +00005022 return -ENOMEM;
Marc Zyngier20b3d542016-12-20 15:23:22 +00005023
5024 /* Use the last possible DevID */
Marc Zyngier576a8342019-11-08 16:58:00 +00005025 devid = GENMASK(device_ids(its) - 1, 0);
Marc Zyngier20b3d542016-12-20 15:23:22 +00005026 vpe_proxy.dev = its_create_device(its, devid, entries, false);
5027 if (!vpe_proxy.dev) {
5028 kfree(vpe_proxy.vpes);
5029 pr_err("ITS: Can't allocate GICv4 proxy device\n");
5030 return -ENOMEM;
5031 }
5032
Shanker Donthinenic427a472017-09-23 13:50:19 -05005033 BUG_ON(entries > vpe_proxy.dev->nr_ites);
Marc Zyngier20b3d542016-12-20 15:23:22 +00005034
5035 raw_spin_lock_init(&vpe_proxy.lock);
5036 vpe_proxy.next_victim = 0;
5037 pr_info("ITS: Allocated DevID %x as GICv4 proxy device (%d slots)\n",
5038 devid, vpe_proxy.dev->nr_ites);
5039
Marc Zyngier8fff27a2016-12-20 13:41:55 +00005040 return 0;
5041}
5042
Marc Zyngier9585a492023-10-06 14:59:27 +02005043static int __init its_compute_its_list_map(struct its_node *its)
Marc Zyngier3dfa5762016-12-19 17:25:54 +00005044{
5045 int its_number;
5046 u32 ctlr;
5047
5048 /*
5049 * This is assumed to be done early enough that we're
5050 * guaranteed to be single-threaded, hence no
5051 * locking. Should this change, we should address
5052 * this.
5053 */
Marc Zyngierab604912017-10-08 18:48:06 +01005054 its_number = find_first_zero_bit(&its_list_map, GICv4_ITS_LIST_MAX);
5055 if (its_number >= GICv4_ITS_LIST_MAX) {
Marc Zyngier3dfa5762016-12-19 17:25:54 +00005056 pr_err("ITS@%pa: No ITSList entry available!\n",
Marc Zyngier9585a492023-10-06 14:59:27 +02005057 &its->phys_base);
Marc Zyngier3dfa5762016-12-19 17:25:54 +00005058 return -EINVAL;
5059 }
5060
Marc Zyngier9585a492023-10-06 14:59:27 +02005061 ctlr = readl_relaxed(its->base + GITS_CTLR);
Marc Zyngier3dfa5762016-12-19 17:25:54 +00005062 ctlr &= ~GITS_CTLR_ITS_NUMBER;
5063 ctlr |= its_number << GITS_CTLR_ITS_NUMBER_SHIFT;
Marc Zyngier9585a492023-10-06 14:59:27 +02005064 writel_relaxed(ctlr, its->base + GITS_CTLR);
5065 ctlr = readl_relaxed(its->base + GITS_CTLR);
Marc Zyngier3dfa5762016-12-19 17:25:54 +00005066 if ((ctlr & GITS_CTLR_ITS_NUMBER) != (its_number << GITS_CTLR_ITS_NUMBER_SHIFT)) {
5067 its_number = ctlr & GITS_CTLR_ITS_NUMBER;
5068 its_number >>= GITS_CTLR_ITS_NUMBER_SHIFT;
5069 }
5070
5071 if (test_and_set_bit(its_number, &its_list_map)) {
5072 pr_err("ITS@%pa: Duplicate ITSList entry %d\n",
Marc Zyngier9585a492023-10-06 14:59:27 +02005073 &its->phys_base, its_number);
Marc Zyngier3dfa5762016-12-19 17:25:54 +00005074 return -EINVAL;
5075 }
5076
5077 return its_number;
5078}
5079
Marc Zyngier9585a492023-10-06 14:59:27 +02005080static int __init its_probe_one(struct its_node *its)
Marc Zyngier4c21f3c2014-11-24 14:35:17 +00005081{
Marc Zyngier9585a492023-10-06 14:59:27 +02005082 u64 baser, tmp;
Shanker Donthineni539d3782019-01-14 09:50:19 +00005083 struct page *page;
Marc Zyngierc733ebb2022-01-24 13:38:09 +00005084 u32 ctlr;
Marc Zyngier4c21f3c2014-11-24 14:35:17 +00005085 int err;
5086
Marc Zyngier8b02da02024-02-13 10:12:05 +00005087 its_enable_quirks(its);
5088
Marc Zyngier0dd57fe2019-11-08 16:57:58 +00005089 if (is_v4(its)) {
Marc Zyngier9585a492023-10-06 14:59:27 +02005090 if (!(its->typer & GITS_TYPER_VMOVP)) {
5091 err = its_compute_its_list_map(its);
Marc Zyngier3dfa5762016-12-19 17:25:54 +00005092 if (err < 0)
Marc Zyngier9585a492023-10-06 14:59:27 +02005093 goto out;
Marc Zyngier3dfa5762016-12-19 17:25:54 +00005094
Marc Zyngierdebf6d02017-10-08 18:44:42 +01005095 its->list_nr = err;
5096
Marc Zyngier3dfa5762016-12-19 17:25:54 +00005097 pr_info("ITS@%pa: Using ITS number %d\n",
Marc Zyngier9585a492023-10-06 14:59:27 +02005098 &its->phys_base, err);
Marc Zyngier3dfa5762016-12-19 17:25:54 +00005099 } else {
Marc Zyngier9585a492023-10-06 14:59:27 +02005100 pr_info("ITS@%pa: Single VMOVP capable\n", &its->phys_base);
Marc Zyngier3dfa5762016-12-19 17:25:54 +00005101 }
Marc Zyngier5e516842019-12-24 11:10:28 +00005102
5103 if (is_v4_1(its)) {
Marc Zyngier9585a492023-10-06 14:59:27 +02005104 u32 svpet = FIELD_GET(GITS_TYPER_SVPET, its->typer);
Marc Zyngier5e46a482020-03-04 20:33:14 +00005105
Marc Zyngier9585a492023-10-06 14:59:27 +02005106 its->sgir_base = ioremap(its->phys_base + SZ_128K, SZ_64K);
Marc Zyngier5e46a482020-03-04 20:33:14 +00005107 if (!its->sgir_base) {
5108 err = -ENOMEM;
Marc Zyngier9585a492023-10-06 14:59:27 +02005109 goto out;
Marc Zyngier5e46a482020-03-04 20:33:14 +00005110 }
5111
Marc Zyngier9585a492023-10-06 14:59:27 +02005112 its->mpidr = readl_relaxed(its->base + GITS_MPIDR);
Marc Zyngier5e516842019-12-24 11:10:28 +00005113
5114 pr_info("ITS@%pa: Using GICv4.1 mode %08x %08x\n",
Marc Zyngier9585a492023-10-06 14:59:27 +02005115 &its->phys_base, its->mpidr, svpet);
Marc Zyngier5e516842019-12-24 11:10:28 +00005116 }
Marc Zyngier3dfa5762016-12-19 17:25:54 +00005117 }
5118
Shanker Donthineni539d3782019-01-14 09:50:19 +00005119 page = alloc_pages_node(its->numa_node, GFP_KERNEL | __GFP_ZERO,
5120 get_order(ITS_CMD_QUEUE_SZ));
5121 if (!page) {
Marc Zyngier4c21f3c2014-11-24 14:35:17 +00005122 err = -ENOMEM;
Marc Zyngier5e46a482020-03-04 20:33:14 +00005123 goto out_unmap_sgir;
Marc Zyngier4c21f3c2014-11-24 14:35:17 +00005124 }
Shanker Donthineni539d3782019-01-14 09:50:19 +00005125 its->cmd_base = (void *)page_address(page);
Marc Zyngier4c21f3c2014-11-24 14:35:17 +00005126 its->cmd_write = its->cmd_base;
5127
Shanker Donthineni0e0b0f62016-06-06 18:17:31 -05005128 err = its_alloc_tables(its);
Marc Zyngier4c21f3c2014-11-24 14:35:17 +00005129 if (err)
5130 goto out_free_cmd;
5131
5132 err = its_alloc_collections(its);
5133 if (err)
5134 goto out_free_tables;
5135
5136 baser = (virt_to_phys(its->cmd_base) |
Shanker Donthineni2fd632a2017-01-25 21:51:41 -06005137 GITS_CBASER_RaWaWb |
Marc Zyngier4c21f3c2014-11-24 14:35:17 +00005138 GITS_CBASER_InnerShareable |
5139 (ITS_CMD_QUEUE_SZ / SZ_4K - 1) |
5140 GITS_CBASER_VALID);
5141
Vladimir Murzin0968a612016-11-02 11:54:06 +00005142 gits_write_cbaser(baser, its->base + GITS_CBASER);
5143 tmp = gits_read_cbaser(its->base + GITS_CBASER);
Marc Zyngier4c21f3c2014-11-24 14:35:17 +00005144
Sebastian Reichela8707f52023-04-18 16:21:08 +02005145 if (its->flags & ITS_FLAGS_FORCE_NON_SHAREABLE)
5146 tmp &= ~GITS_CBASER_SHAREABILITY_MASK;
5147
Marc Zyngier4ad3e362015-03-27 14:15:04 +00005148 if ((tmp ^ baser) & GITS_CBASER_SHAREABILITY_MASK) {
Marc Zyngier241a3862015-03-27 14:15:05 +00005149 if (!(tmp & GITS_CBASER_SHAREABILITY_MASK)) {
5150 /*
5151 * The HW reports non-shareable, we must
5152 * remove the cacheability attributes as
5153 * well.
5154 */
5155 baser &= ~(GITS_CBASER_SHAREABILITY_MASK |
5156 GITS_CBASER_CACHEABILITY_MASK);
5157 baser |= GITS_CBASER_nC;
Vladimir Murzin0968a612016-11-02 11:54:06 +00005158 gits_write_cbaser(baser, its->base + GITS_CBASER);
Marc Zyngier241a3862015-03-27 14:15:05 +00005159 }
Marc Zyngier4c21f3c2014-11-24 14:35:17 +00005160 pr_info("ITS: using cache flushing for cmd queue\n");
5161 its->flags |= ITS_FLAGS_CMDQ_NEEDS_FLUSHING;
5162 }
5163
Vladimir Murzin0968a612016-11-02 11:54:06 +00005164 gits_write_cwriter(0, its->base + GITS_CWRITER);
Marc Zyngier3dfa5762016-12-19 17:25:54 +00005165 ctlr = readl_relaxed(its->base + GITS_CTLR);
Marc Zyngierd51c4b42017-06-27 21:24:25 +01005166 ctlr |= GITS_CTLR_ENABLE;
Marc Zyngier0dd57fe2019-11-08 16:57:58 +00005167 if (is_v4(its))
Marc Zyngierd51c4b42017-06-27 21:24:25 +01005168 ctlr |= GITS_CTLR_ImDe;
5169 writel_relaxed(ctlr, its->base + GITS_CTLR);
Marc Zyngier241a3862015-03-27 14:15:05 +00005170
Marc Zyngier9585a492023-10-06 14:59:27 +02005171 err = its_init_domain(its);
Tomasz Nowickid14ae5e2016-09-12 20:32:23 +02005172 if (err)
5173 goto out_free_tables;
Marc Zyngier4c21f3c2014-11-24 14:35:17 +00005174
Sebastian Andrzej Siewiora8db7452018-07-18 17:42:04 +02005175 raw_spin_lock(&its_lock);
Marc Zyngier4c21f3c2014-11-24 14:35:17 +00005176 list_add(&its->entry, &its_nodes);
Sebastian Andrzej Siewiora8db7452018-07-18 17:42:04 +02005177 raw_spin_unlock(&its_lock);
Marc Zyngier4c21f3c2014-11-24 14:35:17 +00005178
5179 return 0;
5180
Marc Zyngier4c21f3c2014-11-24 14:35:17 +00005181out_free_tables:
5182 its_free_tables(its);
5183out_free_cmd:
Robert Richter5bc13c22017-02-01 18:38:25 +01005184 free_pages((unsigned long)its->cmd_base, get_order(ITS_CMD_QUEUE_SZ));
Marc Zyngier5e46a482020-03-04 20:33:14 +00005185out_unmap_sgir:
5186 if (its->sgir_base)
5187 iounmap(its->sgir_base);
Marc Zyngier9585a492023-10-06 14:59:27 +02005188out:
5189 pr_err("ITS@%pa: failed probing (%d)\n", &its->phys_base, err);
Marc Zyngier4c21f3c2014-11-24 14:35:17 +00005190 return err;
5191}
5192
5193static bool gic_rdists_supports_plpis(void)
5194{
Marc Zyngier589ce5f2016-10-14 15:13:07 +01005195 return !!(gic_read_typer(gic_data_rdist_rd_base() + GICR_TYPER) & GICR_TYPER_PLPIS);
Marc Zyngier4c21f3c2014-11-24 14:35:17 +00005196}
5197
Shanker Donthineni6eb486b2018-03-21 20:58:49 -05005198static int redist_disable_lpis(void)
5199{
5200 void __iomem *rbase = gic_data_rdist_rd_base();
5201 u64 timeout = USEC_PER_SEC;
5202 u64 val;
5203
5204 if (!gic_rdists_supports_plpis()) {
5205 pr_info("CPU%d: LPIs not supported\n", smp_processor_id());
5206 return -ENXIO;
5207 }
5208
5209 val = readl_relaxed(rbase + GICR_CTLR);
5210 if (!(val & GICR_CTLR_ENABLE_LPIS))
5211 return 0;
5212
Marc Zyngier11e37d32018-07-27 13:38:54 +01005213 /*
5214 * If coming via a CPU hotplug event, we don't need to disable
5215 * LPIs before trying to re-enable them. They are already
5216 * configured and all is well in the world.
Marc Zyngierc440a9d2018-07-27 15:40:13 +01005217 *
5218 * If running with preallocated tables, there is nothing to do.
Marc Zyngier11e37d32018-07-27 13:38:54 +01005219 */
Valentin Schneiderc0cdc8902021-10-27 16:15:04 +01005220 if ((gic_data_rdist()->flags & RD_LOCAL_LPI_ENABLED) ||
Marc Zyngierc440a9d2018-07-27 15:40:13 +01005221 (gic_rdists->flags & RDIST_FLAGS_RD_TABLES_PREALLOCATED))
Marc Zyngier11e37d32018-07-27 13:38:54 +01005222 return 0;
5223
5224 /*
5225 * From that point on, we only try to do some damage control.
5226 */
5227 pr_warn("GICv3: CPU%d: Booted with LPIs enabled, memory probably corrupted\n",
Shanker Donthineni6eb486b2018-03-21 20:58:49 -05005228 smp_processor_id());
5229 add_taint(TAINT_CRAP, LOCKDEP_STILL_OK);
5230
5231 /* Disable LPIs */
5232 val &= ~GICR_CTLR_ENABLE_LPIS;
5233 writel_relaxed(val, rbase + GICR_CTLR);
5234
5235 /* Make sure any change to GICR_CTLR is observable by the GIC */
5236 dsb(sy);
5237
5238 /*
5239 * Software must observe RWP==0 after clearing GICR_CTLR.EnableLPIs
5240 * from 1 to 0 before programming GICR_PEND{PROP}BASER registers.
5241 * Error out if we time out waiting for RWP to clear.
5242 */
5243 while (readl_relaxed(rbase + GICR_CTLR) & GICR_CTLR_RWP) {
5244 if (!timeout) {
5245 pr_err("CPU%d: Timeout while disabling LPIs\n",
5246 smp_processor_id());
5247 return -ETIMEDOUT;
5248 }
5249 udelay(1);
5250 timeout--;
5251 }
5252
5253 /*
5254 * After it has been written to 1, it is IMPLEMENTATION
5255 * DEFINED whether GICR_CTLR.EnableLPI becomes RES1 or can be
5256 * cleared to 0. Error out if clearing the bit failed.
5257 */
5258 if (readl_relaxed(rbase + GICR_CTLR) & GICR_CTLR_ENABLE_LPIS) {
5259 pr_err("CPU%d: Failed to disable LPIs\n", smp_processor_id());
5260 return -EBUSY;
5261 }
5262
5263 return 0;
5264}
5265
Marc Zyngier4c21f3c2014-11-24 14:35:17 +00005266int its_cpu_init(void)
5267{
Marc Zyngier4c21f3c2014-11-24 14:35:17 +00005268 if (!list_empty(&its_nodes)) {
Shanker Donthineni6eb486b2018-03-21 20:58:49 -05005269 int ret;
5270
5271 ret = redist_disable_lpis();
5272 if (ret)
5273 return ret;
5274
Marc Zyngier4c21f3c2014-11-24 14:35:17 +00005275 its_cpu_init_lpis();
Derek Basehore920181c2018-02-28 21:48:20 -08005276 its_cpu_init_collections();
Marc Zyngier4c21f3c2014-11-24 14:35:17 +00005277 }
5278
5279 return 0;
5280}
5281
Valentin Schneider835f4422021-10-27 16:15:06 +01005282static void rdist_memreserve_cpuhp_cleanup_workfn(struct work_struct *work)
5283{
5284 cpuhp_remove_state_nocalls(gic_rdists->cpuhp_memreserve_state);
5285 gic_rdists->cpuhp_memreserve_state = CPUHP_INVALID;
5286}
5287
5288static DECLARE_WORK(rdist_memreserve_cpuhp_cleanup_work,
5289 rdist_memreserve_cpuhp_cleanup_workfn);
5290
Valentin Schneiderd23bc2b2021-10-27 16:15:05 +01005291static int its_cpu_memreserve_lpi(unsigned int cpu)
5292{
5293 struct page *pend_page;
5294 int ret = 0;
5295
5296 /* This gets to run exactly once per CPU */
5297 if (gic_data_rdist()->flags & RD_LOCAL_MEMRESERVE_DONE)
5298 return 0;
5299
5300 pend_page = gic_data_rdist()->pend_page;
5301 if (WARN_ON(!pend_page)) {
5302 ret = -ENOMEM;
5303 goto out;
5304 }
5305 /*
5306 * If the pending table was pre-programmed, free the memory we
5307 * preemptively allocated. Otherwise, reserve that memory for
5308 * later kexecs.
5309 */
5310 if (gic_data_rdist()->flags & RD_LOCAL_PENDTABLE_PREALLOCATED) {
5311 its_free_pending_table(pend_page);
5312 gic_data_rdist()->pend_page = NULL;
5313 } else {
5314 phys_addr_t paddr = page_to_phys(pend_page);
5315 WARN_ON(gic_reserve_range(paddr, LPI_PENDBASE_SZ));
5316 }
5317
5318out:
Valentin Schneider835f4422021-10-27 16:15:06 +01005319 /* Last CPU being brought up gets to issue the cleanup */
Ard Biesheuvel16436f72022-01-22 16:16:14 +01005320 if (!IS_ENABLED(CONFIG_SMP) ||
5321 cpumask_equal(&cpus_booted_once_mask, cpu_possible_mask))
Valentin Schneider835f4422021-10-27 16:15:06 +01005322 schedule_work(&rdist_memreserve_cpuhp_cleanup_work);
5323
Valentin Schneiderd23bc2b2021-10-27 16:15:05 +01005324 gic_data_rdist()->flags |= RD_LOCAL_MEMRESERVE_DONE;
5325 return ret;
5326}
5327
Marc Zyngierc733ebb2022-01-24 13:38:09 +00005328/* Mark all the BASER registers as invalid before they get reprogrammed */
5329static int __init its_reset_one(struct resource *res)
5330{
5331 void __iomem *its_base;
5332 int err, i;
5333
5334 its_base = its_map_one(res, &err);
5335 if (!its_base)
5336 return err;
5337
5338 for (i = 0; i < GITS_BASER_NR_REGS; i++)
5339 gits_write_baser(0, its_base + GITS_BASER + (i << 3));
5340
5341 iounmap(its_base);
5342 return 0;
5343}
5344
Arvind Yadav935bba72017-06-22 16:05:30 +05305345static const struct of_device_id its_device_id[] = {
Marc Zyngier4c21f3c2014-11-24 14:35:17 +00005346 { .compatible = "arm,gic-v3-its", },
5347 {},
5348};
5349
Marc Zyngier9585a492023-10-06 14:59:27 +02005350static struct its_node __init *its_node_init(struct resource *res,
5351 struct fwnode_handle *handle, int numa_node)
5352{
5353 void __iomem *its_base;
5354 struct its_node *its;
5355 int err;
5356
5357 its_base = its_map_one(res, &err);
5358 if (!its_base)
5359 return NULL;
5360
5361 pr_info("ITS %pR\n", res);
5362
5363 its = kzalloc(sizeof(*its), GFP_KERNEL);
5364 if (!its)
5365 goto out_unmap;
5366
5367 raw_spin_lock_init(&its->lock);
5368 mutex_init(&its->dev_alloc_lock);
5369 INIT_LIST_HEAD(&its->entry);
5370 INIT_LIST_HEAD(&its->its_device_list);
5371
5372 its->typer = gic_read_typer(its_base + GITS_TYPER);
5373 its->base = its_base;
5374 its->phys_base = res->start;
Marc Zyngierf199bf5b2023-10-24 15:34:31 +01005375 its->get_msi_base = its_irq_get_msi_base;
5376 its->msi_domain_flags = IRQ_DOMAIN_FLAG_ISOLATED_MSI;
Marc Zyngier9585a492023-10-06 14:59:27 +02005377
5378 its->numa_node = numa_node;
5379 its->fwnode_handle = handle;
5380
5381 return its;
5382
5383out_unmap:
5384 iounmap(its_base);
5385 return NULL;
5386}
5387
5388static void its_node_destroy(struct its_node *its)
5389{
5390 iounmap(its->base);
5391 kfree(its);
5392}
5393
Tomasz Nowickidb40f0a2016-09-12 20:32:24 +02005394static int __init its_of_probe(struct device_node *node)
Marc Zyngier4c21f3c2014-11-24 14:35:17 +00005395{
5396 struct device_node *np;
Tomasz Nowickidb40f0a2016-09-12 20:32:24 +02005397 struct resource res;
Marc Zyngier9585a492023-10-06 14:59:27 +02005398 int err;
Marc Zyngier4c21f3c2014-11-24 14:35:17 +00005399
Marc Zyngierc733ebb2022-01-24 13:38:09 +00005400 /*
5401 * Make sure *all* the ITS are reset before we probe any, as
5402 * they may be sharing memory. If any of the ITS fails to
5403 * reset, don't even try to go any further, as this could
5404 * result in something even worse.
5405 */
5406 for (np = of_find_matching_node(node, its_device_id); np;
5407 np = of_find_matching_node(np, its_device_id)) {
Marc Zyngierc733ebb2022-01-24 13:38:09 +00005408 if (!of_device_is_available(np) ||
5409 !of_property_read_bool(np, "msi-controller") ||
5410 of_address_to_resource(np, 0, &res))
5411 continue;
5412
5413 err = its_reset_one(&res);
5414 if (err)
5415 return err;
5416 }
5417
Marc Zyngier4c21f3c2014-11-24 14:35:17 +00005418 for (np = of_find_matching_node(node, its_device_id); np;
5419 np = of_find_matching_node(np, its_device_id)) {
Marc Zyngier9585a492023-10-06 14:59:27 +02005420 struct its_node *its;
5421
Stephen Boyd95a25622018-02-01 09:03:29 -08005422 if (!of_device_is_available(np))
5423 continue;
Tomasz Nowickid14ae5e2016-09-12 20:32:23 +02005424 if (!of_property_read_bool(np, "msi-controller")) {
Rob Herringe81f54c2017-07-18 16:43:10 -05005425 pr_warn("%pOF: no msi-controller property, ITS ignored\n",
5426 np);
Tomasz Nowickid14ae5e2016-09-12 20:32:23 +02005427 continue;
5428 }
5429
Tomasz Nowickidb40f0a2016-09-12 20:32:24 +02005430 if (of_address_to_resource(np, 0, &res)) {
Rob Herringe81f54c2017-07-18 16:43:10 -05005431 pr_warn("%pOF: no regs?\n", np);
Tomasz Nowickidb40f0a2016-09-12 20:32:24 +02005432 continue;
5433 }
5434
Marc Zyngier9585a492023-10-06 14:59:27 +02005435
5436 its = its_node_init(&res, &np->fwnode, of_node_to_nid(np));
5437 if (!its)
5438 return -ENOMEM;
5439
Marc Zyngier9585a492023-10-06 14:59:27 +02005440 err = its_probe_one(its);
5441 if (err) {
5442 its_node_destroy(its);
5443 return err;
5444 }
Marc Zyngier4c21f3c2014-11-24 14:35:17 +00005445 }
Tomasz Nowickidb40f0a2016-09-12 20:32:24 +02005446 return 0;
5447}
5448
Tomasz Nowicki3f010cf2016-09-12 20:32:25 +02005449#ifdef CONFIG_ACPI
5450
5451#define ACPI_GICV3_ITS_MEM_SIZE (SZ_128K)
5452
Robert Richterd1ce2632017-07-12 15:25:09 +02005453#ifdef CONFIG_ACPI_NUMA
Ganapatrao Kulkarnidbd2b822017-06-22 11:40:12 +05305454struct its_srat_map {
5455 /* numa node id */
5456 u32 numa_node;
5457 /* GIC ITS ID */
5458 u32 its_id;
5459};
5460
Hanjun Guofdf6e7a2017-07-26 18:15:49 +08005461static struct its_srat_map *its_srat_maps __initdata;
Ganapatrao Kulkarnidbd2b822017-06-22 11:40:12 +05305462static int its_in_srat __initdata;
5463
5464static int __init acpi_get_its_numa_node(u32 its_id)
5465{
5466 int i;
5467
5468 for (i = 0; i < its_in_srat; i++) {
5469 if (its_id == its_srat_maps[i].its_id)
5470 return its_srat_maps[i].numa_node;
5471 }
5472 return NUMA_NO_NODE;
5473}
5474
Keith Busch60574d12019-03-11 14:55:57 -06005475static int __init gic_acpi_match_srat_its(union acpi_subtable_headers *header,
Hanjun Guofdf6e7a2017-07-26 18:15:49 +08005476 const unsigned long end)
5477{
5478 return 0;
5479}
5480
Keith Busch60574d12019-03-11 14:55:57 -06005481static int __init gic_acpi_parse_srat_its(union acpi_subtable_headers *header,
Ganapatrao Kulkarnidbd2b822017-06-22 11:40:12 +05305482 const unsigned long end)
5483{
5484 int node;
5485 struct acpi_srat_gic_its_affinity *its_affinity;
5486
5487 its_affinity = (struct acpi_srat_gic_its_affinity *)header;
5488 if (!its_affinity)
5489 return -EINVAL;
5490
5491 if (its_affinity->header.length < sizeof(*its_affinity)) {
5492 pr_err("SRAT: Invalid header length %d in ITS affinity\n",
5493 its_affinity->header.length);
5494 return -EINVAL;
5495 }
5496
Jonathan Cameron95ac5bf2020-08-18 22:24:30 +08005497 /*
5498 * Note that in theory a new proximity node could be created by this
5499 * entry as it is an SRAT resource allocation structure.
5500 * We do not currently support doing so.
5501 */
5502 node = pxm_to_node(its_affinity->proximity_domain);
Ganapatrao Kulkarnidbd2b822017-06-22 11:40:12 +05305503
5504 if (node == NUMA_NO_NODE || node >= MAX_NUMNODES) {
5505 pr_err("SRAT: Invalid NUMA node %d in ITS affinity\n", node);
5506 return 0;
5507 }
5508
5509 its_srat_maps[its_in_srat].numa_node = node;
5510 its_srat_maps[its_in_srat].its_id = its_affinity->its_id;
5511 its_in_srat++;
5512 pr_info("SRAT: PXM %d -> ITS %d -> Node %d\n",
5513 its_affinity->proximity_domain, its_affinity->its_id, node);
5514
5515 return 0;
5516}
5517
5518static void __init acpi_table_parse_srat_its(void)
5519{
Hanjun Guofdf6e7a2017-07-26 18:15:49 +08005520 int count;
5521
5522 count = acpi_table_parse_entries(ACPI_SIG_SRAT,
5523 sizeof(struct acpi_table_srat),
5524 ACPI_SRAT_TYPE_GIC_ITS_AFFINITY,
5525 gic_acpi_match_srat_its, 0);
5526 if (count <= 0)
5527 return;
5528
Kees Cook6da2ec52018-06-12 13:55:00 -07005529 its_srat_maps = kmalloc_array(count, sizeof(struct its_srat_map),
5530 GFP_KERNEL);
Zhen Lei944a1a12021-06-09 22:06:42 +08005531 if (!its_srat_maps)
Hanjun Guofdf6e7a2017-07-26 18:15:49 +08005532 return;
Hanjun Guofdf6e7a2017-07-26 18:15:49 +08005533
Ganapatrao Kulkarnidbd2b822017-06-22 11:40:12 +05305534 acpi_table_parse_entries(ACPI_SIG_SRAT,
5535 sizeof(struct acpi_table_srat),
5536 ACPI_SRAT_TYPE_GIC_ITS_AFFINITY,
5537 gic_acpi_parse_srat_its, 0);
5538}
Hanjun Guofdf6e7a2017-07-26 18:15:49 +08005539
5540/* free the its_srat_maps after ITS probing */
5541static void __init acpi_its_srat_maps_free(void)
5542{
5543 kfree(its_srat_maps);
5544}
Ganapatrao Kulkarnidbd2b822017-06-22 11:40:12 +05305545#else
5546static void __init acpi_table_parse_srat_its(void) { }
5547static int __init acpi_get_its_numa_node(u32 its_id) { return NUMA_NO_NODE; }
Hanjun Guofdf6e7a2017-07-26 18:15:49 +08005548static void __init acpi_its_srat_maps_free(void) { }
Ganapatrao Kulkarnidbd2b822017-06-22 11:40:12 +05305549#endif
5550
Keith Busch60574d12019-03-11 14:55:57 -06005551static int __init gic_acpi_parse_madt_its(union acpi_subtable_headers *header,
Tomasz Nowicki3f010cf2016-09-12 20:32:25 +02005552 const unsigned long end)
5553{
5554 struct acpi_madt_generic_translator *its_entry;
5555 struct fwnode_handle *dom_handle;
Marc Zyngier9585a492023-10-06 14:59:27 +02005556 struct its_node *its;
Tomasz Nowicki3f010cf2016-09-12 20:32:25 +02005557 struct resource res;
5558 int err;
5559
5560 its_entry = (struct acpi_madt_generic_translator *)header;
5561 memset(&res, 0, sizeof(res));
5562 res.start = its_entry->base_address;
5563 res.end = its_entry->base_address + ACPI_GICV3_ITS_MEM_SIZE - 1;
5564 res.flags = IORESOURCE_MEM;
5565
Marc Zyngier5778cc72019-07-31 16:13:42 +01005566 dom_handle = irq_domain_alloc_fwnode(&res.start);
Tomasz Nowicki3f010cf2016-09-12 20:32:25 +02005567 if (!dom_handle) {
5568 pr_err("ITS@%pa: Unable to allocate GICv3 ITS domain token\n",
5569 &res.start);
5570 return -ENOMEM;
5571 }
5572
Shameer Kolothum8b4282e2018-02-13 15:20:50 +00005573 err = iort_register_domain_token(its_entry->translation_id, res.start,
5574 dom_handle);
Tomasz Nowicki3f010cf2016-09-12 20:32:25 +02005575 if (err) {
5576 pr_err("ITS@%pa: Unable to register GICv3 ITS domain token (ITS ID %d) to IORT\n",
5577 &res.start, its_entry->translation_id);
5578 goto dom_err;
5579 }
5580
Marc Zyngier9585a492023-10-06 14:59:27 +02005581 its = its_node_init(&res, dom_handle,
5582 acpi_get_its_numa_node(its_entry->translation_id));
5583 if (!its) {
5584 err = -ENOMEM;
5585 goto node_err;
5586 }
5587
Lorenzo Pieralisiababa162024-06-06 11:42:38 +02005588 if (acpi_get_madt_revision() >= 7 &&
5589 (its_entry->flags & ACPI_MADT_ITS_NON_COHERENT))
5590 its->flags |= ITS_FLAGS_FORCE_NON_SHAREABLE;
5591
Marc Zyngier9585a492023-10-06 14:59:27 +02005592 err = its_probe_one(its);
Tomasz Nowicki3f010cf2016-09-12 20:32:25 +02005593 if (!err)
5594 return 0;
5595
Marc Zyngier9585a492023-10-06 14:59:27 +02005596node_err:
Tomasz Nowicki3f010cf2016-09-12 20:32:25 +02005597 iort_deregister_domain_token(its_entry->translation_id);
5598dom_err:
5599 irq_domain_free_fwnode(dom_handle);
5600 return err;
5601}
5602
Marc Zyngierc733ebb2022-01-24 13:38:09 +00005603static int __init its_acpi_reset(union acpi_subtable_headers *header,
5604 const unsigned long end)
5605{
5606 struct acpi_madt_generic_translator *its_entry;
5607 struct resource res;
5608
5609 its_entry = (struct acpi_madt_generic_translator *)header;
5610 res = (struct resource) {
5611 .start = its_entry->base_address,
5612 .end = its_entry->base_address + ACPI_GICV3_ITS_MEM_SIZE - 1,
5613 .flags = IORESOURCE_MEM,
5614 };
5615
5616 return its_reset_one(&res);
5617}
5618
Tomasz Nowicki3f010cf2016-09-12 20:32:25 +02005619static void __init its_acpi_probe(void)
5620{
Ganapatrao Kulkarnidbd2b822017-06-22 11:40:12 +05305621 acpi_table_parse_srat_its();
Marc Zyngierc733ebb2022-01-24 13:38:09 +00005622 /*
5623 * Make sure *all* the ITS are reset before we probe any, as
5624 * they may be sharing memory. If any of the ITS fails to
5625 * reset, don't even try to go any further, as this could
5626 * result in something even worse.
5627 */
5628 if (acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_TRANSLATOR,
5629 its_acpi_reset, 0) > 0)
5630 acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_TRANSLATOR,
5631 gic_acpi_parse_madt_its, 0);
Hanjun Guofdf6e7a2017-07-26 18:15:49 +08005632 acpi_its_srat_maps_free();
Tomasz Nowicki3f010cf2016-09-12 20:32:25 +02005633}
5634#else
5635static void __init its_acpi_probe(void) { }
5636#endif
5637
Valentin Schneiderd23bc2b2021-10-27 16:15:05 +01005638int __init its_lpi_memreserve_init(void)
5639{
5640 int state;
5641
5642 if (!efi_enabled(EFI_CONFIG_TABLES))
5643 return 0;
5644
Marc Zyngiereba1e442022-02-02 10:34:54 +00005645 if (list_empty(&its_nodes))
5646 return 0;
5647
Valentin Schneider835f4422021-10-27 16:15:06 +01005648 gic_rdists->cpuhp_memreserve_state = CPUHP_INVALID;
Valentin Schneiderd23bc2b2021-10-27 16:15:05 +01005649 state = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN,
5650 "irqchip/arm/gicv3/memreserve:online",
5651 its_cpu_memreserve_lpi,
5652 NULL);
5653 if (state < 0)
5654 return state;
5655
Valentin Schneider835f4422021-10-27 16:15:06 +01005656 gic_rdists->cpuhp_memreserve_state = state;
5657
Valentin Schneiderd23bc2b2021-10-27 16:15:05 +01005658 return 0;
5659}
5660
Tomasz Nowickidb40f0a2016-09-12 20:32:24 +02005661int __init its_init(struct fwnode_handle *handle, struct rdists *rdists,
Mark Rutlanda6156e72024-06-17 12:18:39 +01005662 struct irq_domain *parent_domain, u8 irq_prio)
Tomasz Nowickidb40f0a2016-09-12 20:32:24 +02005663{
5664 struct device_node *of_node;
Marc Zyngier8fff27a2016-12-20 13:41:55 +00005665 struct its_node *its;
5666 bool has_v4 = false;
Marc Zyngier3c407062020-03-04 20:33:13 +00005667 bool has_v4_1 = false;
Marc Zyngier8fff27a2016-12-20 13:41:55 +00005668 int err;
Tomasz Nowickidb40f0a2016-09-12 20:32:24 +02005669
Marc Zyngier5e516842019-12-24 11:10:28 +00005670 gic_rdists = rdists;
5671
Mark Rutlanda6156e72024-06-17 12:18:39 +01005672 lpi_prop_prio = irq_prio;
Tomasz Nowickidb40f0a2016-09-12 20:32:24 +02005673 its_parent = parent_domain;
5674 of_node = to_of_node(handle);
5675 if (of_node)
5676 its_of_probe(of_node);
5677 else
Tomasz Nowicki3f010cf2016-09-12 20:32:25 +02005678 its_acpi_probe();
Marc Zyngier4c21f3c2014-11-24 14:35:17 +00005679
5680 if (list_empty(&its_nodes)) {
5681 pr_warn("ITS: No ITS available, not enabling LPIs\n");
5682 return -ENXIO;
5683 }
5684
Marc Zyngier11e37d32018-07-27 13:38:54 +01005685 err = allocate_lpi_tables();
Marc Zyngier8fff27a2016-12-20 13:41:55 +00005686 if (err)
5687 return err;
5688
Marc Zyngier3c407062020-03-04 20:33:13 +00005689 list_for_each_entry(its, &its_nodes, entry) {
Marc Zyngier0dd57fe2019-11-08 16:57:58 +00005690 has_v4 |= is_v4(its);
Marc Zyngier3c407062020-03-04 20:33:13 +00005691 has_v4_1 |= is_v4_1(its);
5692 }
5693
5694 /* Don't bother with inconsistent systems */
5695 if (WARN_ON(!has_v4_1 && rdists->has_rvpeid))
5696 rdists->has_rvpeid = false;
Marc Zyngier8fff27a2016-12-20 13:41:55 +00005697
5698 if (has_v4 & rdists->has_vlpis) {
Marc Zyngier166cba72020-03-04 20:33:15 +00005699 const struct irq_domain_ops *sgi_ops;
5700
5701 if (has_v4_1)
5702 sgi_ops = &its_sgi_domain_ops;
5703 else
5704 sgi_ops = NULL;
5705
Marc Zyngier3d63cb52016-12-20 15:31:54 +00005706 if (its_init_vpe_domain() ||
Marc Zyngier166cba72020-03-04 20:33:15 +00005707 its_init_v4(parent_domain, &its_vpe_domain_ops, sgi_ops)) {
Marc Zyngier8fff27a2016-12-20 13:41:55 +00005708 rdists->has_vlpis = false;
5709 pr_err("ITS: Disabling GICv4 support\n");
5710 }
5711 }
5712
Derek Basehoredba0bc72018-02-28 21:48:18 -08005713 register_syscore_ops(&its_syscore_ops);
5714
Marc Zyngier8fff27a2016-12-20 13:41:55 +00005715 return 0;
Marc Zyngier4c21f3c2014-11-24 14:35:17 +00005716}