Thomas Gleixner | 52a65ff | 2018-03-14 22:15:19 +0100 | [diff] [blame] | 1 | // SPDX-License-Identifier: GPL-2.0 |
Thomas Gleixner | 3795de2 | 2010-09-22 17:09:43 +0200 | [diff] [blame] | 2 | /* |
| 3 | * Copyright (C) 1992, 1998-2006 Linus Torvalds, Ingo Molnar |
| 4 | * Copyright (C) 2005-2006, Thomas Gleixner, Russell King |
| 5 | * |
Thomas Gleixner | 99bfce5 | 2018-03-14 22:15:16 +0100 | [diff] [blame] | 6 | * This file contains the interrupt descriptor management code. Detailed |
| 7 | * information is available in Documentation/core-api/genericirq.rst |
Thomas Gleixner | 3795de2 | 2010-09-22 17:09:43 +0200 | [diff] [blame] | 8 | * |
| 9 | */ |
| 10 | #include <linux/irq.h> |
| 11 | #include <linux/slab.h> |
Paul Gortmaker | ec53cf2 | 2011-09-19 20:33:19 -0400 | [diff] [blame] | 12 | #include <linux/export.h> |
Thomas Gleixner | 3795de2 | 2010-09-22 17:09:43 +0200 | [diff] [blame] | 13 | #include <linux/interrupt.h> |
| 14 | #include <linux/kernel_stat.h> |
| 15 | #include <linux/radix-tree.h> |
Thomas Gleixner | 1f5a5b8 | 2010-09-27 17:48:26 +0200 | [diff] [blame] | 16 | #include <linux/bitmap.h> |
Marc Zyngier | 76ba59f | 2014-08-26 11:03:16 +0100 | [diff] [blame] | 17 | #include <linux/irqdomain.h> |
Craig Gallek | ecb3f39 | 2016-09-13 12:14:51 -0400 | [diff] [blame] | 18 | #include <linux/sysfs.h> |
Thomas Gleixner | 3795de2 | 2010-09-22 17:09:43 +0200 | [diff] [blame] | 19 | |
| 20 | #include "internals.h" |
| 21 | |
| 22 | /* |
| 23 | * lockdep: we want to handle all irq_desc locks as a single lock-class: |
| 24 | */ |
Thomas Gleixner | 78f90d9 | 2010-09-29 17:18:47 +0200 | [diff] [blame] | 25 | static struct lock_class_key irq_desc_lock_class; |
Thomas Gleixner | 3795de2 | 2010-09-22 17:09:43 +0200 | [diff] [blame] | 26 | |
Thomas Gleixner | fe05143 | 2011-05-18 12:53:03 +0200 | [diff] [blame] | 27 | #if defined(CONFIG_SMP) |
Thomas Gleixner | fbf1980 | 2016-02-03 19:52:23 +0100 | [diff] [blame] | 28 | static int __init irq_affinity_setup(char *str) |
| 29 | { |
Rakib Mullick | 10d94ff | 2017-11-01 10:14:51 +0600 | [diff] [blame] | 30 | alloc_bootmem_cpumask_var(&irq_default_affinity); |
Thomas Gleixner | fbf1980 | 2016-02-03 19:52:23 +0100 | [diff] [blame] | 31 | cpulist_parse(str, irq_default_affinity); |
| 32 | /* |
| 33 | * Set at least the boot cpu. We don't want to end up with |
| 34 | * bugreports caused by random comandline masks |
| 35 | */ |
| 36 | cpumask_set_cpu(smp_processor_id(), irq_default_affinity); |
| 37 | return 1; |
| 38 | } |
| 39 | __setup("irqaffinity=", irq_affinity_setup); |
| 40 | |
Thomas Gleixner | 3795de2 | 2010-09-22 17:09:43 +0200 | [diff] [blame] | 41 | static void __init init_irq_default_affinity(void) |
| 42 | { |
Rakib Mullick | 10d94ff | 2017-11-01 10:14:51 +0600 | [diff] [blame] | 43 | if (!cpumask_available(irq_default_affinity)) |
Thomas Gleixner | fbf1980 | 2016-02-03 19:52:23 +0100 | [diff] [blame] | 44 | zalloc_cpumask_var(&irq_default_affinity, GFP_NOWAIT); |
Thomas Gleixner | fbf1980 | 2016-02-03 19:52:23 +0100 | [diff] [blame] | 45 | if (cpumask_empty(irq_default_affinity)) |
| 46 | cpumask_setall(irq_default_affinity); |
Thomas Gleixner | 3795de2 | 2010-09-22 17:09:43 +0200 | [diff] [blame] | 47 | } |
| 48 | #else |
| 49 | static void __init init_irq_default_affinity(void) |
| 50 | { |
| 51 | } |
| 52 | #endif |
| 53 | |
Thomas Gleixner | 1f5a5b8 | 2010-09-27 17:48:26 +0200 | [diff] [blame] | 54 | #ifdef CONFIG_SMP |
Thomas Gleixner | 4ab764c | 2017-06-20 01:37:36 +0200 | [diff] [blame] | 55 | static int alloc_masks(struct irq_desc *desc, int node) |
Thomas Gleixner | 1f5a5b8 | 2010-09-27 17:48:26 +0200 | [diff] [blame] | 56 | { |
Jiang Liu | 9df872f | 2015-06-03 11:47:50 +0800 | [diff] [blame] | 57 | if (!zalloc_cpumask_var_node(&desc->irq_common_data.affinity, |
Thomas Gleixner | 4ab764c | 2017-06-20 01:37:36 +0200 | [diff] [blame] | 58 | GFP_KERNEL, node)) |
Thomas Gleixner | 1f5a5b8 | 2010-09-27 17:48:26 +0200 | [diff] [blame] | 59 | return -ENOMEM; |
| 60 | |
Thomas Gleixner | 0d3f542 | 2017-06-20 01:37:38 +0200 | [diff] [blame] | 61 | #ifdef CONFIG_GENERIC_IRQ_EFFECTIVE_AFF_MASK |
| 62 | if (!zalloc_cpumask_var_node(&desc->irq_common_data.effective_affinity, |
| 63 | GFP_KERNEL, node)) { |
| 64 | free_cpumask_var(desc->irq_common_data.affinity); |
| 65 | return -ENOMEM; |
| 66 | } |
| 67 | #endif |
| 68 | |
Thomas Gleixner | 1f5a5b8 | 2010-09-27 17:48:26 +0200 | [diff] [blame] | 69 | #ifdef CONFIG_GENERIC_PENDING_IRQ |
Thomas Gleixner | 4ab764c | 2017-06-20 01:37:36 +0200 | [diff] [blame] | 70 | if (!zalloc_cpumask_var_node(&desc->pending_mask, GFP_KERNEL, node)) { |
Thomas Gleixner | 0d3f542 | 2017-06-20 01:37:38 +0200 | [diff] [blame] | 71 | #ifdef CONFIG_GENERIC_IRQ_EFFECTIVE_AFF_MASK |
| 72 | free_cpumask_var(desc->irq_common_data.effective_affinity); |
| 73 | #endif |
Jiang Liu | 9df872f | 2015-06-03 11:47:50 +0800 | [diff] [blame] | 74 | free_cpumask_var(desc->irq_common_data.affinity); |
Thomas Gleixner | 1f5a5b8 | 2010-09-27 17:48:26 +0200 | [diff] [blame] | 75 | return -ENOMEM; |
| 76 | } |
| 77 | #endif |
| 78 | return 0; |
| 79 | } |
| 80 | |
Thomas Gleixner | 45ddcec | 2016-07-04 17:39:25 +0900 | [diff] [blame] | 81 | static void desc_smp_init(struct irq_desc *desc, int node, |
| 82 | const struct cpumask *affinity) |
Thomas Gleixner | 1f5a5b8 | 2010-09-27 17:48:26 +0200 | [diff] [blame] | 83 | { |
Thomas Gleixner | 45ddcec | 2016-07-04 17:39:25 +0900 | [diff] [blame] | 84 | if (!affinity) |
| 85 | affinity = irq_default_affinity; |
| 86 | cpumask_copy(desc->irq_common_data.affinity, affinity); |
| 87 | |
Thomas Gleixner | b7b2933 | 2010-09-29 18:46:55 +0200 | [diff] [blame] | 88 | #ifdef CONFIG_GENERIC_PENDING_IRQ |
| 89 | cpumask_clear(desc->pending_mask); |
| 90 | #endif |
Jiang Liu | 449e9ca | 2015-06-01 16:05:16 +0800 | [diff] [blame] | 91 | #ifdef CONFIG_NUMA |
| 92 | desc->irq_common_data.node = node; |
| 93 | #endif |
Thomas Gleixner | b7b2933 | 2010-09-29 18:46:55 +0200 | [diff] [blame] | 94 | } |
| 95 | |
Thomas Gleixner | 1f5a5b8 | 2010-09-27 17:48:26 +0200 | [diff] [blame] | 96 | #else |
| 97 | static inline int |
Thomas Gleixner | 4ab764c | 2017-06-20 01:37:36 +0200 | [diff] [blame] | 98 | alloc_masks(struct irq_desc *desc, int node) { return 0; } |
Thomas Gleixner | 45ddcec | 2016-07-04 17:39:25 +0900 | [diff] [blame] | 99 | static inline void |
| 100 | desc_smp_init(struct irq_desc *desc, int node, const struct cpumask *affinity) { } |
Thomas Gleixner | 1f5a5b8 | 2010-09-27 17:48:26 +0200 | [diff] [blame] | 101 | #endif |
| 102 | |
Sebastian Andrzej Siewior | b687380 | 2011-07-11 12:17:31 +0200 | [diff] [blame] | 103 | static void desc_set_defaults(unsigned int irq, struct irq_desc *desc, int node, |
Thomas Gleixner | 45ddcec | 2016-07-04 17:39:25 +0900 | [diff] [blame] | 104 | const struct cpumask *affinity, struct module *owner) |
Thomas Gleixner | 1f5a5b8 | 2010-09-27 17:48:26 +0200 | [diff] [blame] | 105 | { |
Eric Dumazet | 6c9ae00 | 2011-01-13 15:45:38 -0800 | [diff] [blame] | 106 | int cpu; |
| 107 | |
Jiang Liu | af7080e | 2015-06-01 16:05:21 +0800 | [diff] [blame] | 108 | desc->irq_common_data.handler_data = NULL; |
Jiang Liu | b237721 | 2015-06-01 16:05:43 +0800 | [diff] [blame] | 109 | desc->irq_common_data.msi_desc = NULL; |
Jiang Liu | af7080e | 2015-06-01 16:05:21 +0800 | [diff] [blame] | 110 | |
Jiang Liu | 0d0b4c8 | 2015-06-01 16:05:12 +0800 | [diff] [blame] | 111 | desc->irq_data.common = &desc->irq_common_data; |
Thomas Gleixner | 1f5a5b8 | 2010-09-27 17:48:26 +0200 | [diff] [blame] | 112 | desc->irq_data.irq = irq; |
| 113 | desc->irq_data.chip = &no_irq_chip; |
| 114 | desc->irq_data.chip_data = NULL; |
Thomas Gleixner | f9e4989 | 2011-02-09 14:54:49 +0100 | [diff] [blame] | 115 | irq_settings_clr_and_set(desc, ~0, _IRQ_DEFAULT_INIT_FLAGS); |
Thomas Gleixner | 801a0e9 | 2011-03-27 11:02:49 +0200 | [diff] [blame] | 116 | irqd_set(&desc->irq_data, IRQD_IRQ_DISABLED); |
Jeffy Chen | d829b8f | 2017-06-26 19:33:33 +0800 | [diff] [blame] | 117 | irqd_set(&desc->irq_data, IRQD_IRQ_MASKED); |
Thomas Gleixner | 1f5a5b8 | 2010-09-27 17:48:26 +0200 | [diff] [blame] | 118 | desc->handle_irq = handle_bad_irq; |
| 119 | desc->depth = 1; |
Thomas Gleixner | b7b2933 | 2010-09-29 18:46:55 +0200 | [diff] [blame] | 120 | desc->irq_count = 0; |
| 121 | desc->irqs_unhandled = 0; |
Thomas Gleixner | 1f5a5b8 | 2010-09-27 17:48:26 +0200 | [diff] [blame] | 122 | desc->name = NULL; |
Sebastian Andrzej Siewior | b687380 | 2011-07-11 12:17:31 +0200 | [diff] [blame] | 123 | desc->owner = owner; |
Eric Dumazet | 6c9ae00 | 2011-01-13 15:45:38 -0800 | [diff] [blame] | 124 | for_each_possible_cpu(cpu) |
| 125 | *per_cpu_ptr(desc->kstat_irqs, cpu) = 0; |
Thomas Gleixner | 45ddcec | 2016-07-04 17:39:25 +0900 | [diff] [blame] | 126 | desc_smp_init(desc, node, affinity); |
Thomas Gleixner | 1f5a5b8 | 2010-09-27 17:48:26 +0200 | [diff] [blame] | 127 | } |
| 128 | |
Thomas Gleixner | 3795de2 | 2010-09-22 17:09:43 +0200 | [diff] [blame] | 129 | int nr_irqs = NR_IRQS; |
| 130 | EXPORT_SYMBOL_GPL(nr_irqs); |
| 131 | |
Thomas Gleixner | a05a900 | 2010-10-08 12:47:53 +0200 | [diff] [blame] | 132 | static DEFINE_MUTEX(sparse_irq_lock); |
Thomas Gleixner | c1ee626 | 2011-02-17 17:45:15 +0100 | [diff] [blame] | 133 | static DECLARE_BITMAP(allocated_irqs, IRQ_BITMAP_BITS); |
Thomas Gleixner | 1f5a5b8 | 2010-09-27 17:48:26 +0200 | [diff] [blame] | 134 | |
Thomas Gleixner | 3795de2 | 2010-09-22 17:09:43 +0200 | [diff] [blame] | 135 | #ifdef CONFIG_SPARSE_IRQ |
| 136 | |
Craig Gallek | ecb3f39 | 2016-09-13 12:14:51 -0400 | [diff] [blame] | 137 | static void irq_kobj_release(struct kobject *kobj); |
| 138 | |
| 139 | #ifdef CONFIG_SYSFS |
| 140 | static struct kobject *irq_kobj_base; |
| 141 | |
| 142 | #define IRQ_ATTR_RO(_name) \ |
| 143 | static struct kobj_attribute _name##_attr = __ATTR_RO(_name) |
| 144 | |
| 145 | static ssize_t per_cpu_count_show(struct kobject *kobj, |
| 146 | struct kobj_attribute *attr, char *buf) |
| 147 | { |
| 148 | struct irq_desc *desc = container_of(kobj, struct irq_desc, kobj); |
| 149 | int cpu, irq = desc->irq_data.irq; |
| 150 | ssize_t ret = 0; |
| 151 | char *p = ""; |
| 152 | |
| 153 | for_each_possible_cpu(cpu) { |
| 154 | unsigned int c = kstat_irqs_cpu(irq, cpu); |
| 155 | |
| 156 | ret += scnprintf(buf + ret, PAGE_SIZE - ret, "%s%u", p, c); |
| 157 | p = ","; |
| 158 | } |
| 159 | |
| 160 | ret += scnprintf(buf + ret, PAGE_SIZE - ret, "\n"); |
| 161 | return ret; |
| 162 | } |
| 163 | IRQ_ATTR_RO(per_cpu_count); |
| 164 | |
| 165 | static ssize_t chip_name_show(struct kobject *kobj, |
| 166 | struct kobj_attribute *attr, char *buf) |
| 167 | { |
| 168 | struct irq_desc *desc = container_of(kobj, struct irq_desc, kobj); |
| 169 | ssize_t ret = 0; |
| 170 | |
| 171 | raw_spin_lock_irq(&desc->lock); |
| 172 | if (desc->irq_data.chip && desc->irq_data.chip->name) { |
| 173 | ret = scnprintf(buf, PAGE_SIZE, "%s\n", |
| 174 | desc->irq_data.chip->name); |
| 175 | } |
| 176 | raw_spin_unlock_irq(&desc->lock); |
| 177 | |
| 178 | return ret; |
| 179 | } |
| 180 | IRQ_ATTR_RO(chip_name); |
| 181 | |
| 182 | static ssize_t hwirq_show(struct kobject *kobj, |
| 183 | struct kobj_attribute *attr, char *buf) |
| 184 | { |
| 185 | struct irq_desc *desc = container_of(kobj, struct irq_desc, kobj); |
| 186 | ssize_t ret = 0; |
| 187 | |
| 188 | raw_spin_lock_irq(&desc->lock); |
| 189 | if (desc->irq_data.domain) |
| 190 | ret = sprintf(buf, "%d\n", (int)desc->irq_data.hwirq); |
| 191 | raw_spin_unlock_irq(&desc->lock); |
| 192 | |
| 193 | return ret; |
| 194 | } |
| 195 | IRQ_ATTR_RO(hwirq); |
| 196 | |
| 197 | static ssize_t type_show(struct kobject *kobj, |
| 198 | struct kobj_attribute *attr, char *buf) |
| 199 | { |
| 200 | struct irq_desc *desc = container_of(kobj, struct irq_desc, kobj); |
| 201 | ssize_t ret = 0; |
| 202 | |
| 203 | raw_spin_lock_irq(&desc->lock); |
| 204 | ret = sprintf(buf, "%s\n", |
| 205 | irqd_is_level_type(&desc->irq_data) ? "level" : "edge"); |
| 206 | raw_spin_unlock_irq(&desc->lock); |
| 207 | |
| 208 | return ret; |
| 209 | |
| 210 | } |
| 211 | IRQ_ATTR_RO(type); |
| 212 | |
Andy Shevchenko | d61e294 | 2018-02-26 17:50:43 +0200 | [diff] [blame] | 213 | static ssize_t wakeup_show(struct kobject *kobj, |
| 214 | struct kobj_attribute *attr, char *buf) |
| 215 | { |
| 216 | struct irq_desc *desc = container_of(kobj, struct irq_desc, kobj); |
| 217 | ssize_t ret = 0; |
| 218 | |
| 219 | raw_spin_lock_irq(&desc->lock); |
| 220 | ret = sprintf(buf, "%s\n", |
| 221 | irqd_is_wakeup_set(&desc->irq_data) ? "enabled" : "disabled"); |
| 222 | raw_spin_unlock_irq(&desc->lock); |
| 223 | |
| 224 | return ret; |
| 225 | |
| 226 | } |
| 227 | IRQ_ATTR_RO(wakeup); |
| 228 | |
Craig Gallek | ecb3f39 | 2016-09-13 12:14:51 -0400 | [diff] [blame] | 229 | static ssize_t name_show(struct kobject *kobj, |
| 230 | struct kobj_attribute *attr, char *buf) |
| 231 | { |
| 232 | struct irq_desc *desc = container_of(kobj, struct irq_desc, kobj); |
| 233 | ssize_t ret = 0; |
| 234 | |
| 235 | raw_spin_lock_irq(&desc->lock); |
| 236 | if (desc->name) |
| 237 | ret = scnprintf(buf, PAGE_SIZE, "%s\n", desc->name); |
| 238 | raw_spin_unlock_irq(&desc->lock); |
| 239 | |
| 240 | return ret; |
| 241 | } |
| 242 | IRQ_ATTR_RO(name); |
| 243 | |
| 244 | static ssize_t actions_show(struct kobject *kobj, |
| 245 | struct kobj_attribute *attr, char *buf) |
| 246 | { |
| 247 | struct irq_desc *desc = container_of(kobj, struct irq_desc, kobj); |
| 248 | struct irqaction *action; |
| 249 | ssize_t ret = 0; |
| 250 | char *p = ""; |
| 251 | |
| 252 | raw_spin_lock_irq(&desc->lock); |
| 253 | for (action = desc->action; action != NULL; action = action->next) { |
| 254 | ret += scnprintf(buf + ret, PAGE_SIZE - ret, "%s%s", |
| 255 | p, action->name); |
| 256 | p = ","; |
| 257 | } |
| 258 | raw_spin_unlock_irq(&desc->lock); |
| 259 | |
| 260 | if (ret) |
| 261 | ret += scnprintf(buf + ret, PAGE_SIZE - ret, "\n"); |
| 262 | |
| 263 | return ret; |
| 264 | } |
| 265 | IRQ_ATTR_RO(actions); |
| 266 | |
| 267 | static struct attribute *irq_attrs[] = { |
| 268 | &per_cpu_count_attr.attr, |
| 269 | &chip_name_attr.attr, |
| 270 | &hwirq_attr.attr, |
| 271 | &type_attr.attr, |
Andy Shevchenko | d61e294 | 2018-02-26 17:50:43 +0200 | [diff] [blame] | 272 | &wakeup_attr.attr, |
Craig Gallek | ecb3f39 | 2016-09-13 12:14:51 -0400 | [diff] [blame] | 273 | &name_attr.attr, |
| 274 | &actions_attr.attr, |
| 275 | NULL |
| 276 | }; |
| 277 | |
| 278 | static struct kobj_type irq_kobj_type = { |
| 279 | .release = irq_kobj_release, |
| 280 | .sysfs_ops = &kobj_sysfs_ops, |
| 281 | .default_attrs = irq_attrs, |
| 282 | }; |
| 283 | |
| 284 | static void irq_sysfs_add(int irq, struct irq_desc *desc) |
| 285 | { |
| 286 | if (irq_kobj_base) { |
| 287 | /* |
| 288 | * Continue even in case of failure as this is nothing |
| 289 | * crucial. |
| 290 | */ |
| 291 | if (kobject_add(&desc->kobj, irq_kobj_base, "%d", irq)) |
| 292 | pr_warn("Failed to add kobject for irq %d\n", irq); |
| 293 | } |
| 294 | } |
| 295 | |
| 296 | static int __init irq_sysfs_init(void) |
| 297 | { |
| 298 | struct irq_desc *desc; |
| 299 | int irq; |
| 300 | |
| 301 | /* Prevent concurrent irq alloc/free */ |
| 302 | irq_lock_sparse(); |
| 303 | |
| 304 | irq_kobj_base = kobject_create_and_add("irq", kernel_kobj); |
| 305 | if (!irq_kobj_base) { |
| 306 | irq_unlock_sparse(); |
| 307 | return -ENOMEM; |
| 308 | } |
| 309 | |
| 310 | /* Add the already allocated interrupts */ |
| 311 | for_each_irq_desc(irq, desc) |
| 312 | irq_sysfs_add(irq, desc); |
| 313 | irq_unlock_sparse(); |
| 314 | |
| 315 | return 0; |
| 316 | } |
| 317 | postcore_initcall(irq_sysfs_init); |
| 318 | |
| 319 | #else /* !CONFIG_SYSFS */ |
| 320 | |
| 321 | static struct kobj_type irq_kobj_type = { |
| 322 | .release = irq_kobj_release, |
| 323 | }; |
| 324 | |
| 325 | static void irq_sysfs_add(int irq, struct irq_desc *desc) {} |
| 326 | |
| 327 | #endif /* CONFIG_SYSFS */ |
| 328 | |
Thomas Gleixner | baa0d23 | 2010-10-05 15:14:35 +0200 | [diff] [blame] | 329 | static RADIX_TREE(irq_desc_tree, GFP_KERNEL); |
Thomas Gleixner | 3795de2 | 2010-09-22 17:09:43 +0200 | [diff] [blame] | 330 | |
Thomas Gleixner | 1f5a5b8 | 2010-09-27 17:48:26 +0200 | [diff] [blame] | 331 | static void irq_insert_desc(unsigned int irq, struct irq_desc *desc) |
Thomas Gleixner | 3795de2 | 2010-09-22 17:09:43 +0200 | [diff] [blame] | 332 | { |
| 333 | radix_tree_insert(&irq_desc_tree, irq, desc); |
| 334 | } |
| 335 | |
| 336 | struct irq_desc *irq_to_desc(unsigned int irq) |
| 337 | { |
| 338 | return radix_tree_lookup(&irq_desc_tree, irq); |
| 339 | } |
Jiri Kosina | 3911ff3 | 2012-05-13 12:13:15 +0200 | [diff] [blame] | 340 | EXPORT_SYMBOL(irq_to_desc); |
Thomas Gleixner | 3795de2 | 2010-09-22 17:09:43 +0200 | [diff] [blame] | 341 | |
Thomas Gleixner | 1f5a5b8 | 2010-09-27 17:48:26 +0200 | [diff] [blame] | 342 | static void delete_irq_desc(unsigned int irq) |
| 343 | { |
| 344 | radix_tree_delete(&irq_desc_tree, irq); |
| 345 | } |
| 346 | |
| 347 | #ifdef CONFIG_SMP |
| 348 | static void free_masks(struct irq_desc *desc) |
| 349 | { |
| 350 | #ifdef CONFIG_GENERIC_PENDING_IRQ |
| 351 | free_cpumask_var(desc->pending_mask); |
| 352 | #endif |
Jiang Liu | 9df872f | 2015-06-03 11:47:50 +0800 | [diff] [blame] | 353 | free_cpumask_var(desc->irq_common_data.affinity); |
Thomas Gleixner | 0d3f542 | 2017-06-20 01:37:38 +0200 | [diff] [blame] | 354 | #ifdef CONFIG_GENERIC_IRQ_EFFECTIVE_AFF_MASK |
| 355 | free_cpumask_var(desc->irq_common_data.effective_affinity); |
| 356 | #endif |
Thomas Gleixner | 1f5a5b8 | 2010-09-27 17:48:26 +0200 | [diff] [blame] | 357 | } |
| 358 | #else |
| 359 | static inline void free_masks(struct irq_desc *desc) { } |
| 360 | #endif |
| 361 | |
Thomas Gleixner | c291ee6 | 2014-12-11 23:01:41 +0100 | [diff] [blame] | 362 | void irq_lock_sparse(void) |
| 363 | { |
| 364 | mutex_lock(&sparse_irq_lock); |
| 365 | } |
| 366 | |
| 367 | void irq_unlock_sparse(void) |
| 368 | { |
| 369 | mutex_unlock(&sparse_irq_lock); |
| 370 | } |
| 371 | |
Thomas Gleixner | 45ddcec | 2016-07-04 17:39:25 +0900 | [diff] [blame] | 372 | static struct irq_desc *alloc_desc(int irq, int node, unsigned int flags, |
| 373 | const struct cpumask *affinity, |
| 374 | struct module *owner) |
Thomas Gleixner | 1f5a5b8 | 2010-09-27 17:48:26 +0200 | [diff] [blame] | 375 | { |
| 376 | struct irq_desc *desc; |
Thomas Gleixner | 1f5a5b8 | 2010-09-27 17:48:26 +0200 | [diff] [blame] | 377 | |
Thomas Gleixner | 4ab764c | 2017-06-20 01:37:36 +0200 | [diff] [blame] | 378 | desc = kzalloc_node(sizeof(*desc), GFP_KERNEL, node); |
Thomas Gleixner | 1f5a5b8 | 2010-09-27 17:48:26 +0200 | [diff] [blame] | 379 | if (!desc) |
| 380 | return NULL; |
| 381 | /* allocate based on nr_cpu_ids */ |
Eric Dumazet | 6c9ae00 | 2011-01-13 15:45:38 -0800 | [diff] [blame] | 382 | desc->kstat_irqs = alloc_percpu(unsigned int); |
Thomas Gleixner | 1f5a5b8 | 2010-09-27 17:48:26 +0200 | [diff] [blame] | 383 | if (!desc->kstat_irqs) |
| 384 | goto err_desc; |
| 385 | |
Thomas Gleixner | 4ab764c | 2017-06-20 01:37:36 +0200 | [diff] [blame] | 386 | if (alloc_masks(desc, node)) |
Thomas Gleixner | 1f5a5b8 | 2010-09-27 17:48:26 +0200 | [diff] [blame] | 387 | goto err_kstat; |
| 388 | |
| 389 | raw_spin_lock_init(&desc->lock); |
| 390 | lockdep_set_class(&desc->lock, &irq_desc_lock_class); |
Thomas Gleixner | 9114014 | 2017-06-29 23:33:37 +0200 | [diff] [blame] | 391 | mutex_init(&desc->request_mutex); |
Thomas Gleixner | 425a5072 | 2015-12-13 18:02:22 +0100 | [diff] [blame] | 392 | init_rcu_head(&desc->rcu); |
Thomas Gleixner | 1f5a5b8 | 2010-09-27 17:48:26 +0200 | [diff] [blame] | 393 | |
Thomas Gleixner | 45ddcec | 2016-07-04 17:39:25 +0900 | [diff] [blame] | 394 | desc_set_defaults(irq, desc, node, affinity, owner); |
| 395 | irqd_set(&desc->irq_data, flags); |
Craig Gallek | ecb3f39 | 2016-09-13 12:14:51 -0400 | [diff] [blame] | 396 | kobject_init(&desc->kobj, &irq_kobj_type); |
Thomas Gleixner | 1f5a5b8 | 2010-09-27 17:48:26 +0200 | [diff] [blame] | 397 | |
| 398 | return desc; |
| 399 | |
| 400 | err_kstat: |
Eric Dumazet | 6c9ae00 | 2011-01-13 15:45:38 -0800 | [diff] [blame] | 401 | free_percpu(desc->kstat_irqs); |
Thomas Gleixner | 1f5a5b8 | 2010-09-27 17:48:26 +0200 | [diff] [blame] | 402 | err_desc: |
| 403 | kfree(desc); |
| 404 | return NULL; |
| 405 | } |
| 406 | |
Craig Gallek | ecb3f39 | 2016-09-13 12:14:51 -0400 | [diff] [blame] | 407 | static void irq_kobj_release(struct kobject *kobj) |
Thomas Gleixner | 425a5072 | 2015-12-13 18:02:22 +0100 | [diff] [blame] | 408 | { |
Craig Gallek | ecb3f39 | 2016-09-13 12:14:51 -0400 | [diff] [blame] | 409 | struct irq_desc *desc = container_of(kobj, struct irq_desc, kobj); |
Thomas Gleixner | 425a5072 | 2015-12-13 18:02:22 +0100 | [diff] [blame] | 410 | |
| 411 | free_masks(desc); |
| 412 | free_percpu(desc->kstat_irqs); |
| 413 | kfree(desc); |
| 414 | } |
| 415 | |
Craig Gallek | ecb3f39 | 2016-09-13 12:14:51 -0400 | [diff] [blame] | 416 | static void delayed_free_desc(struct rcu_head *rhp) |
| 417 | { |
| 418 | struct irq_desc *desc = container_of(rhp, struct irq_desc, rcu); |
| 419 | |
| 420 | kobject_put(&desc->kobj); |
| 421 | } |
| 422 | |
Thomas Gleixner | 1f5a5b8 | 2010-09-27 17:48:26 +0200 | [diff] [blame] | 423 | static void free_desc(unsigned int irq) |
| 424 | { |
| 425 | struct irq_desc *desc = irq_to_desc(irq); |
Thomas Gleixner | 1f5a5b8 | 2010-09-27 17:48:26 +0200 | [diff] [blame] | 426 | |
Thomas Gleixner | 087cdfb | 2017-06-20 01:37:17 +0200 | [diff] [blame] | 427 | irq_remove_debugfs_entry(desc); |
Thomas Gleixner | 13bfe99 | 2010-09-30 02:46:07 +0200 | [diff] [blame] | 428 | unregister_irq_proc(irq, desc); |
| 429 | |
Thomas Gleixner | c291ee6 | 2014-12-11 23:01:41 +0100 | [diff] [blame] | 430 | /* |
| 431 | * sparse_irq_lock protects also show_interrupts() and |
| 432 | * kstat_irq_usr(). Once we deleted the descriptor from the |
| 433 | * sparse tree we can free it. Access in proc will fail to |
| 434 | * lookup the descriptor. |
Craig Gallek | ecb3f39 | 2016-09-13 12:14:51 -0400 | [diff] [blame] | 435 | * |
| 436 | * The sysfs entry must be serialized against a concurrent |
| 437 | * irq_sysfs_init() as well. |
Thomas Gleixner | c291ee6 | 2014-12-11 23:01:41 +0100 | [diff] [blame] | 438 | */ |
Craig Gallek | ecb3f39 | 2016-09-13 12:14:51 -0400 | [diff] [blame] | 439 | kobject_del(&desc->kobj); |
Thomas Gleixner | 1f5a5b8 | 2010-09-27 17:48:26 +0200 | [diff] [blame] | 440 | delete_irq_desc(irq); |
Thomas Gleixner | 1f5a5b8 | 2010-09-27 17:48:26 +0200 | [diff] [blame] | 441 | |
Thomas Gleixner | 425a5072 | 2015-12-13 18:02:22 +0100 | [diff] [blame] | 442 | /* |
| 443 | * We free the descriptor, masks and stat fields via RCU. That |
| 444 | * allows demultiplex interrupts to do rcu based management of |
| 445 | * the child interrupts. |
Eric Dumazet | 4a5f4d2 | 2018-06-18 05:56:12 -0700 | [diff] [blame] | 446 | * This also allows us to use rcu in kstat_irqs_usr(). |
Thomas Gleixner | 425a5072 | 2015-12-13 18:02:22 +0100 | [diff] [blame] | 447 | */ |
| 448 | call_rcu(&desc->rcu, delayed_free_desc); |
Thomas Gleixner | 1f5a5b8 | 2010-09-27 17:48:26 +0200 | [diff] [blame] | 449 | } |
| 450 | |
Sebastian Andrzej Siewior | b687380 | 2011-07-11 12:17:31 +0200 | [diff] [blame] | 451 | static int alloc_descs(unsigned int start, unsigned int cnt, int node, |
Dou Liyang | bec0403 | 2018-12-04 23:51:20 +0800 | [diff] [blame] | 452 | const struct irq_affinity_desc *affinity, |
| 453 | struct module *owner) |
Thomas Gleixner | 1f5a5b8 | 2010-09-27 17:48:26 +0200 | [diff] [blame] | 454 | { |
| 455 | struct irq_desc *desc; |
Thomas Gleixner | e75eafb | 2016-09-14 16:18:49 +0200 | [diff] [blame] | 456 | int i; |
Thomas Gleixner | 45ddcec | 2016-07-04 17:39:25 +0900 | [diff] [blame] | 457 | |
Thomas Gleixner | e75eafb | 2016-09-14 16:18:49 +0200 | [diff] [blame] | 458 | /* Validate affinity mask(s) */ |
| 459 | if (affinity) { |
Huacai Chen | 12fee4c | 2019-01-17 11:00:09 +0800 | [diff] [blame] | 460 | for (i = 0; i < cnt; i++) { |
Dou Liyang | bec0403 | 2018-12-04 23:51:20 +0800 | [diff] [blame] | 461 | if (cpumask_empty(&affinity[i].mask)) |
Thomas Gleixner | e75eafb | 2016-09-14 16:18:49 +0200 | [diff] [blame] | 462 | return -EINVAL; |
| 463 | } |
| 464 | } |
Thomas Gleixner | 45ddcec | 2016-07-04 17:39:25 +0900 | [diff] [blame] | 465 | |
Thomas Gleixner | 1f5a5b8 | 2010-09-27 17:48:26 +0200 | [diff] [blame] | 466 | for (i = 0; i < cnt; i++) { |
Dou Liyang | bec0403 | 2018-12-04 23:51:20 +0800 | [diff] [blame] | 467 | const struct cpumask *mask = NULL; |
Dou Liyang | c410abbb | 2018-12-04 23:51:21 +0800 | [diff] [blame] | 468 | unsigned int flags = 0; |
Dou Liyang | bec0403 | 2018-12-04 23:51:20 +0800 | [diff] [blame] | 469 | |
Thomas Gleixner | 45ddcec | 2016-07-04 17:39:25 +0900 | [diff] [blame] | 470 | if (affinity) { |
Dou Liyang | c410abbb | 2018-12-04 23:51:21 +0800 | [diff] [blame] | 471 | if (affinity->is_managed) { |
| 472 | flags = IRQD_AFFINITY_MANAGED | |
| 473 | IRQD_MANAGED_SHUTDOWN; |
| 474 | } |
Dou Liyang | bec0403 | 2018-12-04 23:51:20 +0800 | [diff] [blame] | 475 | mask = &affinity->mask; |
Dou Liyang | c410abbb | 2018-12-04 23:51:21 +0800 | [diff] [blame] | 476 | node = cpu_to_node(cpumask_first(mask)); |
Thomas Gleixner | e75eafb | 2016-09-14 16:18:49 +0200 | [diff] [blame] | 477 | affinity++; |
Thomas Gleixner | 45ddcec | 2016-07-04 17:39:25 +0900 | [diff] [blame] | 478 | } |
Dou Liyang | c410abbb | 2018-12-04 23:51:21 +0800 | [diff] [blame] | 479 | |
Thomas Gleixner | 45ddcec | 2016-07-04 17:39:25 +0900 | [diff] [blame] | 480 | desc = alloc_desc(start + i, node, flags, mask, owner); |
Thomas Gleixner | 1f5a5b8 | 2010-09-27 17:48:26 +0200 | [diff] [blame] | 481 | if (!desc) |
| 482 | goto err; |
Thomas Gleixner | 1f5a5b8 | 2010-09-27 17:48:26 +0200 | [diff] [blame] | 483 | irq_insert_desc(start + i, desc); |
Craig Gallek | ecb3f39 | 2016-09-13 12:14:51 -0400 | [diff] [blame] | 484 | irq_sysfs_add(start + i, desc); |
Thomas Gleixner | e0b4779 | 2017-09-13 23:29:04 +0200 | [diff] [blame] | 485 | irq_add_debugfs_entry(start + i, desc); |
Thomas Gleixner | 1f5a5b8 | 2010-09-27 17:48:26 +0200 | [diff] [blame] | 486 | } |
Thomas Gleixner | 12ac1d0 | 2017-09-05 10:12:20 +0200 | [diff] [blame] | 487 | bitmap_set(allocated_irqs, start, cnt); |
Thomas Gleixner | 1f5a5b8 | 2010-09-27 17:48:26 +0200 | [diff] [blame] | 488 | return start; |
| 489 | |
| 490 | err: |
| 491 | for (i--; i >= 0; i--) |
| 492 | free_desc(start + i); |
Thomas Gleixner | 1f5a5b8 | 2010-09-27 17:48:26 +0200 | [diff] [blame] | 493 | return -ENOMEM; |
| 494 | } |
| 495 | |
Yinghai Lu | ed4dea6 | 2011-02-19 11:07:37 -0800 | [diff] [blame] | 496 | static int irq_expand_nr_irqs(unsigned int nr) |
Thomas Gleixner | e7bcecb | 2011-02-16 17:12:57 +0100 | [diff] [blame] | 497 | { |
Yinghai Lu | ed4dea6 | 2011-02-19 11:07:37 -0800 | [diff] [blame] | 498 | if (nr > IRQ_BITMAP_BITS) |
Thomas Gleixner | e7bcecb | 2011-02-16 17:12:57 +0100 | [diff] [blame] | 499 | return -ENOMEM; |
Yinghai Lu | ed4dea6 | 2011-02-19 11:07:37 -0800 | [diff] [blame] | 500 | nr_irqs = nr; |
Thomas Gleixner | e7bcecb | 2011-02-16 17:12:57 +0100 | [diff] [blame] | 501 | return 0; |
| 502 | } |
| 503 | |
Thomas Gleixner | 3795de2 | 2010-09-22 17:09:43 +0200 | [diff] [blame] | 504 | int __init early_irq_init(void) |
| 505 | { |
Thomas Gleixner | b683de2 | 2010-09-27 20:55:03 +0200 | [diff] [blame] | 506 | int i, initcnt, node = first_online_node; |
Thomas Gleixner | 3795de2 | 2010-09-22 17:09:43 +0200 | [diff] [blame] | 507 | struct irq_desc *desc; |
Thomas Gleixner | 3795de2 | 2010-09-22 17:09:43 +0200 | [diff] [blame] | 508 | |
| 509 | init_irq_default_affinity(); |
| 510 | |
Thomas Gleixner | b683de2 | 2010-09-27 20:55:03 +0200 | [diff] [blame] | 511 | /* Let arch update nr_irqs and return the nr of preallocated irqs */ |
| 512 | initcnt = arch_probe_nr_irqs(); |
Vincent Legoll | 5a29ef2 | 2017-05-09 10:34:09 +0200 | [diff] [blame] | 513 | printk(KERN_INFO "NR_IRQS: %d, nr_irqs: %d, preallocated irqs: %d\n", |
| 514 | NR_IRQS, nr_irqs, initcnt); |
Thomas Gleixner | 3795de2 | 2010-09-22 17:09:43 +0200 | [diff] [blame] | 515 | |
Thomas Gleixner | c1ee626 | 2011-02-17 17:45:15 +0100 | [diff] [blame] | 516 | if (WARN_ON(nr_irqs > IRQ_BITMAP_BITS)) |
| 517 | nr_irqs = IRQ_BITMAP_BITS; |
| 518 | |
| 519 | if (WARN_ON(initcnt > IRQ_BITMAP_BITS)) |
| 520 | initcnt = IRQ_BITMAP_BITS; |
| 521 | |
| 522 | if (initcnt > nr_irqs) |
| 523 | nr_irqs = initcnt; |
| 524 | |
Thomas Gleixner | b683de2 | 2010-09-27 20:55:03 +0200 | [diff] [blame] | 525 | for (i = 0; i < initcnt; i++) { |
Thomas Gleixner | 45ddcec | 2016-07-04 17:39:25 +0900 | [diff] [blame] | 526 | desc = alloc_desc(i, node, 0, NULL, NULL); |
Thomas Gleixner | aa99ec0 | 2010-09-27 20:02:56 +0200 | [diff] [blame] | 527 | set_bit(i, allocated_irqs); |
| 528 | irq_insert_desc(i, desc); |
Thomas Gleixner | 3795de2 | 2010-09-22 17:09:43 +0200 | [diff] [blame] | 529 | } |
Thomas Gleixner | 3795de2 | 2010-09-22 17:09:43 +0200 | [diff] [blame] | 530 | return arch_early_irq_init(); |
| 531 | } |
| 532 | |
Thomas Gleixner | 3795de2 | 2010-09-22 17:09:43 +0200 | [diff] [blame] | 533 | #else /* !CONFIG_SPARSE_IRQ */ |
| 534 | |
| 535 | struct irq_desc irq_desc[NR_IRQS] __cacheline_aligned_in_smp = { |
| 536 | [0 ... NR_IRQS-1] = { |
Thomas Gleixner | 3795de2 | 2010-09-22 17:09:43 +0200 | [diff] [blame] | 537 | .handle_irq = handle_bad_irq, |
| 538 | .depth = 1, |
| 539 | .lock = __RAW_SPIN_LOCK_UNLOCKED(irq_desc->lock), |
| 540 | } |
| 541 | }; |
| 542 | |
Thomas Gleixner | 3795de2 | 2010-09-22 17:09:43 +0200 | [diff] [blame] | 543 | int __init early_irq_init(void) |
| 544 | { |
Thomas Gleixner | aa99ec0 | 2010-09-27 20:02:56 +0200 | [diff] [blame] | 545 | int count, i, node = first_online_node; |
Thomas Gleixner | 3795de2 | 2010-09-22 17:09:43 +0200 | [diff] [blame] | 546 | struct irq_desc *desc; |
Thomas Gleixner | 3795de2 | 2010-09-22 17:09:43 +0200 | [diff] [blame] | 547 | |
| 548 | init_irq_default_affinity(); |
| 549 | |
Vincent Legoll | 5a29ef2 | 2017-05-09 10:34:09 +0200 | [diff] [blame] | 550 | printk(KERN_INFO "NR_IRQS: %d\n", NR_IRQS); |
Thomas Gleixner | 3795de2 | 2010-09-22 17:09:43 +0200 | [diff] [blame] | 551 | |
| 552 | desc = irq_desc; |
| 553 | count = ARRAY_SIZE(irq_desc); |
| 554 | |
| 555 | for (i = 0; i < count; i++) { |
Eric Dumazet | 6c9ae00 | 2011-01-13 15:45:38 -0800 | [diff] [blame] | 556 | desc[i].kstat_irqs = alloc_percpu(unsigned int); |
Thomas Gleixner | 4ab764c | 2017-06-20 01:37:36 +0200 | [diff] [blame] | 557 | alloc_masks(&desc[i], node); |
Linus Walleij | e7fbad3 | 2011-05-31 18:14:39 +0200 | [diff] [blame] | 558 | raw_spin_lock_init(&desc[i].lock); |
Thomas Gleixner | 154cd38 | 2010-09-22 15:58:45 +0200 | [diff] [blame] | 559 | lockdep_set_class(&desc[i].lock, &irq_desc_lock_class); |
Thomas Gleixner | 45ddcec | 2016-07-04 17:39:25 +0900 | [diff] [blame] | 560 | desc_set_defaults(i, &desc[i], node, NULL, NULL); |
Thomas Gleixner | 3795de2 | 2010-09-22 17:09:43 +0200 | [diff] [blame] | 561 | } |
| 562 | return arch_early_irq_init(); |
| 563 | } |
| 564 | |
| 565 | struct irq_desc *irq_to_desc(unsigned int irq) |
| 566 | { |
| 567 | return (irq < NR_IRQS) ? irq_desc + irq : NULL; |
| 568 | } |
Paul Gortmaker | 2c45aad | 2014-02-10 13:39:53 -0500 | [diff] [blame] | 569 | EXPORT_SYMBOL(irq_to_desc); |
Thomas Gleixner | 3795de2 | 2010-09-22 17:09:43 +0200 | [diff] [blame] | 570 | |
Thomas Gleixner | 1f5a5b8 | 2010-09-27 17:48:26 +0200 | [diff] [blame] | 571 | static void free_desc(unsigned int irq) |
| 572 | { |
Thomas Gleixner | d8179bc | 2014-05-07 15:44:23 +0000 | [diff] [blame] | 573 | struct irq_desc *desc = irq_to_desc(irq); |
| 574 | unsigned long flags; |
| 575 | |
| 576 | raw_spin_lock_irqsave(&desc->lock, flags); |
Thomas Gleixner | 45ddcec | 2016-07-04 17:39:25 +0900 | [diff] [blame] | 577 | desc_set_defaults(irq, desc, irq_desc_get_node(desc), NULL, NULL); |
Thomas Gleixner | d8179bc | 2014-05-07 15:44:23 +0000 | [diff] [blame] | 578 | raw_spin_unlock_irqrestore(&desc->lock, flags); |
Thomas Gleixner | 1f5a5b8 | 2010-09-27 17:48:26 +0200 | [diff] [blame] | 579 | } |
| 580 | |
Sebastian Andrzej Siewior | b687380 | 2011-07-11 12:17:31 +0200 | [diff] [blame] | 581 | static inline int alloc_descs(unsigned int start, unsigned int cnt, int node, |
Dou Liyang | bec0403 | 2018-12-04 23:51:20 +0800 | [diff] [blame] | 582 | const struct irq_affinity_desc *affinity, |
Sebastian Andrzej Siewior | b687380 | 2011-07-11 12:17:31 +0200 | [diff] [blame] | 583 | struct module *owner) |
Thomas Gleixner | 1f5a5b8 | 2010-09-27 17:48:26 +0200 | [diff] [blame] | 584 | { |
Sebastian Andrzej Siewior | b687380 | 2011-07-11 12:17:31 +0200 | [diff] [blame] | 585 | u32 i; |
| 586 | |
| 587 | for (i = 0; i < cnt; i++) { |
| 588 | struct irq_desc *desc = irq_to_desc(start + i); |
| 589 | |
| 590 | desc->owner = owner; |
| 591 | } |
Thomas Gleixner | 12ac1d0 | 2017-09-05 10:12:20 +0200 | [diff] [blame] | 592 | bitmap_set(allocated_irqs, start, cnt); |
Thomas Gleixner | 1f5a5b8 | 2010-09-27 17:48:26 +0200 | [diff] [blame] | 593 | return start; |
| 594 | } |
Thomas Gleixner | e7bcecb | 2011-02-16 17:12:57 +0100 | [diff] [blame] | 595 | |
Yinghai Lu | ed4dea6 | 2011-02-19 11:07:37 -0800 | [diff] [blame] | 596 | static int irq_expand_nr_irqs(unsigned int nr) |
Thomas Gleixner | e7bcecb | 2011-02-16 17:12:57 +0100 | [diff] [blame] | 597 | { |
| 598 | return -ENOMEM; |
| 599 | } |
| 600 | |
Thomas Gleixner | f63b6a0 | 2014-05-07 15:44:21 +0000 | [diff] [blame] | 601 | void irq_mark_irq(unsigned int irq) |
| 602 | { |
| 603 | mutex_lock(&sparse_irq_lock); |
| 604 | bitmap_set(allocated_irqs, irq, 1); |
| 605 | mutex_unlock(&sparse_irq_lock); |
| 606 | } |
| 607 | |
Thomas Gleixner | c940e01 | 2014-05-07 15:44:22 +0000 | [diff] [blame] | 608 | #ifdef CONFIG_GENERIC_IRQ_LEGACY |
| 609 | void irq_init_desc(unsigned int irq) |
| 610 | { |
Thomas Gleixner | d8179bc | 2014-05-07 15:44:23 +0000 | [diff] [blame] | 611 | free_desc(irq); |
Thomas Gleixner | c940e01 | 2014-05-07 15:44:22 +0000 | [diff] [blame] | 612 | } |
| 613 | #endif |
| 614 | |
Thomas Gleixner | 3795de2 | 2010-09-22 17:09:43 +0200 | [diff] [blame] | 615 | #endif /* !CONFIG_SPARSE_IRQ */ |
| 616 | |
Thomas Gleixner | fe12bc2 | 2011-05-18 12:48:00 +0200 | [diff] [blame] | 617 | /** |
| 618 | * generic_handle_irq - Invoke the handler for a particular irq |
| 619 | * @irq: The irq number to handle |
| 620 | * |
| 621 | */ |
| 622 | int generic_handle_irq(unsigned int irq) |
| 623 | { |
| 624 | struct irq_desc *desc = irq_to_desc(irq); |
| 625 | |
| 626 | if (!desc) |
| 627 | return -EINVAL; |
Thomas Gleixner | bd0b9ac | 2015-09-14 10:42:37 +0200 | [diff] [blame] | 628 | generic_handle_irq_desc(desc); |
Thomas Gleixner | fe12bc2 | 2011-05-18 12:48:00 +0200 | [diff] [blame] | 629 | return 0; |
| 630 | } |
Jonathan Cameron | edf76f8 | 2011-05-18 10:39:04 +0100 | [diff] [blame] | 631 | EXPORT_SYMBOL_GPL(generic_handle_irq); |
Thomas Gleixner | fe12bc2 | 2011-05-18 12:48:00 +0200 | [diff] [blame] | 632 | |
Marc Zyngier | 76ba59f | 2014-08-26 11:03:16 +0100 | [diff] [blame] | 633 | #ifdef CONFIG_HANDLE_DOMAIN_IRQ |
| 634 | /** |
| 635 | * __handle_domain_irq - Invoke the handler for a HW irq belonging to a domain |
| 636 | * @domain: The domain where to perform the lookup |
| 637 | * @hwirq: The HW irq number to convert to a logical one |
| 638 | * @lookup: Whether to perform the domain lookup or not |
| 639 | * @regs: Register file coming from the low-level handling code |
| 640 | * |
| 641 | * Returns: 0 on success, or -EINVAL if conversion has failed |
| 642 | */ |
| 643 | int __handle_domain_irq(struct irq_domain *domain, unsigned int hwirq, |
| 644 | bool lookup, struct pt_regs *regs) |
| 645 | { |
| 646 | struct pt_regs *old_regs = set_irq_regs(regs); |
| 647 | unsigned int irq = hwirq; |
| 648 | int ret = 0; |
| 649 | |
| 650 | irq_enter(); |
| 651 | |
| 652 | #ifdef CONFIG_IRQ_DOMAIN |
| 653 | if (lookup) |
| 654 | irq = irq_find_mapping(domain, hwirq); |
| 655 | #endif |
| 656 | |
| 657 | /* |
| 658 | * Some hardware gives randomly wrong interrupts. Rather |
| 659 | * than crashing, do something sensible. |
| 660 | */ |
| 661 | if (unlikely(!irq || irq >= nr_irqs)) { |
| 662 | ack_bad_irq(irq); |
| 663 | ret = -EINVAL; |
| 664 | } else { |
| 665 | generic_handle_irq(irq); |
| 666 | } |
| 667 | |
| 668 | irq_exit(); |
| 669 | set_irq_regs(old_regs); |
| 670 | return ret; |
| 671 | } |
| 672 | #endif |
| 673 | |
Thomas Gleixner | 1f5a5b8 | 2010-09-27 17:48:26 +0200 | [diff] [blame] | 674 | /* Dynamic interrupt handling */ |
| 675 | |
| 676 | /** |
| 677 | * irq_free_descs - free irq descriptors |
| 678 | * @from: Start of descriptor range |
| 679 | * @cnt: Number of consecutive irqs to free |
| 680 | */ |
| 681 | void irq_free_descs(unsigned int from, unsigned int cnt) |
| 682 | { |
Thomas Gleixner | 1f5a5b8 | 2010-09-27 17:48:26 +0200 | [diff] [blame] | 683 | int i; |
| 684 | |
| 685 | if (from >= nr_irqs || (from + cnt) > nr_irqs) |
| 686 | return; |
| 687 | |
Thomas Gleixner | 12ac1d0 | 2017-09-05 10:12:20 +0200 | [diff] [blame] | 688 | mutex_lock(&sparse_irq_lock); |
Thomas Gleixner | 1f5a5b8 | 2010-09-27 17:48:26 +0200 | [diff] [blame] | 689 | for (i = 0; i < cnt; i++) |
| 690 | free_desc(from + i); |
| 691 | |
Thomas Gleixner | 1f5a5b8 | 2010-09-27 17:48:26 +0200 | [diff] [blame] | 692 | bitmap_clear(allocated_irqs, from, cnt); |
Thomas Gleixner | a05a900 | 2010-10-08 12:47:53 +0200 | [diff] [blame] | 693 | mutex_unlock(&sparse_irq_lock); |
Thomas Gleixner | 1f5a5b8 | 2010-09-27 17:48:26 +0200 | [diff] [blame] | 694 | } |
Jonathan Cameron | edf76f8 | 2011-05-18 10:39:04 +0100 | [diff] [blame] | 695 | EXPORT_SYMBOL_GPL(irq_free_descs); |
Thomas Gleixner | 1f5a5b8 | 2010-09-27 17:48:26 +0200 | [diff] [blame] | 696 | |
| 697 | /** |
| 698 | * irq_alloc_descs - allocate and initialize a range of irq descriptors |
| 699 | * @irq: Allocate for specific irq number if irq >= 0 |
| 700 | * @from: Start the search from this irq number |
| 701 | * @cnt: Number of consecutive irqs to allocate. |
| 702 | * @node: Preferred node on which the irq descriptor should be allocated |
Randy Dunlap | d522a0d | 2011-08-18 12:19:27 -0700 | [diff] [blame] | 703 | * @owner: Owning module (can be NULL) |
Thomas Gleixner | e75eafb | 2016-09-14 16:18:49 +0200 | [diff] [blame] | 704 | * @affinity: Optional pointer to an affinity mask array of size @cnt which |
| 705 | * hints where the irq descriptors should be allocated and which |
| 706 | * default affinities to use |
Thomas Gleixner | 1f5a5b8 | 2010-09-27 17:48:26 +0200 | [diff] [blame] | 707 | * |
| 708 | * Returns the first irq number or error code |
| 709 | */ |
| 710 | int __ref |
Sebastian Andrzej Siewior | b687380 | 2011-07-11 12:17:31 +0200 | [diff] [blame] | 711 | __irq_alloc_descs(int irq, unsigned int from, unsigned int cnt, int node, |
Dou Liyang | bec0403 | 2018-12-04 23:51:20 +0800 | [diff] [blame] | 712 | struct module *owner, const struct irq_affinity_desc *affinity) |
Thomas Gleixner | 1f5a5b8 | 2010-09-27 17:48:26 +0200 | [diff] [blame] | 713 | { |
Thomas Gleixner | 1f5a5b8 | 2010-09-27 17:48:26 +0200 | [diff] [blame] | 714 | int start, ret; |
| 715 | |
| 716 | if (!cnt) |
| 717 | return -EINVAL; |
| 718 | |
Mark Brown | c5182b8 | 2011-06-02 18:55:13 +0100 | [diff] [blame] | 719 | if (irq >= 0) { |
| 720 | if (from > irq) |
| 721 | return -EINVAL; |
| 722 | from = irq; |
Thomas Gleixner | 62a08ae | 2014-04-24 09:50:53 +0200 | [diff] [blame] | 723 | } else { |
| 724 | /* |
| 725 | * For interrupts which are freely allocated the |
| 726 | * architecture can force a lower bound to the @from |
| 727 | * argument. x86 uses this to exclude the GSI space. |
| 728 | */ |
| 729 | from = arch_dynirq_lower_bound(from); |
Mark Brown | c5182b8 | 2011-06-02 18:55:13 +0100 | [diff] [blame] | 730 | } |
| 731 | |
Thomas Gleixner | a05a900 | 2010-10-08 12:47:53 +0200 | [diff] [blame] | 732 | mutex_lock(&sparse_irq_lock); |
Thomas Gleixner | 1f5a5b8 | 2010-09-27 17:48:26 +0200 | [diff] [blame] | 733 | |
Yinghai Lu | ed4dea6 | 2011-02-19 11:07:37 -0800 | [diff] [blame] | 734 | start = bitmap_find_next_zero_area(allocated_irqs, IRQ_BITMAP_BITS, |
| 735 | from, cnt, 0); |
Thomas Gleixner | 1f5a5b8 | 2010-09-27 17:48:26 +0200 | [diff] [blame] | 736 | ret = -EEXIST; |
| 737 | if (irq >=0 && start != irq) |
Thomas Gleixner | 12ac1d0 | 2017-09-05 10:12:20 +0200 | [diff] [blame] | 738 | goto unlock; |
Thomas Gleixner | 1f5a5b8 | 2010-09-27 17:48:26 +0200 | [diff] [blame] | 739 | |
Yinghai Lu | ed4dea6 | 2011-02-19 11:07:37 -0800 | [diff] [blame] | 740 | if (start + cnt > nr_irqs) { |
| 741 | ret = irq_expand_nr_irqs(start + cnt); |
Thomas Gleixner | e7bcecb | 2011-02-16 17:12:57 +0100 | [diff] [blame] | 742 | if (ret) |
Thomas Gleixner | 12ac1d0 | 2017-09-05 10:12:20 +0200 | [diff] [blame] | 743 | goto unlock; |
Thomas Gleixner | e7bcecb | 2011-02-16 17:12:57 +0100 | [diff] [blame] | 744 | } |
Thomas Gleixner | 12ac1d0 | 2017-09-05 10:12:20 +0200 | [diff] [blame] | 745 | ret = alloc_descs(start, cnt, node, affinity, owner); |
| 746 | unlock: |
Thomas Gleixner | a05a900 | 2010-10-08 12:47:53 +0200 | [diff] [blame] | 747 | mutex_unlock(&sparse_irq_lock); |
Thomas Gleixner | 1f5a5b8 | 2010-09-27 17:48:26 +0200 | [diff] [blame] | 748 | return ret; |
| 749 | } |
Sebastian Andrzej Siewior | b687380 | 2011-07-11 12:17:31 +0200 | [diff] [blame] | 750 | EXPORT_SYMBOL_GPL(__irq_alloc_descs); |
Thomas Gleixner | 1f5a5b8 | 2010-09-27 17:48:26 +0200 | [diff] [blame] | 751 | |
Thomas Gleixner | 7b6ef12 | 2014-05-07 15:44:05 +0000 | [diff] [blame] | 752 | #ifdef CONFIG_GENERIC_IRQ_LEGACY_ALLOC_HWIRQ |
| 753 | /** |
| 754 | * irq_alloc_hwirqs - Allocate an irq descriptor and initialize the hardware |
| 755 | * @cnt: number of interrupts to allocate |
| 756 | * @node: node on which to allocate |
| 757 | * |
| 758 | * Returns an interrupt number > 0 or 0, if the allocation fails. |
| 759 | */ |
| 760 | unsigned int irq_alloc_hwirqs(int cnt, int node) |
| 761 | { |
Thomas Gleixner | 06ee6d5 | 2016-07-04 17:39:24 +0900 | [diff] [blame] | 762 | int i, irq = __irq_alloc_descs(-1, 0, cnt, node, NULL, NULL); |
Thomas Gleixner | 7b6ef12 | 2014-05-07 15:44:05 +0000 | [diff] [blame] | 763 | |
| 764 | if (irq < 0) |
| 765 | return 0; |
| 766 | |
| 767 | for (i = irq; cnt > 0; i++, cnt--) { |
| 768 | if (arch_setup_hwirq(i, node)) |
| 769 | goto err; |
| 770 | irq_clear_status_flags(i, _IRQ_NOREQUEST); |
| 771 | } |
| 772 | return irq; |
| 773 | |
| 774 | err: |
| 775 | for (i--; i >= irq; i--) { |
| 776 | irq_set_status_flags(i, _IRQ_NOREQUEST | _IRQ_NOPROBE); |
| 777 | arch_teardown_hwirq(i); |
| 778 | } |
| 779 | irq_free_descs(irq, cnt); |
| 780 | return 0; |
| 781 | } |
| 782 | EXPORT_SYMBOL_GPL(irq_alloc_hwirqs); |
| 783 | |
| 784 | /** |
| 785 | * irq_free_hwirqs - Free irq descriptor and cleanup the hardware |
| 786 | * @from: Free from irq number |
| 787 | * @cnt: number of interrupts to free |
| 788 | * |
| 789 | */ |
| 790 | void irq_free_hwirqs(unsigned int from, int cnt) |
| 791 | { |
Keith Busch | 8844aad | 2014-06-30 16:24:44 -0600 | [diff] [blame] | 792 | int i, j; |
Thomas Gleixner | 7b6ef12 | 2014-05-07 15:44:05 +0000 | [diff] [blame] | 793 | |
Keith Busch | 8844aad | 2014-06-30 16:24:44 -0600 | [diff] [blame] | 794 | for (i = from, j = cnt; j > 0; i++, j--) { |
Thomas Gleixner | 7b6ef12 | 2014-05-07 15:44:05 +0000 | [diff] [blame] | 795 | irq_set_status_flags(i, _IRQ_NOREQUEST | _IRQ_NOPROBE); |
| 796 | arch_teardown_hwirq(i); |
| 797 | } |
| 798 | irq_free_descs(from, cnt); |
| 799 | } |
| 800 | EXPORT_SYMBOL_GPL(irq_free_hwirqs); |
| 801 | #endif |
| 802 | |
Thomas Gleixner | a98d24b71 | 2010-09-30 10:45:07 +0200 | [diff] [blame] | 803 | /** |
| 804 | * irq_get_next_irq - get next allocated irq number |
| 805 | * @offset: where to start the search |
| 806 | * |
| 807 | * Returns next irq number after offset or nr_irqs if none is found. |
| 808 | */ |
| 809 | unsigned int irq_get_next_irq(unsigned int offset) |
| 810 | { |
| 811 | return find_next_bit(allocated_irqs, nr_irqs, offset); |
| 812 | } |
| 813 | |
Thomas Gleixner | d5eb4ad2 | 2011-02-12 12:16:16 +0100 | [diff] [blame] | 814 | struct irq_desc * |
Marc Zyngier | 31d9d9b | 2011-09-23 17:03:06 +0100 | [diff] [blame] | 815 | __irq_get_desc_lock(unsigned int irq, unsigned long *flags, bool bus, |
| 816 | unsigned int check) |
Thomas Gleixner | d5eb4ad2 | 2011-02-12 12:16:16 +0100 | [diff] [blame] | 817 | { |
| 818 | struct irq_desc *desc = irq_to_desc(irq); |
| 819 | |
| 820 | if (desc) { |
Marc Zyngier | 31d9d9b | 2011-09-23 17:03:06 +0100 | [diff] [blame] | 821 | if (check & _IRQ_DESC_CHECK) { |
| 822 | if ((check & _IRQ_DESC_PERCPU) && |
| 823 | !irq_settings_is_per_cpu_devid(desc)) |
| 824 | return NULL; |
| 825 | |
| 826 | if (!(check & _IRQ_DESC_PERCPU) && |
| 827 | irq_settings_is_per_cpu_devid(desc)) |
| 828 | return NULL; |
| 829 | } |
| 830 | |
Thomas Gleixner | d5eb4ad2 | 2011-02-12 12:16:16 +0100 | [diff] [blame] | 831 | if (bus) |
| 832 | chip_bus_lock(desc); |
| 833 | raw_spin_lock_irqsave(&desc->lock, *flags); |
| 834 | } |
| 835 | return desc; |
| 836 | } |
| 837 | |
| 838 | void __irq_put_desc_unlock(struct irq_desc *desc, unsigned long flags, bool bus) |
| 839 | { |
| 840 | raw_spin_unlock_irqrestore(&desc->lock, flags); |
| 841 | if (bus) |
| 842 | chip_bus_sync_unlock(desc); |
| 843 | } |
| 844 | |
Marc Zyngier | 222df54 | 2016-04-11 09:57:52 +0100 | [diff] [blame] | 845 | int irq_set_percpu_devid_partition(unsigned int irq, |
| 846 | const struct cpumask *affinity) |
Marc Zyngier | 31d9d9b | 2011-09-23 17:03:06 +0100 | [diff] [blame] | 847 | { |
| 848 | struct irq_desc *desc = irq_to_desc(irq); |
| 849 | |
| 850 | if (!desc) |
| 851 | return -EINVAL; |
| 852 | |
| 853 | if (desc->percpu_enabled) |
| 854 | return -EINVAL; |
| 855 | |
| 856 | desc->percpu_enabled = kzalloc(sizeof(*desc->percpu_enabled), GFP_KERNEL); |
| 857 | |
| 858 | if (!desc->percpu_enabled) |
| 859 | return -ENOMEM; |
| 860 | |
Marc Zyngier | 222df54 | 2016-04-11 09:57:52 +0100 | [diff] [blame] | 861 | if (affinity) |
| 862 | desc->percpu_affinity = affinity; |
| 863 | else |
| 864 | desc->percpu_affinity = cpu_possible_mask; |
| 865 | |
Marc Zyngier | 31d9d9b | 2011-09-23 17:03:06 +0100 | [diff] [blame] | 866 | irq_set_percpu_devid_flags(irq); |
| 867 | return 0; |
| 868 | } |
| 869 | |
Marc Zyngier | 222df54 | 2016-04-11 09:57:52 +0100 | [diff] [blame] | 870 | int irq_set_percpu_devid(unsigned int irq) |
| 871 | { |
| 872 | return irq_set_percpu_devid_partition(irq, NULL); |
| 873 | } |
| 874 | |
| 875 | int irq_get_percpu_devid_partition(unsigned int irq, struct cpumask *affinity) |
| 876 | { |
| 877 | struct irq_desc *desc = irq_to_desc(irq); |
| 878 | |
| 879 | if (!desc || !desc->percpu_enabled) |
| 880 | return -EINVAL; |
| 881 | |
| 882 | if (affinity) |
| 883 | cpumask_copy(affinity, desc->percpu_affinity); |
| 884 | |
| 885 | return 0; |
| 886 | } |
Will Deacon | 5ffeb05 | 2016-07-25 16:07:10 +0100 | [diff] [blame] | 887 | EXPORT_SYMBOL_GPL(irq_get_percpu_devid_partition); |
Marc Zyngier | 222df54 | 2016-04-11 09:57:52 +0100 | [diff] [blame] | 888 | |
Thomas Gleixner | 792d001 | 2014-02-23 21:40:14 +0000 | [diff] [blame] | 889 | void kstat_incr_irq_this_cpu(unsigned int irq) |
| 890 | { |
Jiang Liu | b51bf95 | 2015-06-04 12:13:25 +0800 | [diff] [blame] | 891 | kstat_incr_irqs_this_cpu(irq_to_desc(irq)); |
Thomas Gleixner | 792d001 | 2014-02-23 21:40:14 +0000 | [diff] [blame] | 892 | } |
| 893 | |
Thomas Gleixner | c291ee6 | 2014-12-11 23:01:41 +0100 | [diff] [blame] | 894 | /** |
| 895 | * kstat_irqs_cpu - Get the statistics for an interrupt on a cpu |
| 896 | * @irq: The interrupt number |
| 897 | * @cpu: The cpu number |
| 898 | * |
| 899 | * Returns the sum of interrupt counts on @cpu since boot for |
| 900 | * @irq. The caller must ensure that the interrupt is not removed |
| 901 | * concurrently. |
| 902 | */ |
Thomas Gleixner | 3795de2 | 2010-09-22 17:09:43 +0200 | [diff] [blame] | 903 | unsigned int kstat_irqs_cpu(unsigned int irq, int cpu) |
| 904 | { |
| 905 | struct irq_desc *desc = irq_to_desc(irq); |
Eric Dumazet | 6c9ae00 | 2011-01-13 15:45:38 -0800 | [diff] [blame] | 906 | |
| 907 | return desc && desc->kstat_irqs ? |
| 908 | *per_cpu_ptr(desc->kstat_irqs, cpu) : 0; |
Thomas Gleixner | 3795de2 | 2010-09-22 17:09:43 +0200 | [diff] [blame] | 909 | } |
KAMEZAWA Hiroyuki | 478735e3 | 2010-10-27 15:34:15 -0700 | [diff] [blame] | 910 | |
Thomas Gleixner | c291ee6 | 2014-12-11 23:01:41 +0100 | [diff] [blame] | 911 | /** |
| 912 | * kstat_irqs - Get the statistics for an interrupt |
| 913 | * @irq: The interrupt number |
| 914 | * |
| 915 | * Returns the sum of interrupt counts on all cpus since boot for |
| 916 | * @irq. The caller must ensure that the interrupt is not removed |
| 917 | * concurrently. |
| 918 | */ |
KAMEZAWA Hiroyuki | 478735e3 | 2010-10-27 15:34:15 -0700 | [diff] [blame] | 919 | unsigned int kstat_irqs(unsigned int irq) |
| 920 | { |
| 921 | struct irq_desc *desc = irq_to_desc(irq); |
| 922 | int cpu; |
Nicholas Mc Guire | 5e9662f | 2015-05-03 10:48:50 +0200 | [diff] [blame] | 923 | unsigned int sum = 0; |
KAMEZAWA Hiroyuki | 478735e3 | 2010-10-27 15:34:15 -0700 | [diff] [blame] | 924 | |
Eric Dumazet | 6c9ae00 | 2011-01-13 15:45:38 -0800 | [diff] [blame] | 925 | if (!desc || !desc->kstat_irqs) |
KAMEZAWA Hiroyuki | 478735e3 | 2010-10-27 15:34:15 -0700 | [diff] [blame] | 926 | return 0; |
| 927 | for_each_possible_cpu(cpu) |
Eric Dumazet | 6c9ae00 | 2011-01-13 15:45:38 -0800 | [diff] [blame] | 928 | sum += *per_cpu_ptr(desc->kstat_irqs, cpu); |
KAMEZAWA Hiroyuki | 478735e3 | 2010-10-27 15:34:15 -0700 | [diff] [blame] | 929 | return sum; |
| 930 | } |
Thomas Gleixner | c291ee6 | 2014-12-11 23:01:41 +0100 | [diff] [blame] | 931 | |
| 932 | /** |
| 933 | * kstat_irqs_usr - Get the statistics for an interrupt |
| 934 | * @irq: The interrupt number |
| 935 | * |
Eric Dumazet | 4a5f4d2 | 2018-06-18 05:56:12 -0700 | [diff] [blame] | 936 | * Returns the sum of interrupt counts on all cpus since boot for @irq. |
| 937 | * Contrary to kstat_irqs() this can be called from any context. |
| 938 | * It uses rcu since a concurrent removal of an interrupt descriptor is |
| 939 | * observing an rcu grace period before delayed_free_desc()/irq_kobj_release(). |
Thomas Gleixner | c291ee6 | 2014-12-11 23:01:41 +0100 | [diff] [blame] | 940 | */ |
| 941 | unsigned int kstat_irqs_usr(unsigned int irq) |
| 942 | { |
Nicholas Mc Guire | 7df0b27 | 2015-05-03 10:49:11 +0200 | [diff] [blame] | 943 | unsigned int sum; |
Thomas Gleixner | c291ee6 | 2014-12-11 23:01:41 +0100 | [diff] [blame] | 944 | |
Eric Dumazet | 4a5f4d2 | 2018-06-18 05:56:12 -0700 | [diff] [blame] | 945 | rcu_read_lock(); |
Thomas Gleixner | c291ee6 | 2014-12-11 23:01:41 +0100 | [diff] [blame] | 946 | sum = kstat_irqs(irq); |
Eric Dumazet | 4a5f4d2 | 2018-06-18 05:56:12 -0700 | [diff] [blame] | 947 | rcu_read_unlock(); |
Thomas Gleixner | c291ee6 | 2014-12-11 23:01:41 +0100 | [diff] [blame] | 948 | return sum; |
| 949 | } |