blob: cb466ed7eb5ef1d718dfc256829cd603871d7409 [file] [log] [blame]
Greg Kroah-Hartmanb2441312017-11-01 15:07:57 +01001// SPDX-License-Identifier: GPL-2.0
Linus Torvalds1da177e2005-04-16 15:20:36 -07002/*
Peter Oberparleiter4e8e56c2008-01-26 14:10:44 +01003 * Support for adapter interruptions
Linus Torvalds1da177e2005-04-16 15:20:36 -07004 *
Heiko Carstensa53c8fa2012-07-20 11:15:04 +02005 * Copyright IBM Corp. 1999, 2007
Peter Oberparleiter4e8e56c2008-01-26 14:10:44 +01006 * Author(s): Ingo Adlung <adlung@de.ibm.com>
7 * Cornelia Huck <cornelia.huck@de.ibm.com>
8 * Arnd Bergmann <arndb@de.ibm.com>
9 * Peter Oberparleiter <peter.oberparleiter@de.ibm.com>
Linus Torvalds1da177e2005-04-16 15:20:36 -070010 */
11
12#include <linux/init.h>
Martin Schwidefskyf4eae942013-06-24 10:30:41 +020013#include <linux/irq.h>
14#include <linux/kernel_stat.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070015#include <linux/module.h>
Martin Schwidefskyf4eae942013-06-24 10:30:41 +020016#include <linux/mutex.h>
17#include <linux/rculist.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070018#include <linux/slab.h>
Halil Pasicb50623e2018-09-13 18:57:16 +020019#include <linux/dmapool.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070020
Peter Oberparleiter4e8e56c2008-01-26 14:10:44 +010021#include <asm/airq.h>
Cornelia Huckda7c5af2008-07-14 09:58:59 +020022#include <asm/isc.h>
Halil Pasicb50623e2018-09-13 18:57:16 +020023#include <asm/cio.h>
Peter Oberparleiter4e8e56c2008-01-26 14:10:44 +010024
25#include "cio.h"
Linus Torvalds1da177e2005-04-16 15:20:36 -070026#include "cio_debug.h"
Martin Schwidefskyf4eae942013-06-24 10:30:41 +020027#include "ioasm.h"
Linus Torvalds1da177e2005-04-16 15:20:36 -070028
Martin Schwidefskyf4eae942013-06-24 10:30:41 +020029static DEFINE_SPINLOCK(airq_lists_lock);
30static struct hlist_head airq_lists[MAX_ISC+1];
Peter Oberparleiter4e8e56c2008-01-26 14:10:44 +010031
Halil Pasicb50623e2018-09-13 18:57:16 +020032static struct dma_pool *airq_iv_cache;
Sebastian Ott414cbd12019-02-27 13:56:08 +010033
Peter Oberparleiter4e8e56c2008-01-26 14:10:44 +010034/**
Martin Schwidefskyf4eae942013-06-24 10:30:41 +020035 * register_adapter_interrupt() - register adapter interrupt handler
36 * @airq: pointer to adapter interrupt descriptor
Linus Torvalds1da177e2005-04-16 15:20:36 -070037 *
Martin Schwidefskyf4eae942013-06-24 10:30:41 +020038 * Returns 0 on success, or -EINVAL.
Linus Torvalds1da177e2005-04-16 15:20:36 -070039 */
Martin Schwidefskyf4eae942013-06-24 10:30:41 +020040int register_adapter_interrupt(struct airq_struct *airq)
Linus Torvalds1da177e2005-04-16 15:20:36 -070041{
Martin Schwidefskyf4eae942013-06-24 10:30:41 +020042 char dbf_txt[32];
Linus Torvalds1da177e2005-04-16 15:20:36 -070043
Martin Schwidefskyf4eae942013-06-24 10:30:41 +020044 if (!airq->handler || airq->isc > MAX_ISC)
45 return -EINVAL;
46 if (!airq->lsi_ptr) {
47 airq->lsi_ptr = kzalloc(1, GFP_KERNEL);
48 if (!airq->lsi_ptr)
49 return -ENOMEM;
50 airq->flags |= AIRQ_PTR_ALLOCATED;
Linus Torvalds1da177e2005-04-16 15:20:36 -070051 }
Martin Schwidefskyf4eae942013-06-24 10:30:41 +020052 if (!airq->lsi_mask)
53 airq->lsi_mask = 0xff;
54 snprintf(dbf_txt, sizeof(dbf_txt), "rairq:%p", airq);
Peter Oberparleiter4e8e56c2008-01-26 14:10:44 +010055 CIO_TRACE_EVENT(4, dbf_txt);
Martin Schwidefskyf4eae942013-06-24 10:30:41 +020056 isc_register(airq->isc);
57 spin_lock(&airq_lists_lock);
58 hlist_add_head_rcu(&airq->list, &airq_lists[airq->isc]);
59 spin_unlock(&airq_lists_lock);
60 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -070061}
Martin Schwidefskyf4eae942013-06-24 10:30:41 +020062EXPORT_SYMBOL(register_adapter_interrupt);
Linus Torvalds1da177e2005-04-16 15:20:36 -070063
Peter Oberparleiter4e8e56c2008-01-26 14:10:44 +010064/**
Martin Schwidefskyf4eae942013-06-24 10:30:41 +020065 * unregister_adapter_interrupt - unregister adapter interrupt handler
66 * @airq: pointer to adapter interrupt descriptor
Peter Oberparleiter4e8e56c2008-01-26 14:10:44 +010067 */
Martin Schwidefskyf4eae942013-06-24 10:30:41 +020068void unregister_adapter_interrupt(struct airq_struct *airq)
Linus Torvalds1da177e2005-04-16 15:20:36 -070069{
Martin Schwidefskyf4eae942013-06-24 10:30:41 +020070 char dbf_txt[32];
Linus Torvalds1da177e2005-04-16 15:20:36 -070071
Martin Schwidefskyf4eae942013-06-24 10:30:41 +020072 if (hlist_unhashed(&airq->list))
73 return;
74 snprintf(dbf_txt, sizeof(dbf_txt), "urairq:%p", airq);
Peter Oberparleiter4e8e56c2008-01-26 14:10:44 +010075 CIO_TRACE_EVENT(4, dbf_txt);
Martin Schwidefskyf4eae942013-06-24 10:30:41 +020076 spin_lock(&airq_lists_lock);
77 hlist_del_rcu(&airq->list);
78 spin_unlock(&airq_lists_lock);
79 synchronize_rcu();
80 isc_unregister(airq->isc);
81 if (airq->flags & AIRQ_PTR_ALLOCATED) {
82 kfree(airq->lsi_ptr);
83 airq->lsi_ptr = NULL;
84 airq->flags &= ~AIRQ_PTR_ALLOCATED;
85 }
Linus Torvalds1da177e2005-04-16 15:20:36 -070086}
Martin Schwidefskyf4eae942013-06-24 10:30:41 +020087EXPORT_SYMBOL(unregister_adapter_interrupt);
Peter Oberparleiter4e8e56c2008-01-26 14:10:44 +010088
Martin Schwidefsky1f44a2252013-06-27 09:01:09 +020089static irqreturn_t do_airq_interrupt(int irq, void *dummy)
Peter Oberparleiter4e8e56c2008-01-26 14:10:44 +010090{
Martin Schwidefsky1f44a2252013-06-27 09:01:09 +020091 struct tpi_info *tpi_info;
Martin Schwidefskyf4eae942013-06-24 10:30:41 +020092 struct airq_struct *airq;
93 struct hlist_head *head;
Peter Oberparleiter4e8e56c2008-01-26 14:10:44 +010094
Martin Schwidefskyfe0f4972014-09-30 17:37:52 +020095 set_cpu_flag(CIF_NOHZ_DELAY);
Martin Schwidefsky1f44a2252013-06-27 09:01:09 +020096 tpi_info = (struct tpi_info *) &get_irq_regs()->int_code;
Peter Oberparleiter42248972015-12-18 12:59:36 +010097 trace_s390_cio_adapter_int(tpi_info);
Martin Schwidefsky1f44a2252013-06-27 09:01:09 +020098 head = &airq_lists[tpi_info->isc];
Martin Schwidefskyf4eae942013-06-24 10:30:41 +020099 rcu_read_lock();
100 hlist_for_each_entry_rcu(airq, head, list)
101 if ((*airq->lsi_ptr & airq->lsi_mask) != 0)
Sebastian Ott30e63ef2018-10-28 11:51:56 +0100102 airq->handler(airq, !tpi_info->directed_irq);
Martin Schwidefskyf4eae942013-06-24 10:30:41 +0200103 rcu_read_unlock();
Martin Schwidefsky1f44a2252013-06-27 09:01:09 +0200104
105 return IRQ_HANDLED;
106}
107
Martin Schwidefsky1f44a2252013-06-27 09:01:09 +0200108void __init init_airq_interrupts(void)
109{
110 irq_set_chip_and_handler(THIN_INTERRUPT,
111 &dummy_irq_chip, handle_percpu_irq);
afzal mohammed8719b6d2020-03-04 06:20:48 +0530112 if (request_irq(THIN_INTERRUPT, do_airq_interrupt, 0, "AIO", NULL))
113 panic("Failed to register AIO interrupt\n");
Peter Oberparleiter4e8e56c2008-01-26 14:10:44 +0100114}
Martin Schwidefskya9a6f032013-06-25 14:17:57 +0200115
Halil Pasicb50623e2018-09-13 18:57:16 +0200116static inline unsigned long iv_size(unsigned long bits)
117{
118 return BITS_TO_LONGS(bits) * sizeof(unsigned long);
119}
120
Martin Schwidefskya9a6f032013-06-25 14:17:57 +0200121/**
122 * airq_iv_create - create an interrupt vector
123 * @bits: number of bits in the interrupt vector
124 * @flags: allocation flags
125 *
126 * Returns a pointer to an interrupt vector structure
127 */
128struct airq_iv *airq_iv_create(unsigned long bits, unsigned long flags)
129{
130 struct airq_iv *iv;
131 unsigned long size;
132
133 iv = kzalloc(sizeof(*iv), GFP_KERNEL);
134 if (!iv)
135 goto out;
136 iv->bits = bits;
Sebastian Ott414cbd12019-02-27 13:56:08 +0100137 iv->flags = flags;
Halil Pasicb50623e2018-09-13 18:57:16 +0200138 size = iv_size(bits);
Sebastian Ott414cbd12019-02-27 13:56:08 +0100139
140 if (flags & AIRQ_IV_CACHELINE) {
Halil Pasicb50623e2018-09-13 18:57:16 +0200141 if ((cache_line_size() * BITS_PER_BYTE) < bits
142 || !airq_iv_cache)
Sebastian Ott414cbd12019-02-27 13:56:08 +0100143 goto out_free;
144
Halil Pasicb50623e2018-09-13 18:57:16 +0200145 iv->vector = dma_pool_zalloc(airq_iv_cache, GFP_KERNEL,
146 &iv->vector_dma);
Sebastian Ott414cbd12019-02-27 13:56:08 +0100147 if (!iv->vector)
148 goto out_free;
149 } else {
Halil Pasicb50623e2018-09-13 18:57:16 +0200150 iv->vector = cio_dma_zalloc(size);
Sebastian Ott414cbd12019-02-27 13:56:08 +0100151 if (!iv->vector)
152 goto out_free;
153 }
Martin Schwidefskya9a6f032013-06-25 14:17:57 +0200154 if (flags & AIRQ_IV_ALLOC) {
155 iv->avail = kmalloc(size, GFP_KERNEL);
156 if (!iv->avail)
157 goto out_free;
158 memset(iv->avail, 0xff, size);
159 iv->end = 0;
160 } else
161 iv->end = bits;
162 if (flags & AIRQ_IV_BITLOCK) {
163 iv->bitlock = kzalloc(size, GFP_KERNEL);
164 if (!iv->bitlock)
165 goto out_free;
166 }
167 if (flags & AIRQ_IV_PTR) {
168 size = bits * sizeof(unsigned long);
169 iv->ptr = kzalloc(size, GFP_KERNEL);
170 if (!iv->ptr)
171 goto out_free;
172 }
173 if (flags & AIRQ_IV_DATA) {
174 size = bits * sizeof(unsigned int);
175 iv->data = kzalloc(size, GFP_KERNEL);
176 if (!iv->data)
177 goto out_free;
178 }
179 spin_lock_init(&iv->lock);
180 return iv;
181
182out_free:
183 kfree(iv->ptr);
184 kfree(iv->bitlock);
185 kfree(iv->avail);
Halil Pasicb50623e2018-09-13 18:57:16 +0200186 if (iv->flags & AIRQ_IV_CACHELINE && iv->vector)
187 dma_pool_free(airq_iv_cache, iv->vector, iv->vector_dma);
Sebastian Ott414cbd12019-02-27 13:56:08 +0100188 else
Halil Pasicb50623e2018-09-13 18:57:16 +0200189 cio_dma_free(iv->vector, size);
Martin Schwidefskya9a6f032013-06-25 14:17:57 +0200190 kfree(iv);
191out:
192 return NULL;
193}
194EXPORT_SYMBOL(airq_iv_create);
195
196/**
197 * airq_iv_release - release an interrupt vector
198 * @iv: pointer to interrupt vector structure
199 */
200void airq_iv_release(struct airq_iv *iv)
201{
202 kfree(iv->data);
203 kfree(iv->ptr);
204 kfree(iv->bitlock);
Sebastian Ott414cbd12019-02-27 13:56:08 +0100205 if (iv->flags & AIRQ_IV_CACHELINE)
Halil Pasicb50623e2018-09-13 18:57:16 +0200206 dma_pool_free(airq_iv_cache, iv->vector, iv->vector_dma);
Sebastian Ott414cbd12019-02-27 13:56:08 +0100207 else
Halil Pasicb50623e2018-09-13 18:57:16 +0200208 cio_dma_free(iv->vector, iv_size(iv->bits));
Martin Schwidefskya9a6f032013-06-25 14:17:57 +0200209 kfree(iv->avail);
210 kfree(iv);
211}
212EXPORT_SYMBOL(airq_iv_release);
213
214/**
Martin Schwidefskyfe7c30a2014-02-13 13:02:32 +0100215 * airq_iv_alloc - allocate irq bits from an interrupt vector
Martin Schwidefskya9a6f032013-06-25 14:17:57 +0200216 * @iv: pointer to an interrupt vector structure
Martin Schwidefskyfe7c30a2014-02-13 13:02:32 +0100217 * @num: number of consecutive irq bits to allocate
Martin Schwidefskya9a6f032013-06-25 14:17:57 +0200218 *
Martin Schwidefskyfe7c30a2014-02-13 13:02:32 +0100219 * Returns the bit number of the first irq in the allocated block of irqs,
220 * or -1UL if no bit is available or the AIRQ_IV_ALLOC flag has not been
221 * specified
Martin Schwidefskya9a6f032013-06-25 14:17:57 +0200222 */
Martin Schwidefskyfe7c30a2014-02-13 13:02:32 +0100223unsigned long airq_iv_alloc(struct airq_iv *iv, unsigned long num)
Martin Schwidefskya9a6f032013-06-25 14:17:57 +0200224{
Sebastian Ott0eb69a02014-06-05 14:30:19 +0200225 unsigned long bit, i, flags;
Martin Schwidefskya9a6f032013-06-25 14:17:57 +0200226
Martin Schwidefskyfe7c30a2014-02-13 13:02:32 +0100227 if (!iv->avail || num == 0)
Martin Schwidefskya9a6f032013-06-25 14:17:57 +0200228 return -1UL;
Sebastian Ott0eb69a02014-06-05 14:30:19 +0200229 spin_lock_irqsave(&iv->lock, flags);
Heiko Carstens7d7c7b22013-09-23 12:01:44 +0200230 bit = find_first_bit_inv(iv->avail, iv->bits);
Martin Schwidefskyfe7c30a2014-02-13 13:02:32 +0100231 while (bit + num <= iv->bits) {
232 for (i = 1; i < num; i++)
233 if (!test_bit_inv(bit + i, iv->avail))
234 break;
235 if (i >= num) {
236 /* Found a suitable block of irqs */
237 for (i = 0; i < num; i++)
238 clear_bit_inv(bit + i, iv->avail);
239 if (bit + num >= iv->end)
240 iv->end = bit + num + 1;
241 break;
242 }
243 bit = find_next_bit_inv(iv->avail, iv->bits, bit + i + 1);
244 }
245 if (bit + num > iv->bits)
Martin Schwidefskya9a6f032013-06-25 14:17:57 +0200246 bit = -1UL;
Sebastian Ott0eb69a02014-06-05 14:30:19 +0200247 spin_unlock_irqrestore(&iv->lock, flags);
Martin Schwidefskya9a6f032013-06-25 14:17:57 +0200248 return bit;
Martin Schwidefskya9a6f032013-06-25 14:17:57 +0200249}
Martin Schwidefskyfe7c30a2014-02-13 13:02:32 +0100250EXPORT_SYMBOL(airq_iv_alloc);
Martin Schwidefskya9a6f032013-06-25 14:17:57 +0200251
252/**
Martin Schwidefskyfe7c30a2014-02-13 13:02:32 +0100253 * airq_iv_free - free irq bits of an interrupt vector
Martin Schwidefskya9a6f032013-06-25 14:17:57 +0200254 * @iv: pointer to interrupt vector structure
Martin Schwidefskyfe7c30a2014-02-13 13:02:32 +0100255 * @bit: number of the first irq bit to free
256 * @num: number of consecutive irq bits to free
Martin Schwidefskya9a6f032013-06-25 14:17:57 +0200257 */
Martin Schwidefskyfe7c30a2014-02-13 13:02:32 +0100258void airq_iv_free(struct airq_iv *iv, unsigned long bit, unsigned long num)
Martin Schwidefskya9a6f032013-06-25 14:17:57 +0200259{
Sebastian Ott0eb69a02014-06-05 14:30:19 +0200260 unsigned long i, flags;
Martin Schwidefskyfe7c30a2014-02-13 13:02:32 +0100261
262 if (!iv->avail || num == 0)
Martin Schwidefskya9a6f032013-06-25 14:17:57 +0200263 return;
Sebastian Ott0eb69a02014-06-05 14:30:19 +0200264 spin_lock_irqsave(&iv->lock, flags);
Martin Schwidefskyfe7c30a2014-02-13 13:02:32 +0100265 for (i = 0; i < num; i++) {
266 /* Clear (possibly left over) interrupt bit */
267 clear_bit_inv(bit + i, iv->vector);
268 /* Make the bit positions available again */
269 set_bit_inv(bit + i, iv->avail);
270 }
271 if (bit + num >= iv->end) {
Martin Schwidefskya9a6f032013-06-25 14:17:57 +0200272 /* Find new end of bit-field */
Martin Schwidefskyfe7c30a2014-02-13 13:02:32 +0100273 while (iv->end > 0 && !test_bit_inv(iv->end - 1, iv->avail))
274 iv->end--;
Martin Schwidefskya9a6f032013-06-25 14:17:57 +0200275 }
Sebastian Ott0eb69a02014-06-05 14:30:19 +0200276 spin_unlock_irqrestore(&iv->lock, flags);
Martin Schwidefskya9a6f032013-06-25 14:17:57 +0200277}
Martin Schwidefskyfe7c30a2014-02-13 13:02:32 +0100278EXPORT_SYMBOL(airq_iv_free);
Martin Schwidefskya9a6f032013-06-25 14:17:57 +0200279
280/**
281 * airq_iv_scan - scan interrupt vector for non-zero bits
282 * @iv: pointer to interrupt vector structure
283 * @start: bit number to start the search
284 * @end: bit number to end the search
285 *
286 * Returns the bit number of the next non-zero interrupt bit, or
287 * -1UL if the scan completed without finding any more any non-zero bits.
288 */
289unsigned long airq_iv_scan(struct airq_iv *iv, unsigned long start,
290 unsigned long end)
291{
Martin Schwidefskya9a6f032013-06-25 14:17:57 +0200292 unsigned long bit;
293
294 /* Find non-zero bit starting from 'ivs->next'. */
Heiko Carstens7d7c7b22013-09-23 12:01:44 +0200295 bit = find_next_bit_inv(iv->vector, end, start);
Martin Schwidefskya9a6f032013-06-25 14:17:57 +0200296 if (bit >= end)
297 return -1UL;
Heiko Carstens7d7c7b22013-09-23 12:01:44 +0200298 clear_bit_inv(bit, iv->vector);
Martin Schwidefskya9a6f032013-06-25 14:17:57 +0200299 return bit;
300}
301EXPORT_SYMBOL(airq_iv_scan);
Sebastian Ott414cbd12019-02-27 13:56:08 +0100302
Halil Pasicb50623e2018-09-13 18:57:16 +0200303int __init airq_init(void)
Sebastian Ott414cbd12019-02-27 13:56:08 +0100304{
Halil Pasicb50623e2018-09-13 18:57:16 +0200305 airq_iv_cache = dma_pool_create("airq_iv_cache", cio_get_dma_css_dev(),
306 cache_line_size(),
307 cache_line_size(), PAGE_SIZE);
Sebastian Ott414cbd12019-02-27 13:56:08 +0100308 if (!airq_iv_cache)
309 return -ENOMEM;
310 return 0;
311}