blob: 80de699bf6af4b7b77f582c0469c43e978f67a43 [file] [log] [blame]
Thomas Gleixnerb4d0d232019-05-20 19:08:01 +02001/* SPDX-License-Identifier: GPL-2.0-or-later */
David Howells739d8752018-03-08 09:48:46 +00002/* Generic I/O port emulation.
Arnd Bergmann3f7e212d2009-05-13 22:56:35 +00003 *
4 * Copyright (C) 2007 Red Hat, Inc. All Rights Reserved.
5 * Written by David Howells (dhowells@redhat.com)
Arnd Bergmann3f7e212d2009-05-13 22:56:35 +00006 */
7#ifndef __ASM_GENERIC_IO_H
8#define __ASM_GENERIC_IO_H
9
10#include <asm/page.h> /* I/O is all done through memory accesses */
Thierry Reding9216efa2014-10-01 15:20:33 +020011#include <linux/string.h> /* for memset() and memcpy() */
Kent Overstreet00694552024-03-21 09:36:23 -070012#include <linux/sizes.h>
Arnd Bergmann3f7e212d2009-05-13 22:56:35 +000013#include <linux/types.h>
Sai Prakash Ranjan21003192022-05-18 22:14:16 +053014#include <linux/instruction_pointer.h>
Arnd Bergmann3f7e212d2009-05-13 22:56:35 +000015
16#ifdef CONFIG_GENERIC_IOMAP
17#include <asm-generic/iomap.h>
18#endif
19
Will Deacon60ca1e52019-02-22 12:59:59 +000020#include <asm/mmiowb.h>
Michael S. Tsirkin66eab4d2011-11-24 20:45:20 +020021#include <asm-generic/pci_iomap.h>
22
Sinan Kaya64e2c672018-04-05 09:09:09 -040023#ifndef __io_br
24#define __io_br() barrier()
25#endif
26
27/* prevent prefetching of coherent DMA data ahead of a dma-complete */
28#ifndef __io_ar
29#ifdef rmb
Will Deaconabbbbc832019-02-22 18:04:52 +000030#define __io_ar(v) rmb()
Sinan Kaya64e2c672018-04-05 09:09:09 -040031#else
Will Deaconabbbbc832019-02-22 18:04:52 +000032#define __io_ar(v) barrier()
Sinan Kaya64e2c672018-04-05 09:09:09 -040033#endif
34#endif
35
36/* flush writes to coherent DMA data before possibly triggering a DMA read */
37#ifndef __io_bw
38#ifdef wmb
39#define __io_bw() wmb()
40#else
41#define __io_bw() barrier()
42#endif
43#endif
44
45/* serialize device access against a spin_unlock, usually handled there. */
46#ifndef __io_aw
Will Deacon60ca1e52019-02-22 12:59:59 +000047#define __io_aw() mmiowb_set_pending()
Sinan Kaya64e2c672018-04-05 09:09:09 -040048#endif
49
50#ifndef __io_pbw
51#define __io_pbw() __io_bw()
52#endif
53
54#ifndef __io_paw
55#define __io_paw() __io_aw()
56#endif
57
58#ifndef __io_pbr
59#define __io_pbr() __io_br()
60#endif
61
62#ifndef __io_par
Will Deaconabbbbc832019-02-22 18:04:52 +000063#define __io_par(v) __io_ar(v)
Sinan Kaya64e2c672018-04-05 09:09:09 -040064#endif
65
Sai Prakash Ranjan21003192022-05-18 22:14:16 +053066/*
67 * "__DISABLE_TRACE_MMIO__" flag can be used to disable MMIO tracing for
68 * specific kernel drivers in case of excessive/unwanted logging.
69 *
70 * Usage: Add a #define flag at the beginning of the driver file.
71 * Ex: #define __DISABLE_TRACE_MMIO__
72 * #include <...>
73 * ...
74 */
75#if IS_ENABLED(CONFIG_TRACE_MMIO_ACCESS) && !(defined(__DISABLE_TRACE_MMIO__))
76#include <linux/tracepoint-defs.h>
77
78DECLARE_TRACEPOINT(rwmmio_write);
79DECLARE_TRACEPOINT(rwmmio_post_write);
80DECLARE_TRACEPOINT(rwmmio_read);
81DECLARE_TRACEPOINT(rwmmio_post_read);
82
83void log_write_mmio(u64 val, u8 width, volatile void __iomem *addr,
Sai Prakash Ranjan5e5ff732022-10-17 20:04:50 +053084 unsigned long caller_addr, unsigned long caller_addr0);
Sai Prakash Ranjan21003192022-05-18 22:14:16 +053085void log_post_write_mmio(u64 val, u8 width, volatile void __iomem *addr,
Sai Prakash Ranjan5e5ff732022-10-17 20:04:50 +053086 unsigned long caller_addr, unsigned long caller_addr0);
Sai Prakash Ranjan21003192022-05-18 22:14:16 +053087void log_read_mmio(u8 width, const volatile void __iomem *addr,
Sai Prakash Ranjan5e5ff732022-10-17 20:04:50 +053088 unsigned long caller_addr, unsigned long caller_addr0);
Sai Prakash Ranjan21003192022-05-18 22:14:16 +053089void log_post_read_mmio(u64 val, u8 width, const volatile void __iomem *addr,
Sai Prakash Ranjan5e5ff732022-10-17 20:04:50 +053090 unsigned long caller_addr, unsigned long caller_addr0);
Sai Prakash Ranjan21003192022-05-18 22:14:16 +053091
92#else
93
94static inline void log_write_mmio(u64 val, u8 width, volatile void __iomem *addr,
Sai Prakash Ranjan5e5ff732022-10-17 20:04:50 +053095 unsigned long caller_addr, unsigned long caller_addr0) {}
Sai Prakash Ranjan21003192022-05-18 22:14:16 +053096static inline void log_post_write_mmio(u64 val, u8 width, volatile void __iomem *addr,
Sai Prakash Ranjan5e5ff732022-10-17 20:04:50 +053097 unsigned long caller_addr, unsigned long caller_addr0) {}
Sai Prakash Ranjan21003192022-05-18 22:14:16 +053098static inline void log_read_mmio(u8 width, const volatile void __iomem *addr,
Sai Prakash Ranjan5e5ff732022-10-17 20:04:50 +053099 unsigned long caller_addr, unsigned long caller_addr0) {}
Sai Prakash Ranjan21003192022-05-18 22:14:16 +0530100static inline void log_post_read_mmio(u64 val, u8 width, const volatile void __iomem *addr,
Sai Prakash Ranjan5e5ff732022-10-17 20:04:50 +0530101 unsigned long caller_addr, unsigned long caller_addr0) {}
Sai Prakash Ranjan21003192022-05-18 22:14:16 +0530102
103#endif /* CONFIG_TRACE_MMIO_ACCESS */
Sinan Kaya64e2c672018-04-05 09:09:09 -0400104
Arnd Bergmann3f7e212d2009-05-13 22:56:35 +0000105/*
Thierry Reding9216efa2014-10-01 15:20:33 +0200106 * __raw_{read,write}{b,w,l,q}() access memory in native endianness.
107 *
108 * On some architectures memory mapped IO needs to be accessed differently.
109 * On the simple architectures, we just read/write the memory location
110 * directly.
Arnd Bergmann3f7e212d2009-05-13 22:56:35 +0000111 */
Thierry Reding9216efa2014-10-01 15:20:33 +0200112
Mike Frysinger35dbc0e2010-10-18 03:09:39 -0400113#ifndef __raw_readb
Thierry Reding9216efa2014-10-01 15:20:33 +0200114#define __raw_readb __raw_readb
Arnd Bergmann3f7e212d2009-05-13 22:56:35 +0000115static inline u8 __raw_readb(const volatile void __iomem *addr)
116{
Thierry Reding9216efa2014-10-01 15:20:33 +0200117 return *(const volatile u8 __force *)addr;
Arnd Bergmann3f7e212d2009-05-13 22:56:35 +0000118}
Mike Frysinger35dbc0e2010-10-18 03:09:39 -0400119#endif
Arnd Bergmann3f7e212d2009-05-13 22:56:35 +0000120
Mike Frysinger35dbc0e2010-10-18 03:09:39 -0400121#ifndef __raw_readw
Thierry Reding9216efa2014-10-01 15:20:33 +0200122#define __raw_readw __raw_readw
Arnd Bergmann3f7e212d2009-05-13 22:56:35 +0000123static inline u16 __raw_readw(const volatile void __iomem *addr)
124{
Thierry Reding9216efa2014-10-01 15:20:33 +0200125 return *(const volatile u16 __force *)addr;
Arnd Bergmann3f7e212d2009-05-13 22:56:35 +0000126}
Mike Frysinger35dbc0e2010-10-18 03:09:39 -0400127#endif
Arnd Bergmann3f7e212d2009-05-13 22:56:35 +0000128
Mike Frysinger35dbc0e2010-10-18 03:09:39 -0400129#ifndef __raw_readl
Thierry Reding9216efa2014-10-01 15:20:33 +0200130#define __raw_readl __raw_readl
Arnd Bergmann3f7e212d2009-05-13 22:56:35 +0000131static inline u32 __raw_readl(const volatile void __iomem *addr)
132{
Thierry Reding9216efa2014-10-01 15:20:33 +0200133 return *(const volatile u32 __force *)addr;
Arnd Bergmann3f7e212d2009-05-13 22:56:35 +0000134}
Mike Frysinger35dbc0e2010-10-18 03:09:39 -0400135#endif
Arnd Bergmann3f7e212d2009-05-13 22:56:35 +0000136
Thierry Reding9216efa2014-10-01 15:20:33 +0200137#ifdef CONFIG_64BIT
138#ifndef __raw_readq
139#define __raw_readq __raw_readq
140static inline u64 __raw_readq(const volatile void __iomem *addr)
141{
142 return *(const volatile u64 __force *)addr;
143}
144#endif
145#endif /* CONFIG_64BIT */
Heiko Carstens7292e7e2013-01-07 14:17:23 +0100146
Thierry Reding9216efa2014-10-01 15:20:33 +0200147#ifndef __raw_writeb
148#define __raw_writeb __raw_writeb
149static inline void __raw_writeb(u8 value, volatile void __iomem *addr)
150{
151 *(volatile u8 __force *)addr = value;
152}
153#endif
154
155#ifndef __raw_writew
156#define __raw_writew __raw_writew
157static inline void __raw_writew(u16 value, volatile void __iomem *addr)
158{
159 *(volatile u16 __force *)addr = value;
160}
161#endif
162
163#ifndef __raw_writel
164#define __raw_writel __raw_writel
165static inline void __raw_writel(u32 value, volatile void __iomem *addr)
166{
167 *(volatile u32 __force *)addr = value;
168}
169#endif
170
171#ifdef CONFIG_64BIT
172#ifndef __raw_writeq
173#define __raw_writeq __raw_writeq
174static inline void __raw_writeq(u64 value, volatile void __iomem *addr)
175{
176 *(volatile u64 __force *)addr = value;
177}
178#endif
179#endif /* CONFIG_64BIT */
180
181/*
182 * {read,write}{b,w,l,q}() access little endian memory and return result in
183 * native endianness.
184 */
185
186#ifndef readb
187#define readb readb
188static inline u8 readb(const volatile void __iomem *addr)
189{
Sinan Kaya032d59e2018-04-05 09:09:10 -0400190 u8 val;
191
Sai Prakash Ranjan5e5ff732022-10-17 20:04:50 +0530192 log_read_mmio(8, addr, _THIS_IP_, _RET_IP_);
Sinan Kaya032d59e2018-04-05 09:09:10 -0400193 __io_br();
194 val = __raw_readb(addr);
Will Deaconabbbbc832019-02-22 18:04:52 +0000195 __io_ar(val);
Sai Prakash Ranjan5e5ff732022-10-17 20:04:50 +0530196 log_post_read_mmio(val, 8, addr, _THIS_IP_, _RET_IP_);
Sinan Kaya032d59e2018-04-05 09:09:10 -0400197 return val;
Thierry Reding9216efa2014-10-01 15:20:33 +0200198}
199#endif
200
201#ifndef readw
Heiko Carstens7292e7e2013-01-07 14:17:23 +0100202#define readw readw
203static inline u16 readw(const volatile void __iomem *addr)
204{
Sinan Kaya032d59e2018-04-05 09:09:10 -0400205 u16 val;
206
Sai Prakash Ranjan5e5ff732022-10-17 20:04:50 +0530207 log_read_mmio(16, addr, _THIS_IP_, _RET_IP_);
Sinan Kaya032d59e2018-04-05 09:09:10 -0400208 __io_br();
Stafford Hornec1d55d52020-07-29 21:03:14 +0900209 val = __le16_to_cpu((__le16 __force)__raw_readw(addr));
Will Deaconabbbbc832019-02-22 18:04:52 +0000210 __io_ar(val);
Sai Prakash Ranjan5e5ff732022-10-17 20:04:50 +0530211 log_post_read_mmio(val, 16, addr, _THIS_IP_, _RET_IP_);
Sinan Kaya032d59e2018-04-05 09:09:10 -0400212 return val;
Heiko Carstens7292e7e2013-01-07 14:17:23 +0100213}
Thierry Reding9216efa2014-10-01 15:20:33 +0200214#endif
Heiko Carstens7292e7e2013-01-07 14:17:23 +0100215
Thierry Reding9216efa2014-10-01 15:20:33 +0200216#ifndef readl
Heiko Carstens7292e7e2013-01-07 14:17:23 +0100217#define readl readl
218static inline u32 readl(const volatile void __iomem *addr)
219{
Sinan Kaya032d59e2018-04-05 09:09:10 -0400220 u32 val;
221
Sai Prakash Ranjan5e5ff732022-10-17 20:04:50 +0530222 log_read_mmio(32, addr, _THIS_IP_, _RET_IP_);
Sinan Kaya032d59e2018-04-05 09:09:10 -0400223 __io_br();
Stafford Hornec1d55d52020-07-29 21:03:14 +0900224 val = __le32_to_cpu((__le32 __force)__raw_readl(addr));
Will Deaconabbbbc832019-02-22 18:04:52 +0000225 __io_ar(val);
Sai Prakash Ranjan5e5ff732022-10-17 20:04:50 +0530226 log_post_read_mmio(val, 32, addr, _THIS_IP_, _RET_IP_);
Sinan Kaya032d59e2018-04-05 09:09:10 -0400227 return val;
Heiko Carstens7292e7e2013-01-07 14:17:23 +0100228}
Mike Frysinger35dbc0e2010-10-18 03:09:39 -0400229#endif
Arnd Bergmann3f7e212d2009-05-13 22:56:35 +0000230
Arnd Bergmann3f7e212d2009-05-13 22:56:35 +0000231#ifdef CONFIG_64BIT
Thierry Reding9216efa2014-10-01 15:20:33 +0200232#ifndef readq
Heiko Carstens7292e7e2013-01-07 14:17:23 +0100233#define readq readq
234static inline u64 readq(const volatile void __iomem *addr)
235{
Sinan Kaya032d59e2018-04-05 09:09:10 -0400236 u64 val;
237
Sai Prakash Ranjan5e5ff732022-10-17 20:04:50 +0530238 log_read_mmio(64, addr, _THIS_IP_, _RET_IP_);
Sinan Kaya032d59e2018-04-05 09:09:10 -0400239 __io_br();
Vladimir Olteand564fa12023-01-09 15:11:52 +0200240 val = __le64_to_cpu((__le64 __force)__raw_readq(addr));
Will Deaconabbbbc832019-02-22 18:04:52 +0000241 __io_ar(val);
Sai Prakash Ranjan5e5ff732022-10-17 20:04:50 +0530242 log_post_read_mmio(val, 64, addr, _THIS_IP_, _RET_IP_);
Sinan Kaya032d59e2018-04-05 09:09:10 -0400243 return val;
Heiko Carstens7292e7e2013-01-07 14:17:23 +0100244}
Arnd Bergmann3f7e212d2009-05-13 22:56:35 +0000245#endif
Jan Glaubercd248342012-11-29 12:50:30 +0100246#endif /* CONFIG_64BIT */
247
Thierry Reding9216efa2014-10-01 15:20:33 +0200248#ifndef writeb
249#define writeb writeb
250static inline void writeb(u8 value, volatile void __iomem *addr)
251{
Sai Prakash Ranjan5e5ff732022-10-17 20:04:50 +0530252 log_write_mmio(value, 8, addr, _THIS_IP_, _RET_IP_);
Sinan Kaya755bd042018-04-05 09:09:11 -0400253 __io_bw();
Thierry Reding9216efa2014-10-01 15:20:33 +0200254 __raw_writeb(value, addr);
Sinan Kaya755bd042018-04-05 09:09:11 -0400255 __io_aw();
Sai Prakash Ranjan5e5ff732022-10-17 20:04:50 +0530256 log_post_write_mmio(value, 8, addr, _THIS_IP_, _RET_IP_);
Thierry Reding9216efa2014-10-01 15:20:33 +0200257}
GuanXuetao7dc59bd2011-02-22 19:06:43 +0800258#endif
259
Thierry Reding9216efa2014-10-01 15:20:33 +0200260#ifndef writew
261#define writew writew
262static inline void writew(u16 value, volatile void __iomem *addr)
Arnd Bergmann3f7e212d2009-05-13 22:56:35 +0000263{
Sai Prakash Ranjan5e5ff732022-10-17 20:04:50 +0530264 log_write_mmio(value, 16, addr, _THIS_IP_, _RET_IP_);
Sinan Kaya755bd042018-04-05 09:09:11 -0400265 __io_bw();
Stafford Hornec1d55d52020-07-29 21:03:14 +0900266 __raw_writew((u16 __force)cpu_to_le16(value), addr);
Sinan Kaya755bd042018-04-05 09:09:11 -0400267 __io_aw();
Sai Prakash Ranjan5e5ff732022-10-17 20:04:50 +0530268 log_post_write_mmio(value, 16, addr, _THIS_IP_, _RET_IP_);
Arnd Bergmann3f7e212d2009-05-13 22:56:35 +0000269}
Thierry Reding9216efa2014-10-01 15:20:33 +0200270#endif
Arnd Bergmann3f7e212d2009-05-13 22:56:35 +0000271
Thierry Reding9216efa2014-10-01 15:20:33 +0200272#ifndef writel
273#define writel writel
274static inline void writel(u32 value, volatile void __iomem *addr)
Arnd Bergmann3f7e212d2009-05-13 22:56:35 +0000275{
Sai Prakash Ranjan5e5ff732022-10-17 20:04:50 +0530276 log_write_mmio(value, 32, addr, _THIS_IP_, _RET_IP_);
Sinan Kaya755bd042018-04-05 09:09:11 -0400277 __io_bw();
Stafford Hornec1d55d52020-07-29 21:03:14 +0900278 __raw_writel((u32 __force)__cpu_to_le32(value), addr);
Sinan Kaya755bd042018-04-05 09:09:11 -0400279 __io_aw();
Sai Prakash Ranjan5e5ff732022-10-17 20:04:50 +0530280 log_post_write_mmio(value, 32, addr, _THIS_IP_, _RET_IP_);
Arnd Bergmann3f7e212d2009-05-13 22:56:35 +0000281}
Thierry Reding9216efa2014-10-01 15:20:33 +0200282#endif
Arnd Bergmann3f7e212d2009-05-13 22:56:35 +0000283
Thierry Reding9216efa2014-10-01 15:20:33 +0200284#ifdef CONFIG_64BIT
285#ifndef writeq
286#define writeq writeq
287static inline void writeq(u64 value, volatile void __iomem *addr)
Arnd Bergmann3f7e212d2009-05-13 22:56:35 +0000288{
Sai Prakash Ranjan5e5ff732022-10-17 20:04:50 +0530289 log_write_mmio(value, 64, addr, _THIS_IP_, _RET_IP_);
Sinan Kaya755bd042018-04-05 09:09:11 -0400290 __io_bw();
Vladimir Olteand564fa12023-01-09 15:11:52 +0200291 __raw_writeq((u64 __force)__cpu_to_le64(value), addr);
Sinan Kaya755bd042018-04-05 09:09:11 -0400292 __io_aw();
Sai Prakash Ranjan5e5ff732022-10-17 20:04:50 +0530293 log_post_write_mmio(value, 64, addr, _THIS_IP_, _RET_IP_);
Arnd Bergmann3f7e212d2009-05-13 22:56:35 +0000294}
Thierry Reding9216efa2014-10-01 15:20:33 +0200295#endif
296#endif /* CONFIG_64BIT */
Arnd Bergmann3f7e212d2009-05-13 22:56:35 +0000297
Thierry Reding9ab3a7a2014-07-04 13:07:57 +0200298/*
Arnd Bergmann1c8d2962014-11-11 19:55:45 +0100299 * {read,write}{b,w,l,q}_relaxed() are like the regular version, but
300 * are not guaranteed to provide ordering against spinlocks or memory
301 * accesses.
302 */
303#ifndef readb_relaxed
Sinan Kaya8875c552018-04-06 14:02:45 -0400304#define readb_relaxed readb_relaxed
305static inline u8 readb_relaxed(const volatile void __iomem *addr)
306{
Sai Prakash Ranjan21003192022-05-18 22:14:16 +0530307 u8 val;
308
Sai Prakash Ranjan5e5ff732022-10-17 20:04:50 +0530309 log_read_mmio(8, addr, _THIS_IP_, _RET_IP_);
Sai Prakash Ranjan21003192022-05-18 22:14:16 +0530310 val = __raw_readb(addr);
Sai Prakash Ranjan5e5ff732022-10-17 20:04:50 +0530311 log_post_read_mmio(val, 8, addr, _THIS_IP_, _RET_IP_);
Sai Prakash Ranjan21003192022-05-18 22:14:16 +0530312 return val;
Sinan Kaya8875c552018-04-06 14:02:45 -0400313}
Arnd Bergmann1c8d2962014-11-11 19:55:45 +0100314#endif
315
316#ifndef readw_relaxed
Sinan Kaya8875c552018-04-06 14:02:45 -0400317#define readw_relaxed readw_relaxed
318static inline u16 readw_relaxed(const volatile void __iomem *addr)
319{
Sai Prakash Ranjan21003192022-05-18 22:14:16 +0530320 u16 val;
321
Sai Prakash Ranjan5e5ff732022-10-17 20:04:50 +0530322 log_read_mmio(16, addr, _THIS_IP_, _RET_IP_);
Vladimir Oltean05d38552023-01-09 15:11:53 +0200323 val = __le16_to_cpu((__le16 __force)__raw_readw(addr));
Sai Prakash Ranjan5e5ff732022-10-17 20:04:50 +0530324 log_post_read_mmio(val, 16, addr, _THIS_IP_, _RET_IP_);
Sai Prakash Ranjan21003192022-05-18 22:14:16 +0530325 return val;
Sinan Kaya8875c552018-04-06 14:02:45 -0400326}
Arnd Bergmann1c8d2962014-11-11 19:55:45 +0100327#endif
328
329#ifndef readl_relaxed
Sinan Kaya8875c552018-04-06 14:02:45 -0400330#define readl_relaxed readl_relaxed
331static inline u32 readl_relaxed(const volatile void __iomem *addr)
332{
Sai Prakash Ranjan21003192022-05-18 22:14:16 +0530333 u32 val;
334
Sai Prakash Ranjan5e5ff732022-10-17 20:04:50 +0530335 log_read_mmio(32, addr, _THIS_IP_, _RET_IP_);
Vladimir Oltean05d38552023-01-09 15:11:53 +0200336 val = __le32_to_cpu((__le32 __force)__raw_readl(addr));
Sai Prakash Ranjan5e5ff732022-10-17 20:04:50 +0530337 log_post_read_mmio(val, 32, addr, _THIS_IP_, _RET_IP_);
Sai Prakash Ranjan21003192022-05-18 22:14:16 +0530338 return val;
Sinan Kaya8875c552018-04-06 14:02:45 -0400339}
Arnd Bergmann1c8d2962014-11-11 19:55:45 +0100340#endif
341
Robin Murphye5112672016-04-26 11:38:20 +0100342#if defined(readq) && !defined(readq_relaxed)
Sinan Kaya8875c552018-04-06 14:02:45 -0400343#define readq_relaxed readq_relaxed
344static inline u64 readq_relaxed(const volatile void __iomem *addr)
345{
Sai Prakash Ranjan21003192022-05-18 22:14:16 +0530346 u64 val;
347
Sai Prakash Ranjan5e5ff732022-10-17 20:04:50 +0530348 log_read_mmio(64, addr, _THIS_IP_, _RET_IP_);
Vladimir Oltean05d38552023-01-09 15:11:53 +0200349 val = __le64_to_cpu((__le64 __force)__raw_readq(addr));
Sai Prakash Ranjan5e5ff732022-10-17 20:04:50 +0530350 log_post_read_mmio(val, 64, addr, _THIS_IP_, _RET_IP_);
Sai Prakash Ranjan21003192022-05-18 22:14:16 +0530351 return val;
Sinan Kaya8875c552018-04-06 14:02:45 -0400352}
Will Deacon9439eb32013-09-03 10:44:00 +0100353#endif
Arnd Bergmann3f7e212d2009-05-13 22:56:35 +0000354
Arnd Bergmann1c8d2962014-11-11 19:55:45 +0100355#ifndef writeb_relaxed
Sinan Kayaa71e7c42018-04-06 14:02:46 -0400356#define writeb_relaxed writeb_relaxed
357static inline void writeb_relaxed(u8 value, volatile void __iomem *addr)
358{
Sai Prakash Ranjan5e5ff732022-10-17 20:04:50 +0530359 log_write_mmio(value, 8, addr, _THIS_IP_, _RET_IP_);
Sinan Kayaa71e7c42018-04-06 14:02:46 -0400360 __raw_writeb(value, addr);
Sai Prakash Ranjan5e5ff732022-10-17 20:04:50 +0530361 log_post_write_mmio(value, 8, addr, _THIS_IP_, _RET_IP_);
Sinan Kayaa71e7c42018-04-06 14:02:46 -0400362}
Arnd Bergmann1c8d2962014-11-11 19:55:45 +0100363#endif
364
365#ifndef writew_relaxed
Sinan Kayaa71e7c42018-04-06 14:02:46 -0400366#define writew_relaxed writew_relaxed
367static inline void writew_relaxed(u16 value, volatile void __iomem *addr)
368{
Sai Prakash Ranjan5e5ff732022-10-17 20:04:50 +0530369 log_write_mmio(value, 16, addr, _THIS_IP_, _RET_IP_);
Vladimir Oltean05d38552023-01-09 15:11:53 +0200370 __raw_writew((u16 __force)cpu_to_le16(value), addr);
Sai Prakash Ranjan5e5ff732022-10-17 20:04:50 +0530371 log_post_write_mmio(value, 16, addr, _THIS_IP_, _RET_IP_);
Sinan Kayaa71e7c42018-04-06 14:02:46 -0400372}
Arnd Bergmann1c8d2962014-11-11 19:55:45 +0100373#endif
374
375#ifndef writel_relaxed
Sinan Kayaa71e7c42018-04-06 14:02:46 -0400376#define writel_relaxed writel_relaxed
377static inline void writel_relaxed(u32 value, volatile void __iomem *addr)
378{
Sai Prakash Ranjan5e5ff732022-10-17 20:04:50 +0530379 log_write_mmio(value, 32, addr, _THIS_IP_, _RET_IP_);
Vladimir Oltean05d38552023-01-09 15:11:53 +0200380 __raw_writel((u32 __force)__cpu_to_le32(value), addr);
Sai Prakash Ranjan5e5ff732022-10-17 20:04:50 +0530381 log_post_write_mmio(value, 32, addr, _THIS_IP_, _RET_IP_);
Sinan Kayaa71e7c42018-04-06 14:02:46 -0400382}
Arnd Bergmann1c8d2962014-11-11 19:55:45 +0100383#endif
384
Robin Murphye5112672016-04-26 11:38:20 +0100385#if defined(writeq) && !defined(writeq_relaxed)
Sinan Kayaa71e7c42018-04-06 14:02:46 -0400386#define writeq_relaxed writeq_relaxed
387static inline void writeq_relaxed(u64 value, volatile void __iomem *addr)
388{
Sai Prakash Ranjan5e5ff732022-10-17 20:04:50 +0530389 log_write_mmio(value, 64, addr, _THIS_IP_, _RET_IP_);
Vladimir Oltean05d38552023-01-09 15:11:53 +0200390 __raw_writeq((u64 __force)__cpu_to_le64(value), addr);
Sai Prakash Ranjan5e5ff732022-10-17 20:04:50 +0530391 log_post_write_mmio(value, 64, addr, _THIS_IP_, _RET_IP_);
Sinan Kayaa71e7c42018-04-06 14:02:46 -0400392}
Arnd Bergmann1c8d2962014-11-11 19:55:45 +0100393#endif
394
395/*
Thierry Reding9ab3a7a2014-07-04 13:07:57 +0200396 * {read,write}s{b,w,l,q}() repeatedly access the same memory address in
397 * native endianness in 8-, 16-, 32- or 64-bit chunks (@count times).
398 */
399#ifndef readsb
400#define readsb readsb
401static inline void readsb(const volatile void __iomem *addr, void *buffer,
402 unsigned int count)
Arnd Bergmann3f7e212d2009-05-13 22:56:35 +0000403{
404 if (count) {
405 u8 *buf = buffer;
Thierry Reding9ab3a7a2014-07-04 13:07:57 +0200406
Arnd Bergmann3f7e212d2009-05-13 22:56:35 +0000407 do {
Thierry Reding9ab3a7a2014-07-04 13:07:57 +0200408 u8 x = __raw_readb(addr);
Arnd Bergmann3f7e212d2009-05-13 22:56:35 +0000409 *buf++ = x;
410 } while (--count);
411 }
412}
Mike Frysinger35dbc0e2010-10-18 03:09:39 -0400413#endif
Arnd Bergmann3f7e212d2009-05-13 22:56:35 +0000414
Thierry Reding9ab3a7a2014-07-04 13:07:57 +0200415#ifndef readsw
416#define readsw readsw
417static inline void readsw(const volatile void __iomem *addr, void *buffer,
418 unsigned int count)
Arnd Bergmann3f7e212d2009-05-13 22:56:35 +0000419{
420 if (count) {
421 u16 *buf = buffer;
Thierry Reding9ab3a7a2014-07-04 13:07:57 +0200422
Arnd Bergmann3f7e212d2009-05-13 22:56:35 +0000423 do {
Thierry Reding9ab3a7a2014-07-04 13:07:57 +0200424 u16 x = __raw_readw(addr);
Arnd Bergmann3f7e212d2009-05-13 22:56:35 +0000425 *buf++ = x;
426 } while (--count);
427 }
428}
Mike Frysinger35dbc0e2010-10-18 03:09:39 -0400429#endif
Arnd Bergmann3f7e212d2009-05-13 22:56:35 +0000430
Thierry Reding9ab3a7a2014-07-04 13:07:57 +0200431#ifndef readsl
432#define readsl readsl
433static inline void readsl(const volatile void __iomem *addr, void *buffer,
434 unsigned int count)
Arnd Bergmann3f7e212d2009-05-13 22:56:35 +0000435{
436 if (count) {
437 u32 *buf = buffer;
Thierry Reding9ab3a7a2014-07-04 13:07:57 +0200438
Arnd Bergmann3f7e212d2009-05-13 22:56:35 +0000439 do {
Thierry Reding9ab3a7a2014-07-04 13:07:57 +0200440 u32 x = __raw_readl(addr);
Arnd Bergmann3f7e212d2009-05-13 22:56:35 +0000441 *buf++ = x;
442 } while (--count);
443 }
444}
Mike Frysinger35dbc0e2010-10-18 03:09:39 -0400445#endif
Arnd Bergmann3f7e212d2009-05-13 22:56:35 +0000446
Thierry Reding9ab3a7a2014-07-04 13:07:57 +0200447#ifdef CONFIG_64BIT
448#ifndef readsq
449#define readsq readsq
450static inline void readsq(const volatile void __iomem *addr, void *buffer,
451 unsigned int count)
452{
453 if (count) {
454 u64 *buf = buffer;
455
456 do {
457 u64 x = __raw_readq(addr);
458 *buf++ = x;
459 } while (--count);
460 }
461}
462#endif
463#endif /* CONFIG_64BIT */
464
465#ifndef writesb
466#define writesb writesb
467static inline void writesb(volatile void __iomem *addr, const void *buffer,
468 unsigned int count)
Arnd Bergmann3f7e212d2009-05-13 22:56:35 +0000469{
470 if (count) {
471 const u8 *buf = buffer;
Thierry Reding9ab3a7a2014-07-04 13:07:57 +0200472
Arnd Bergmann3f7e212d2009-05-13 22:56:35 +0000473 do {
Thierry Reding9ab3a7a2014-07-04 13:07:57 +0200474 __raw_writeb(*buf++, addr);
Arnd Bergmann3f7e212d2009-05-13 22:56:35 +0000475 } while (--count);
476 }
477}
Mike Frysinger35dbc0e2010-10-18 03:09:39 -0400478#endif
Arnd Bergmann3f7e212d2009-05-13 22:56:35 +0000479
Thierry Reding9ab3a7a2014-07-04 13:07:57 +0200480#ifndef writesw
481#define writesw writesw
482static inline void writesw(volatile void __iomem *addr, const void *buffer,
483 unsigned int count)
Arnd Bergmann3f7e212d2009-05-13 22:56:35 +0000484{
485 if (count) {
486 const u16 *buf = buffer;
Thierry Reding9ab3a7a2014-07-04 13:07:57 +0200487
Arnd Bergmann3f7e212d2009-05-13 22:56:35 +0000488 do {
Thierry Reding9ab3a7a2014-07-04 13:07:57 +0200489 __raw_writew(*buf++, addr);
Arnd Bergmann3f7e212d2009-05-13 22:56:35 +0000490 } while (--count);
491 }
492}
Mike Frysinger35dbc0e2010-10-18 03:09:39 -0400493#endif
Arnd Bergmann3f7e212d2009-05-13 22:56:35 +0000494
Thierry Reding9ab3a7a2014-07-04 13:07:57 +0200495#ifndef writesl
496#define writesl writesl
497static inline void writesl(volatile void __iomem *addr, const void *buffer,
498 unsigned int count)
Arnd Bergmann3f7e212d2009-05-13 22:56:35 +0000499{
500 if (count) {
501 const u32 *buf = buffer;
Thierry Reding9ab3a7a2014-07-04 13:07:57 +0200502
Arnd Bergmann3f7e212d2009-05-13 22:56:35 +0000503 do {
Thierry Reding9ab3a7a2014-07-04 13:07:57 +0200504 __raw_writel(*buf++, addr);
Arnd Bergmann3f7e212d2009-05-13 22:56:35 +0000505 } while (--count);
506 }
507}
Mike Frysinger35dbc0e2010-10-18 03:09:39 -0400508#endif
Arnd Bergmann3f7e212d2009-05-13 22:56:35 +0000509
Thierry Reding9ab3a7a2014-07-04 13:07:57 +0200510#ifdef CONFIG_64BIT
511#ifndef writesq
512#define writesq writesq
513static inline void writesq(volatile void __iomem *addr, const void *buffer,
514 unsigned int count)
515{
516 if (count) {
517 const u64 *buf = buffer;
Arnd Bergmann3f7e212d2009-05-13 22:56:35 +0000518
Thierry Reding9ab3a7a2014-07-04 13:07:57 +0200519 do {
520 __raw_writeq(*buf++, addr);
521 } while (--count);
522 }
523}
524#endif
525#endif /* CONFIG_64BIT */
Arnd Bergmann3f7e212d2009-05-13 22:56:35 +0000526
Thierry Reding9216efa2014-10-01 15:20:33 +0200527#ifndef PCI_IOBASE
528#define PCI_IOBASE ((void __iomem *)0)
529#endif
530
GuanXuetao7dc59bd2011-02-22 19:06:43 +0800531#ifndef IO_SPACE_LIMIT
532#define IO_SPACE_LIMIT 0xffff
533#endif
Arnd Bergmann3f7e212d2009-05-13 22:56:35 +0000534
Thierry Reding9216efa2014-10-01 15:20:33 +0200535/*
536 * {in,out}{b,w,l}() access little endian I/O. {in,out}{b,w,l}_p() can be
537 * implemented on hardware that needs an additional delay for I/O accesses to
538 * take effect.
539 */
540
John Garryf009c892020-03-28 00:06:12 +0800541#if !defined(inb) && !defined(_inb)
542#define _inb _inb
Stafford Horne214ba352020-07-26 12:11:54 +0900543static inline u8 _inb(unsigned long addr)
Thierry Reding9216efa2014-10-01 15:20:33 +0200544{
Sinan Kaya87fe2d52018-04-05 09:09:13 -0400545 u8 val;
546
547 __io_pbr();
548 val = __raw_readb(PCI_IOBASE + addr);
Will Deaconabbbbc832019-02-22 18:04:52 +0000549 __io_par(val);
Sinan Kaya87fe2d52018-04-05 09:09:13 -0400550 return val;
Thierry Reding9216efa2014-10-01 15:20:33 +0200551}
552#endif
553
John Garryf009c892020-03-28 00:06:12 +0800554#if !defined(inw) && !defined(_inw)
555#define _inw _inw
556static inline u16 _inw(unsigned long addr)
Thierry Reding9216efa2014-10-01 15:20:33 +0200557{
Sinan Kaya87fe2d52018-04-05 09:09:13 -0400558 u16 val;
559
560 __io_pbr();
Stafford Hornec1d55d52020-07-29 21:03:14 +0900561 val = __le16_to_cpu((__le16 __force)__raw_readw(PCI_IOBASE + addr));
Will Deaconabbbbc832019-02-22 18:04:52 +0000562 __io_par(val);
Sinan Kaya87fe2d52018-04-05 09:09:13 -0400563 return val;
Thierry Reding9216efa2014-10-01 15:20:33 +0200564}
565#endif
566
John Garryf009c892020-03-28 00:06:12 +0800567#if !defined(inl) && !defined(_inl)
568#define _inl _inl
Stafford Horne214ba352020-07-26 12:11:54 +0900569static inline u32 _inl(unsigned long addr)
Thierry Reding9216efa2014-10-01 15:20:33 +0200570{
Sinan Kaya87fe2d52018-04-05 09:09:13 -0400571 u32 val;
572
573 __io_pbr();
Stafford Hornec1d55d52020-07-29 21:03:14 +0900574 val = __le32_to_cpu((__le32 __force)__raw_readl(PCI_IOBASE + addr));
Will Deaconabbbbc832019-02-22 18:04:52 +0000575 __io_par(val);
Sinan Kaya87fe2d52018-04-05 09:09:13 -0400576 return val;
Thierry Reding9216efa2014-10-01 15:20:33 +0200577}
578#endif
579
John Garryf009c892020-03-28 00:06:12 +0800580#if !defined(outb) && !defined(_outb)
581#define _outb _outb
582static inline void _outb(u8 value, unsigned long addr)
Thierry Reding9216efa2014-10-01 15:20:33 +0200583{
Sinan Kayaa7851aa2018-04-05 09:09:12 -0400584 __io_pbw();
585 __raw_writeb(value, PCI_IOBASE + addr);
586 __io_paw();
Thierry Reding9216efa2014-10-01 15:20:33 +0200587}
588#endif
589
John Garryf009c892020-03-28 00:06:12 +0800590#if !defined(outw) && !defined(_outw)
591#define _outw _outw
592static inline void _outw(u16 value, unsigned long addr)
Thierry Reding9216efa2014-10-01 15:20:33 +0200593{
Sinan Kayaa7851aa2018-04-05 09:09:12 -0400594 __io_pbw();
Stafford Hornec1d55d52020-07-29 21:03:14 +0900595 __raw_writew((u16 __force)cpu_to_le16(value), PCI_IOBASE + addr);
Sinan Kayaa7851aa2018-04-05 09:09:12 -0400596 __io_paw();
Thierry Reding9216efa2014-10-01 15:20:33 +0200597}
598#endif
599
John Garryf009c892020-03-28 00:06:12 +0800600#if !defined(outl) && !defined(_outl)
601#define _outl _outl
602static inline void _outl(u32 value, unsigned long addr)
Thierry Reding9216efa2014-10-01 15:20:33 +0200603{
Sinan Kayaa7851aa2018-04-05 09:09:12 -0400604 __io_pbw();
Stafford Hornec1d55d52020-07-29 21:03:14 +0900605 __raw_writel((u32 __force)cpu_to_le32(value), PCI_IOBASE + addr);
Sinan Kayaa7851aa2018-04-05 09:09:12 -0400606 __io_paw();
Thierry Reding9216efa2014-10-01 15:20:33 +0200607}
608#endif
609
John Garryf009c892020-03-28 00:06:12 +0800610#include <linux/logic_pio.h>
611
612#ifndef inb
613#define inb _inb
614#endif
615
616#ifndef inw
617#define inw _inw
618#endif
619
620#ifndef inl
621#define inl _inl
622#endif
623
624#ifndef outb
625#define outb _outb
626#endif
627
628#ifndef outw
629#define outw _outw
630#endif
631
632#ifndef outl
633#define outl _outl
634#endif
635
Thierry Reding9216efa2014-10-01 15:20:33 +0200636#ifndef inb_p
637#define inb_p inb_p
638static inline u8 inb_p(unsigned long addr)
639{
640 return inb(addr);
641}
642#endif
643
644#ifndef inw_p
645#define inw_p inw_p
646static inline u16 inw_p(unsigned long addr)
647{
648 return inw(addr);
649}
650#endif
651
652#ifndef inl_p
653#define inl_p inl_p
654static inline u32 inl_p(unsigned long addr)
655{
656 return inl(addr);
657}
658#endif
659
660#ifndef outb_p
661#define outb_p outb_p
662static inline void outb_p(u8 value, unsigned long addr)
663{
664 outb(value, addr);
665}
666#endif
667
668#ifndef outw_p
669#define outw_p outw_p
670static inline void outw_p(u16 value, unsigned long addr)
671{
672 outw(value, addr);
673}
674#endif
675
676#ifndef outl_p
677#define outl_p outl_p
678static inline void outl_p(u32 value, unsigned long addr)
679{
680 outl(value, addr);
681}
682#endif
683
Thierry Reding9ab3a7a2014-07-04 13:07:57 +0200684/*
685 * {in,out}s{b,w,l}{,_p}() are variants of the above that repeatedly access a
686 * single I/O port multiple times.
687 */
688
689#ifndef insb
690#define insb insb
691static inline void insb(unsigned long addr, void *buffer, unsigned int count)
692{
693 readsb(PCI_IOBASE + addr, buffer, count);
694}
695#endif
696
697#ifndef insw
698#define insw insw
699static inline void insw(unsigned long addr, void *buffer, unsigned int count)
700{
701 readsw(PCI_IOBASE + addr, buffer, count);
702}
703#endif
704
705#ifndef insl
706#define insl insl
707static inline void insl(unsigned long addr, void *buffer, unsigned int count)
708{
709 readsl(PCI_IOBASE + addr, buffer, count);
710}
711#endif
712
713#ifndef outsb
714#define outsb outsb
715static inline void outsb(unsigned long addr, const void *buffer,
716 unsigned int count)
717{
718 writesb(PCI_IOBASE + addr, buffer, count);
719}
720#endif
721
722#ifndef outsw
723#define outsw outsw
724static inline void outsw(unsigned long addr, const void *buffer,
725 unsigned int count)
726{
727 writesw(PCI_IOBASE + addr, buffer, count);
728}
729#endif
730
731#ifndef outsl
732#define outsl outsl
733static inline void outsl(unsigned long addr, const void *buffer,
734 unsigned int count)
735{
736 writesl(PCI_IOBASE + addr, buffer, count);
737}
738#endif
739
740#ifndef insb_p
741#define insb_p insb_p
742static inline void insb_p(unsigned long addr, void *buffer, unsigned int count)
743{
744 insb(addr, buffer, count);
745}
746#endif
747
748#ifndef insw_p
749#define insw_p insw_p
750static inline void insw_p(unsigned long addr, void *buffer, unsigned int count)
751{
752 insw(addr, buffer, count);
753}
754#endif
755
756#ifndef insl_p
757#define insl_p insl_p
758static inline void insl_p(unsigned long addr, void *buffer, unsigned int count)
759{
760 insl(addr, buffer, count);
761}
762#endif
763
764#ifndef outsb_p
765#define outsb_p outsb_p
766static inline void outsb_p(unsigned long addr, const void *buffer,
767 unsigned int count)
768{
769 outsb(addr, buffer, count);
770}
771#endif
772
773#ifndef outsw_p
774#define outsw_p outsw_p
775static inline void outsw_p(unsigned long addr, const void *buffer,
776 unsigned int count)
777{
778 outsw(addr, buffer, count);
779}
780#endif
781
782#ifndef outsl_p
783#define outsl_p outsl_p
784static inline void outsl_p(unsigned long addr, const void *buffer,
785 unsigned int count)
786{
787 outsl(addr, buffer, count);
788}
789#endif
790
Thierry Reding9216efa2014-10-01 15:20:33 +0200791#ifndef CONFIG_GENERIC_IOMAP
792#ifndef ioread8
793#define ioread8 ioread8
794static inline u8 ioread8(const volatile void __iomem *addr)
795{
796 return readb(addr);
797}
798#endif
799
800#ifndef ioread16
801#define ioread16 ioread16
802static inline u16 ioread16(const volatile void __iomem *addr)
803{
804 return readw(addr);
805}
806#endif
807
808#ifndef ioread32
809#define ioread32 ioread32
810static inline u32 ioread32(const volatile void __iomem *addr)
811{
812 return readl(addr);
813}
814#endif
815
Horia Geantă9e44fb12016-05-19 18:10:56 +0300816#ifdef CONFIG_64BIT
817#ifndef ioread64
818#define ioread64 ioread64
819static inline u64 ioread64(const volatile void __iomem *addr)
820{
821 return readq(addr);
822}
823#endif
824#endif /* CONFIG_64BIT */
825
Thierry Reding9216efa2014-10-01 15:20:33 +0200826#ifndef iowrite8
827#define iowrite8 iowrite8
828static inline void iowrite8(u8 value, volatile void __iomem *addr)
829{
830 writeb(value, addr);
831}
832#endif
833
834#ifndef iowrite16
835#define iowrite16 iowrite16
836static inline void iowrite16(u16 value, volatile void __iomem *addr)
837{
838 writew(value, addr);
839}
840#endif
841
842#ifndef iowrite32
843#define iowrite32 iowrite32
844static inline void iowrite32(u32 value, volatile void __iomem *addr)
845{
846 writel(value, addr);
847}
848#endif
849
Horia Geantă9e44fb12016-05-19 18:10:56 +0300850#ifdef CONFIG_64BIT
851#ifndef iowrite64
852#define iowrite64 iowrite64
853static inline void iowrite64(u64 value, volatile void __iomem *addr)
854{
855 writeq(value, addr);
856}
857#endif
858#endif /* CONFIG_64BIT */
859
Thierry Reding9216efa2014-10-01 15:20:33 +0200860#ifndef ioread16be
861#define ioread16be ioread16be
862static inline u16 ioread16be(const volatile void __iomem *addr)
863{
Horia Geantă7a1aedb2016-05-19 18:10:43 +0300864 return swab16(readw(addr));
Thierry Reding9216efa2014-10-01 15:20:33 +0200865}
866#endif
867
868#ifndef ioread32be
869#define ioread32be ioread32be
870static inline u32 ioread32be(const volatile void __iomem *addr)
871{
Horia Geantă7a1aedb2016-05-19 18:10:43 +0300872 return swab32(readl(addr));
Thierry Reding9216efa2014-10-01 15:20:33 +0200873}
874#endif
875
Horia Geantă9e44fb12016-05-19 18:10:56 +0300876#ifdef CONFIG_64BIT
877#ifndef ioread64be
878#define ioread64be ioread64be
879static inline u64 ioread64be(const volatile void __iomem *addr)
880{
881 return swab64(readq(addr));
882}
883#endif
884#endif /* CONFIG_64BIT */
885
Thierry Reding9216efa2014-10-01 15:20:33 +0200886#ifndef iowrite16be
887#define iowrite16be iowrite16be
888static inline void iowrite16be(u16 value, void volatile __iomem *addr)
889{
Horia Geantă7a1aedb2016-05-19 18:10:43 +0300890 writew(swab16(value), addr);
Thierry Reding9216efa2014-10-01 15:20:33 +0200891}
892#endif
893
894#ifndef iowrite32be
895#define iowrite32be iowrite32be
896static inline void iowrite32be(u32 value, volatile void __iomem *addr)
897{
Horia Geantă7a1aedb2016-05-19 18:10:43 +0300898 writel(swab32(value), addr);
Thierry Reding9216efa2014-10-01 15:20:33 +0200899}
900#endif
Thierry Reding9ab3a7a2014-07-04 13:07:57 +0200901
Horia Geantă9e44fb12016-05-19 18:10:56 +0300902#ifdef CONFIG_64BIT
903#ifndef iowrite64be
904#define iowrite64be iowrite64be
905static inline void iowrite64be(u64 value, volatile void __iomem *addr)
906{
907 writeq(swab64(value), addr);
908}
909#endif
910#endif /* CONFIG_64BIT */
911
Thierry Reding9ab3a7a2014-07-04 13:07:57 +0200912#ifndef ioread8_rep
913#define ioread8_rep ioread8_rep
914static inline void ioread8_rep(const volatile void __iomem *addr, void *buffer,
915 unsigned int count)
916{
917 readsb(addr, buffer, count);
918}
919#endif
920
921#ifndef ioread16_rep
922#define ioread16_rep ioread16_rep
923static inline void ioread16_rep(const volatile void __iomem *addr,
924 void *buffer, unsigned int count)
925{
926 readsw(addr, buffer, count);
927}
928#endif
929
930#ifndef ioread32_rep
931#define ioread32_rep ioread32_rep
932static inline void ioread32_rep(const volatile void __iomem *addr,
933 void *buffer, unsigned int count)
934{
935 readsl(addr, buffer, count);
936}
937#endif
938
Horia Geantă9e44fb12016-05-19 18:10:56 +0300939#ifdef CONFIG_64BIT
940#ifndef ioread64_rep
941#define ioread64_rep ioread64_rep
942static inline void ioread64_rep(const volatile void __iomem *addr,
943 void *buffer, unsigned int count)
944{
945 readsq(addr, buffer, count);
946}
947#endif
948#endif /* CONFIG_64BIT */
949
Thierry Reding9ab3a7a2014-07-04 13:07:57 +0200950#ifndef iowrite8_rep
951#define iowrite8_rep iowrite8_rep
952static inline void iowrite8_rep(volatile void __iomem *addr,
953 const void *buffer,
954 unsigned int count)
955{
956 writesb(addr, buffer, count);
957}
958#endif
959
960#ifndef iowrite16_rep
961#define iowrite16_rep iowrite16_rep
962static inline void iowrite16_rep(volatile void __iomem *addr,
963 const void *buffer,
964 unsigned int count)
965{
966 writesw(addr, buffer, count);
967}
968#endif
969
970#ifndef iowrite32_rep
971#define iowrite32_rep iowrite32_rep
972static inline void iowrite32_rep(volatile void __iomem *addr,
973 const void *buffer,
974 unsigned int count)
975{
976 writesl(addr, buffer, count);
977}
978#endif
Horia Geantă9e44fb12016-05-19 18:10:56 +0300979
980#ifdef CONFIG_64BIT
981#ifndef iowrite64_rep
982#define iowrite64_rep iowrite64_rep
983static inline void iowrite64_rep(volatile void __iomem *addr,
984 const void *buffer,
985 unsigned int count)
986{
987 writesq(addr, buffer, count);
988}
989#endif
990#endif /* CONFIG_64BIT */
Thierry Reding9216efa2014-10-01 15:20:33 +0200991#endif /* CONFIG_GENERIC_IOMAP */
992
Arnd Bergmann3f7e212d2009-05-13 22:56:35 +0000993#ifdef __KERNEL__
994
Thierry Reding9216efa2014-10-01 15:20:33 +0200995#define __io_virt(x) ((void __force *)(x))
Arnd Bergmann3f7e212d2009-05-13 22:56:35 +0000996
Arnd Bergmann3f7e212d2009-05-13 22:56:35 +0000997/*
998 * Change virtual addresses to physical addresses and vv.
999 * These are pretty trivial
1000 */
Jan Glaubercd248342012-11-29 12:50:30 +01001001#ifndef virt_to_phys
Thierry Reding9216efa2014-10-01 15:20:33 +02001002#define virt_to_phys virt_to_phys
Arnd Bergmann3f7e212d2009-05-13 22:56:35 +00001003static inline unsigned long virt_to_phys(volatile void *address)
1004{
1005 return __pa((unsigned long)address);
1006}
Thierry Reding9216efa2014-10-01 15:20:33 +02001007#endif
Arnd Bergmann3f7e212d2009-05-13 22:56:35 +00001008
Thierry Reding9216efa2014-10-01 15:20:33 +02001009#ifndef phys_to_virt
1010#define phys_to_virt phys_to_virt
Arnd Bergmann3f7e212d2009-05-13 22:56:35 +00001011static inline void *phys_to_virt(unsigned long address)
1012{
1013 return __va(address);
1014}
Jan Glaubercd248342012-11-29 12:50:30 +01001015#endif
Arnd Bergmann3f7e212d2009-05-13 22:56:35 +00001016
Luis R. Rodriguez8c7ea502015-07-09 17:28:16 -07001017/**
1018 * DOC: ioremap() and ioremap_*() variants
1019 *
Christoph Hellwig97c98012019-08-11 14:53:20 +02001020 * Architectures with an MMU are expected to provide ioremap() and iounmap()
Christoph Hellwig80b0ca92019-08-13 11:24:04 +02001021 * themselves or rely on GENERIC_IOREMAP. For NOMMU architectures we provide
1022 * a default nop-op implementation that expect that the physical address used
1023 * for MMIO are already marked as uncached, and can be used as kernel virtual
1024 * addresses.
Luis R. Rodriguez8c7ea502015-07-09 17:28:16 -07001025 *
Christoph Hellwig97c98012019-08-11 14:53:20 +02001026 * ioremap_wc() and ioremap_wt() can provide more relaxed caching attributes
1027 * for specific drivers if the architecture choses to implement them. If they
Hector Martin7c566bb5e2021-02-11 21:35:46 +09001028 * are not implemented we fall back to plain ioremap. Conversely, ioremap_np()
1029 * can provide stricter non-posted write semantics if the architecture
1030 * implements them.
Luis R. Rodriguez8c7ea502015-07-09 17:28:16 -07001031 */
Christoph Hellwige9713392019-08-13 07:53:05 +02001032#ifndef CONFIG_MMU
Thierry Reding9216efa2014-10-01 15:20:33 +02001033#ifndef ioremap
1034#define ioremap ioremap
1035static inline void __iomem *ioremap(phys_addr_t offset, size_t size)
1036{
1037 return (void __iomem *)(unsigned long)offset;
1038}
1039#endif
1040
Greentime Hub3ada9d2017-11-22 18:57:46 +08001041#ifndef iounmap
1042#define iounmap iounmap
Adam Borowski2fbc3492021-09-12 23:26:06 +02001043static inline void iounmap(volatile void __iomem *addr)
Greentime Hub3ada9d2017-11-22 18:57:46 +08001044{
1045}
1046#endif
Christoph Hellwig80b0ca92019-08-13 11:24:04 +02001047#elif defined(CONFIG_GENERIC_IOREMAP)
Mike Rapoportca5999f2020-06-08 21:32:38 -07001048#include <linux/pgtable.h>
Christoph Hellwig80b0ca92019-08-13 11:24:04 +02001049
Christophe Leroy76133662023-07-06 23:45:05 +08001050void __iomem *generic_ioremap_prot(phys_addr_t phys_addr, size_t size,
1051 pgprot_t prot);
1052
Kefeng Wangabc59922022-06-07 20:50:23 +08001053void __iomem *ioremap_prot(phys_addr_t phys_addr, size_t size,
1054 unsigned long prot);
Christoph Hellwig80b0ca92019-08-13 11:24:04 +02001055void iounmap(volatile void __iomem *addr);
Christophe Leroy76133662023-07-06 23:45:05 +08001056void generic_iounmap(volatile void __iomem *addr);
Christoph Hellwig80b0ca92019-08-13 11:24:04 +02001057
Baoquan Hedfdc6ba2023-07-06 23:45:06 +08001058#ifndef ioremap
1059#define ioremap ioremap
Christoph Hellwig80b0ca92019-08-13 11:24:04 +02001060static inline void __iomem *ioremap(phys_addr_t addr, size_t size)
1061{
1062 /* _PAGE_IOREMAP needs to be supplied by the architecture */
1063 return ioremap_prot(addr, size, _PAGE_IOREMAP);
1064}
Baoquan Hedfdc6ba2023-07-06 23:45:06 +08001065#endif
Christoph Hellwig80b0ca92019-08-13 11:24:04 +02001066#endif /* !CONFIG_MMU || CONFIG_GENERIC_IOREMAP */
Christoph Hellwig97c98012019-08-11 14:53:20 +02001067
Arnd Bergmann3f7e212d2009-05-13 22:56:35 +00001068#ifndef ioremap_wc
Christoph Hellwigd092a872019-10-16 08:09:38 +02001069#define ioremap_wc ioremap
Arnd Bergmann3f7e212d2009-05-13 22:56:35 +00001070#endif
1071
Toshi Kanid8382702015-06-04 18:55:15 +02001072#ifndef ioremap_wt
Christoph Hellwigd092a872019-10-16 08:09:38 +02001073#define ioremap_wt ioremap
Toshi Kanid8382702015-06-04 18:55:15 +02001074#endif
1075
Christoph Hellwige9713392019-08-13 07:53:05 +02001076/*
1077 * ioremap_uc is special in that we do require an explicit architecture
1078 * implementation. In general you do not want to use this function in a
1079 * driver and use plain ioremap, which is uncached by default. Similarly
1080 * architectures should not implement it unless they have a very good
1081 * reason.
1082 */
1083#ifndef ioremap_uc
1084#define ioremap_uc ioremap_uc
1085static inline void __iomem *ioremap_uc(phys_addr_t offset, size_t size)
1086{
1087 return NULL;
1088}
Hector Martinea962922021-04-09 14:20:38 +09001089#endif
Hector Martin7c566bb5e2021-02-11 21:35:46 +09001090
1091/*
1092 * ioremap_np needs an explicit architecture implementation, as it
1093 * requests stronger semantics than regular ioremap(). Portable drivers
1094 * should instead use one of the higher-level abstractions, like
1095 * devm_ioremap_resource(), to choose the correct variant for any given
1096 * device and bus. Portable drivers with a good reason to want non-posted
1097 * write semantics should always provide an ioremap() fallback in case
1098 * ioremap_np() is not available.
1099 */
1100#ifndef ioremap_np
1101#define ioremap_np ioremap_np
1102static inline void __iomem *ioremap_np(phys_addr_t offset, size_t size)
1103{
1104 return NULL;
1105}
1106#endif
1107
Uwe Kleine-Königce816fa2014-04-07 15:39:19 -07001108#ifdef CONFIG_HAS_IOPORT_MAP
Arnd Bergmann3f7e212d2009-05-13 22:56:35 +00001109#ifndef CONFIG_GENERIC_IOMAP
Thierry Reding9216efa2014-10-01 15:20:33 +02001110#ifndef ioport_map
1111#define ioport_map ioport_map
Arnd Bergmann3f7e212d2009-05-13 22:56:35 +00001112static inline void __iomem *ioport_map(unsigned long port, unsigned int nr)
1113{
Andrew Murray500dd2322018-09-13 13:48:27 +01001114 port &= IO_SPACE_LIMIT;
1115 return (port > MMIO_UPPER_LIMIT) ? NULL : PCI_IOBASE + port;
Arnd Bergmann3f7e212d2009-05-13 22:56:35 +00001116}
Linus Torvalds316e8d72021-09-19 17:13:35 -07001117#define ARCH_HAS_GENERIC_IOPORT_MAP
Thierry Reding9216efa2014-10-01 15:20:33 +02001118#endif
Arnd Bergmann3f7e212d2009-05-13 22:56:35 +00001119
Thierry Reding9216efa2014-10-01 15:20:33 +02001120#ifndef ioport_unmap
1121#define ioport_unmap ioport_unmap
Arnd Bergmann3f7e212d2009-05-13 22:56:35 +00001122static inline void ioport_unmap(void __iomem *p)
1123{
1124}
Thierry Reding9216efa2014-10-01 15:20:33 +02001125#endif
Arnd Bergmann3f7e212d2009-05-13 22:56:35 +00001126#else /* CONFIG_GENERIC_IOMAP */
1127extern void __iomem *ioport_map(unsigned long port, unsigned int nr);
1128extern void ioport_unmap(void __iomem *p);
1129#endif /* CONFIG_GENERIC_IOMAP */
Uwe Kleine-Königce816fa2014-04-07 15:39:19 -07001130#endif /* CONFIG_HAS_IOPORT_MAP */
Arnd Bergmann3f7e212d2009-05-13 22:56:35 +00001131
Lorenzo Pieralisif5810e52020-09-16 12:06:58 +01001132#ifndef CONFIG_GENERIC_IOMAP
Lorenzo Pieralisif5810e52020-09-16 12:06:58 +01001133#ifndef pci_iounmap
Linus Torvalds316e8d72021-09-19 17:13:35 -07001134#define ARCH_WANTS_GENERIC_PCI_IOUNMAP
Lorenzo Pieralisif5810e52020-09-16 12:06:58 +01001135#endif
Linus Torvalds316e8d72021-09-19 17:13:35 -07001136#endif
Lorenzo Pieralisif5810e52020-09-16 12:06:58 +01001137
Michael Holzheu576ebd72013-05-21 16:08:22 +02001138#ifndef xlate_dev_mem_ptr
Thierry Reding9216efa2014-10-01 15:20:33 +02001139#define xlate_dev_mem_ptr xlate_dev_mem_ptr
1140static inline void *xlate_dev_mem_ptr(phys_addr_t addr)
1141{
1142 return __va(addr);
1143}
1144#endif
1145
1146#ifndef unxlate_dev_mem_ptr
1147#define unxlate_dev_mem_ptr unxlate_dev_mem_ptr
1148static inline void unxlate_dev_mem_ptr(phys_addr_t phys, void *addr)
1149{
1150}
Michael Holzheu576ebd72013-05-21 16:08:22 +02001151#endif
Arnd Bergmann3f7e212d2009-05-13 22:56:35 +00001152
Jan Glaubercd248342012-11-29 12:50:30 +01001153#ifndef memset_io
Thierry Reding9216efa2014-10-01 15:20:33 +02001154#define memset_io memset_io
Andy Shevchenkoc2327da2017-06-30 20:09:32 +03001155/**
1156 * memset_io Set a range of I/O memory to a constant value
1157 * @addr: The beginning of the I/O-memory range to set
1158 * @val: The value to set the memory to
1159 * @count: The number of bytes to set
1160 *
1161 * Set a range of I/O memory to a given value.
1162 */
Thierry Reding9216efa2014-10-01 15:20:33 +02001163static inline void memset_io(volatile void __iomem *addr, int value,
1164 size_t size)
1165{
1166 memset(__io_virt(addr), value, size);
1167}
Jan Glaubercd248342012-11-29 12:50:30 +01001168#endif
1169
1170#ifndef memcpy_fromio
Thierry Reding9216efa2014-10-01 15:20:33 +02001171#define memcpy_fromio memcpy_fromio
Andy Shevchenkoc2327da2017-06-30 20:09:32 +03001172/**
1173 * memcpy_fromio Copy a block of data from I/O memory
1174 * @dst: The (RAM) destination for the copy
1175 * @src: The (I/O memory) source for the data
1176 * @count: The number of bytes to copy
1177 *
1178 * Copy a block of data from I/O memory.
1179 */
Thierry Reding9216efa2014-10-01 15:20:33 +02001180static inline void memcpy_fromio(void *buffer,
1181 const volatile void __iomem *addr,
1182 size_t size)
1183{
1184 memcpy(buffer, __io_virt(addr), size);
1185}
Jan Glaubercd248342012-11-29 12:50:30 +01001186#endif
Thierry Reding9216efa2014-10-01 15:20:33 +02001187
Jan Glaubercd248342012-11-29 12:50:30 +01001188#ifndef memcpy_toio
Thierry Reding9216efa2014-10-01 15:20:33 +02001189#define memcpy_toio memcpy_toio
Andy Shevchenkoc2327da2017-06-30 20:09:32 +03001190/**
1191 * memcpy_toio Copy a block of data into I/O memory
1192 * @dst: The (I/O memory) destination for the copy
1193 * @src: The (RAM) source for the data
1194 * @count: The number of bytes to copy
1195 *
1196 * Copy a block of data to I/O memory.
1197 */
Thierry Reding9216efa2014-10-01 15:20:33 +02001198static inline void memcpy_toio(volatile void __iomem *addr, const void *buffer,
1199 size_t size)
1200{
1201 memcpy(__io_virt(addr), buffer, size);
1202}
Jan Glaubercd248342012-11-29 12:50:30 +01001203#endif
Arnd Bergmann3f7e212d2009-05-13 22:56:35 +00001204
Palmer Dabbelt527701e2020-07-09 11:43:21 -07001205extern int devmem_is_allowed(unsigned long pfn);
Palmer Dabbelt527701e2020-07-09 11:43:21 -07001206
Arnd Bergmann3f7e212d2009-05-13 22:56:35 +00001207#endif /* __KERNEL__ */
1208
1209#endif /* __ASM_GENERIC_IO_H */