arch/loongarch/include/asm/percpu.h

/* SPDX-License-Identifier: GPL-2.0 */
/*
 * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
 */
#ifndef __ASM_PERCPU_H
#define __ASM_PERCPU_H

#include <asm/cmpxchg.h>
#include <asm/loongarch.h>

/*
 * The "address" (in fact, offset from $r21) of a per-CPU variable is close to
 * the loading address of main kernel image, but far from where the modules are
 * loaded. Tell the compiler this fact when using explicit relocs.
 */
#if defined(MODULE) && defined(CONFIG_AS_HAS_EXPLICIT_RELOCS)
# if __has_attribute(model)
#  define PER_CPU_ATTRIBUTES __attribute__((model("extreme")))
# else
#  error compiler support for the model attribute is necessary when a recent assembler is used
# endif
#endif

/* Use r21 for fast access */
register unsigned long __my_cpu_offset __asm__("$r21");

static inline void set_my_cpu_offset(unsigned long off)
{
	__my_cpu_offset = off;
	csr_write64(off, PERCPU_BASE_KS);
}

#define __my_cpu_offset					\
({							\
	__asm__ __volatile__("":"+r"(__my_cpu_offset));	\
	__my_cpu_offset;				\
})

#define PERCPU_OP(op, asm_op, c_op)					\
static __always_inline unsigned long __percpu_##op(void *ptr,		\
			unsigned long val, int size)			\
{									\
	unsigned long ret;						\
									\
	switch (size) {							\
	case 4:								\
		__asm__ __volatile__(					\
		"am"#asm_op".w"	" %[ret], %[val], %[ptr]	\n"	\
		: [ret] "=&r" (ret), [ptr] "+ZB"(*(u32 *)ptr)		\
		: [val] "r" (val));					\
		break;							\
	case 8:								\
		__asm__ __volatile__(					\
		"am"#asm_op".d" " %[ret], %[val], %[ptr]	\n"	\
		: [ret] "=&r" (ret), [ptr] "+ZB"(*(u64 *)ptr)		\
		: [val] "r" (val));					\
		break;							\
	default:							\
		ret = 0;						\
		BUILD_BUG();						\
	}								\
									\
	return ret c_op val;						\
}

PERCPU_OP(add, add, +)
PERCPU_OP(and, and, &)
PERCPU_OP(or, or, |)
#undef PERCPU_OP

static __always_inline unsigned long __percpu_read(void __percpu *ptr, int size)
{
	unsigned long ret;

	switch (size) {
	case 1:
		__asm__ __volatile__ ("ldx.b %[ret], $r21, %[ptr]	\n"
		: [ret] "=&r"(ret)
		: [ptr] "r"(ptr)
		: "memory");
		break;
	case 2:
		__asm__ __volatile__ ("ldx.h %[ret], $r21, %[ptr]	\n"
		: [ret] "=&r"(ret)
		: [ptr] "r"(ptr)
		: "memory");
		break;
	case 4:
		__asm__ __volatile__ ("ldx.w %[ret], $r21, %[ptr]	\n"
		: [ret] "=&r"(ret)
		: [ptr] "r"(ptr)
		: "memory");
		break;
	case 8:
		__asm__ __volatile__ ("ldx.d %[ret], $r21, %[ptr]	\n"
		: [ret] "=&r"(ret)
		: [ptr] "r"(ptr)
		: "memory");
		break;
	default:
		ret = 0;
		BUILD_BUG();
	}

	return ret;
}

static __always_inline void __percpu_write(void __percpu *ptr, unsigned long val, int size)
{
	switch (size) {
	case 1:
		__asm__ __volatile__("stx.b %[val], $r21, %[ptr]	\n"
		:
		: [val] "r" (val), [ptr] "r" (ptr)
		: "memory");
		break;
	case 2:
		__asm__ __volatile__("stx.h %[val], $r21, %[ptr]	\n"
		:
		: [val] "r" (val), [ptr] "r" (ptr)
		: "memory");
		break;
	case 4:
		__asm__ __volatile__("stx.w %[val], $r21, %[ptr]	\n"
		:
		: [val] "r" (val), [ptr] "r" (ptr)
		: "memory");
		break;
	case 8:
		__asm__ __volatile__("stx.d %[val], $r21, %[ptr]	\n"
		:
		: [val] "r" (val), [ptr] "r" (ptr)
		: "memory");
		break;
	default:
		BUILD_BUG();
	}
}

static __always_inline unsigned long __percpu_xchg(void *ptr, unsigned long val, int size)
{
	switch (size) {
	case 1:
	case 2:
		return __xchg_small((volatile void *)ptr, val, size);

	case 4:
		return __xchg_asm("amswap.w", (volatile u32 *)ptr, (u32)val);

	case 8:
		return __xchg_asm("amswap.d", (volatile u64 *)ptr, (u64)val);

	default:
		BUILD_BUG();
	}

	return 0;
}

/* this_cpu_cmpxchg */
#define _protect_cmpxchg_local(pcp, o, n)			\
({								\
	typeof(*raw_cpu_ptr(&(pcp))) __ret;			\
	preempt_disable_notrace();				\
	__ret = cmpxchg_local(raw_cpu_ptr(&(pcp)), o, n);	\
	preempt_enable_notrace();				\
	__ret;							\
})

#define _percpu_read(pcp)						\
({									\
	typeof(pcp) __retval;						\
	__retval = (typeof(pcp))__percpu_read(&(pcp), sizeof(pcp));	\
	__retval;							\
})

#define _percpu_write(pcp, val)						\
do {									\
	__percpu_write(&(pcp), (unsigned long)(val), sizeof(pcp));	\
} while (0)								\

#define _pcp_protect(operation, pcp, val)			\
({								\
	typeof(pcp) __retval;					\
	preempt_disable_notrace();				\
	__retval = (typeof(pcp))operation(raw_cpu_ptr(&(pcp)),	\
					  (val), sizeof(pcp));	\
	preempt_enable_notrace();				\
	__retval;						\
})

#define _percpu_add(pcp, val) \
	_pcp_protect(__percpu_add, pcp, val)

#define _percpu_add_return(pcp, val) _percpu_add(pcp, val)

#define _percpu_and(pcp, val) \
	_pcp_protect(__percpu_and, pcp, val)

#define _percpu_or(pcp, val) \
	_pcp_protect(__percpu_or, pcp, val)

#define _percpu_xchg(pcp, val) ((typeof(pcp)) \
	_pcp_protect(__percpu_xchg, pcp, (unsigned long)(val)))

#define this_cpu_add_4(pcp, val) _percpu_add(pcp, val)
#define this_cpu_add_8(pcp, val) _percpu_add(pcp, val)

#define this_cpu_add_return_4(pcp, val) _percpu_add_return(pcp, val)
#define this_cpu_add_return_8(pcp, val) _percpu_add_return(pcp, val)

#define this_cpu_and_4(pcp, val) _percpu_and(pcp, val)
#define this_cpu_and_8(pcp, val) _percpu_and(pcp, val)

#define this_cpu_or_4(pcp, val) _percpu_or(pcp, val)
#define this_cpu_or_8(pcp, val) _percpu_or(pcp, val)

#define this_cpu_read_1(pcp) _percpu_read(pcp)
#define this_cpu_read_2(pcp) _percpu_read(pcp)
#define this_cpu_read_4(pcp) _percpu_read(pcp)
#define this_cpu_read_8(pcp) _percpu_read(pcp)

#define this_cpu_write_1(pcp, val) _percpu_write(pcp, val)
#define this_cpu_write_2(pcp, val) _percpu_write(pcp, val)
#define this_cpu_write_4(pcp, val) _percpu_write(pcp, val)
#define this_cpu_write_8(pcp, val) _percpu_write(pcp, val)

#define this_cpu_xchg_1(pcp, val) _percpu_xchg(pcp, val)
#define this_cpu_xchg_2(pcp, val) _percpu_xchg(pcp, val)
#define this_cpu_xchg_4(pcp, val) _percpu_xchg(pcp, val)
#define this_cpu_xchg_8(pcp, val) _percpu_xchg(pcp, val)

#define this_cpu_cmpxchg_1(ptr, o, n) _protect_cmpxchg_local(ptr, o, n)
#define this_cpu_cmpxchg_2(ptr, o, n) _protect_cmpxchg_local(ptr, o, n)
#define this_cpu_cmpxchg_4(ptr, o, n) _protect_cmpxchg_local(ptr, o, n)
#define this_cpu_cmpxchg_8(ptr, o, n) _protect_cmpxchg_local(ptr, o, n)

#include <asm-generic/percpu.h>

#endif /* __ASM_PERCPU_H */