s390: add support for transactional memory
Allow user-space processes to use transactional execution (TX).
If the TX facility is available user space programs can use
transactions for fine-grained serialization based on the data
objects that are referenced during a transaction. This is
useful for lockless data structures and speculative compiler
optimizations.
Signed-off-by: Martin Schwidefsky <schwidefsky@de.ibm.com>
diff --git a/arch/s390/include/asm/elf.h b/arch/s390/include/asm/elf.h
index 9b94a16..db57594 100644
--- a/arch/s390/include/asm/elf.h
+++ b/arch/s390/include/asm/elf.h
@@ -101,6 +101,7 @@
#define HWCAP_S390_HPAGE 128
#define HWCAP_S390_ETF3EH 256
#define HWCAP_S390_HIGH_GPRS 512
+#define HWCAP_S390_TE 1024
/*
* These are used to set parameters in the core dumps.
diff --git a/arch/s390/include/asm/lowcore.h b/arch/s390/include/asm/lowcore.h
index aab5555..bbf8141 100644
--- a/arch/s390/include/asm/lowcore.h
+++ b/arch/s390/include/asm/lowcore.h
@@ -329,9 +329,13 @@
__u8 pad_0x1338[0x1340-0x1338]; /* 0x1338 */
__u32 access_regs_save_area[16]; /* 0x1340 */
__u64 cregs_save_area[16]; /* 0x1380 */
+ __u8 pad_0x1400[0x1800-0x1400]; /* 0x1400 */
+
+ /* Transaction abort diagnostic block */
+ __u8 pgm_tdb[256]; /* 0x1800 */
/* align to the top of the prefix area */
- __u8 pad_0x1400[0x2000-0x1400]; /* 0x1400 */
+ __u8 pad_0x1900[0x2000-0x1900]; /* 0x1900 */
} __packed;
#endif /* CONFIG_32BIT */
diff --git a/arch/s390/include/asm/processor.h b/arch/s390/include/asm/processor.h
index 0fff583..46fe1fbf 100644
--- a/arch/s390/include/asm/processor.h
+++ b/arch/s390/include/asm/processor.h
@@ -76,14 +76,20 @@
unsigned long gmap_addr; /* address of last gmap fault. */
struct per_regs per_user; /* User specified PER registers */
struct per_event per_event; /* Cause of the last PER trap */
+ unsigned long per_flags; /* Flags to control debug behavior */
/* pfault_wait is used to block the process on a pfault event */
unsigned long pfault_wait;
struct list_head list;
/* cpu runtime instrumentation */
struct runtime_instr_cb *ri_cb;
int ri_signum;
+#ifdef CONFIG_64BIT
+ unsigned char trap_tdb[256]; /* Transaction abort diagnose block */
+#endif
};
+#define PER_FLAG_NO_TE 1UL /* Flag to disable transactions. */
+
typedef struct thread_struct thread_struct;
/*
diff --git a/arch/s390/include/asm/ptrace.h b/arch/s390/include/asm/ptrace.h
index 5c32bae..ce20a53 100644
--- a/arch/s390/include/asm/ptrace.h
+++ b/arch/s390/include/asm/ptrace.h
@@ -361,17 +361,19 @@
unsigned char access_id; /* PER trap access identification */
};
-#define PER_EVENT_MASK 0xE9000000UL
+#define PER_EVENT_MASK 0xEB000000UL
#define PER_EVENT_BRANCH 0x80000000UL
#define PER_EVENT_IFETCH 0x40000000UL
#define PER_EVENT_STORE 0x20000000UL
#define PER_EVENT_STORE_REAL 0x08000000UL
+#define PER_EVENT_TRANSACTION_END 0x02000000UL
#define PER_EVENT_NULLIFICATION 0x01000000UL
-#define PER_CONTROL_MASK 0x00a00000UL
+#define PER_CONTROL_MASK 0x00e00000UL
#define PER_CONTROL_BRANCH_ADDRESS 0x00800000UL
+#define PER_CONTROL_SUSPENSION 0x00400000UL
#define PER_CONTROL_ALTERATION 0x00200000UL
#endif
@@ -485,6 +487,8 @@
#define PTRACE_GET_LAST_BREAK 0x5006
#define PTRACE_PEEK_SYSTEM_CALL 0x5007
#define PTRACE_POKE_SYSTEM_CALL 0x5008
+#define PTRACE_ENABLE_TE 0x5009
+#define PTRACE_DISABLE_TE 0x5010
/*
* PT_PROT definition is loosely based on hppa bsd definition in
diff --git a/arch/s390/include/asm/setup.h b/arch/s390/include/asm/setup.h
index e6859d1..908f688 100644
--- a/arch/s390/include/asm/setup.h
+++ b/arch/s390/include/asm/setup.h
@@ -80,6 +80,7 @@
#define MACHINE_FLAG_LPAR (1UL << 12)
#define MACHINE_FLAG_SPP (1UL << 13)
#define MACHINE_FLAG_TOPOLOGY (1UL << 14)
+#define MACHINE_FLAG_TE (1UL << 15)
#define MACHINE_IS_VM (S390_lowcore.machine_flags & MACHINE_FLAG_VM)
#define MACHINE_IS_KVM (S390_lowcore.machine_flags & MACHINE_FLAG_KVM)
@@ -98,6 +99,7 @@
#define MACHINE_HAS_PFMF (0)
#define MACHINE_HAS_SPP (0)
#define MACHINE_HAS_TOPOLOGY (0)
+#define MACHINE_HAS_TE (0)
#else /* CONFIG_64BIT */
#define MACHINE_HAS_IEEE (1)
#define MACHINE_HAS_CSP (1)
@@ -109,6 +111,7 @@
#define MACHINE_HAS_PFMF (S390_lowcore.machine_flags & MACHINE_FLAG_PFMF)
#define MACHINE_HAS_SPP (S390_lowcore.machine_flags & MACHINE_FLAG_SPP)
#define MACHINE_HAS_TOPOLOGY (S390_lowcore.machine_flags & MACHINE_FLAG_TOPOLOGY)
+#define MACHINE_HAS_TE (S390_lowcore.machine_flags & MACHINE_FLAG_TE)
#endif /* CONFIG_64BIT */
#define ZFCPDUMP_HSA_SIZE (32UL<<20)
diff --git a/arch/s390/kernel/asm-offsets.c b/arch/s390/kernel/asm-offsets.c
index 45ef1a7..fface87 100644
--- a/arch/s390/kernel/asm-offsets.c
+++ b/arch/s390/kernel/asm-offsets.c
@@ -157,6 +157,8 @@
DEFINE(__LC_LAST_BREAK, offsetof(struct _lowcore, breaking_event_addr));
DEFINE(__LC_VDSO_PER_CPU, offsetof(struct _lowcore, vdso_per_cpu_data));
DEFINE(__LC_GMAP, offsetof(struct _lowcore, gmap));
+ DEFINE(__LC_PGM_TDB, offsetof(struct _lowcore, pgm_tdb));
+ DEFINE(__THREAD_trap_tdb, offsetof(struct task_struct, thread.trap_tdb));
DEFINE(__GMAP_ASCE, offsetof(struct gmap, asce));
#endif /* CONFIG_32BIT */
return 0;
diff --git a/arch/s390/kernel/dis.c b/arch/s390/kernel/dis.c
index 4bc67db..7b6ad27 100644
--- a/arch/s390/kernel/dis.c
+++ b/arch/s390/kernel/dis.c
@@ -317,6 +317,9 @@
LONG_INSN_RISBLG,
LONG_INSN_RINEXT,
LONG_INSN_RIEMIT,
+ LONG_INSN_TABORT,
+ LONG_INSN_TBEGIN,
+ LONG_INSN_TBEGINC,
};
static char *long_insn_name[] = {
@@ -334,6 +337,9 @@
[LONG_INSN_RISBLG] = "risblk",
[LONG_INSN_RINEXT] = "rinext",
[LONG_INSN_RIEMIT] = "riemit",
+ [LONG_INSN_TABORT] = "tabort",
+ [LONG_INSN_TBEGIN] = "tbegin",
+ [LONG_INSN_TBEGINC] = "tbeginc",
};
static struct insn opcode[] = {
@@ -609,6 +615,9 @@
{ "lpswe", 0xb2, INSTR_S_RD },
{ "srnmt", 0xb9, INSTR_S_RD },
{ "lfas", 0xbd, INSTR_S_RD },
+ { "etndg", 0xec, INSTR_RRE_R0 },
+ { { 0, LONG_INSN_TABORT }, 0xfc, INSTR_S_RD },
+ { "tend", 0xf8, INSTR_S_RD },
#endif
{ "stidp", 0x02, INSTR_S_RD },
{ "sck", 0x04, INSTR_S_RD },
@@ -1165,6 +1174,7 @@
{ "stfh", 0xcb, INSTR_RXY_RRRD },
{ "chf", 0xcd, INSTR_RXY_RRRD },
{ "clhf", 0xcf, INSTR_RXY_RRRD },
+ { "ntstg", 0x25, INSTR_RXY_RRRD },
#endif
{ "lrv", 0x1e, INSTR_RXY_RRRD },
{ "lrvh", 0x1f, INSTR_RXY_RRRD },
@@ -1188,6 +1198,8 @@
{ "mvhhi", 0x44, INSTR_SIL_RDI },
{ "mvhi", 0x4c, INSTR_SIL_RDI },
{ "mvghi", 0x48, INSTR_SIL_RDI },
+ { { 0, LONG_INSN_TBEGIN }, 0x60, INSTR_SIL_RDU },
+ { { 0, LONG_INSN_TBEGINC }, 0x61, INSTR_SIL_RDU },
#endif
{ "lasp", 0x00, INSTR_SSE_RDRD },
{ "tprot", 0x01, INSTR_SSE_RDRD },
diff --git a/arch/s390/kernel/early.c b/arch/s390/kernel/early.c
index e484079..1345ba4 100644
--- a/arch/s390/kernel/early.c
+++ b/arch/s390/kernel/early.c
@@ -370,6 +370,8 @@
S390_lowcore.machine_flags |= MACHINE_FLAG_MVCOS;
if (test_facility(40))
S390_lowcore.machine_flags |= MACHINE_FLAG_SPP;
+ if (test_facility(50) && test_facility(73))
+ S390_lowcore.machine_flags |= MACHINE_FLAG_TE;
#endif
}
@@ -439,7 +441,6 @@
append_to_cmdline(append_ipl_scpdata);
}
-
/*
* Save ipl parameters, clear bss memory, initialize storage keys
* and create a kernel NSS at startup if the SAVESYS= parm is defined
diff --git a/arch/s390/kernel/entry64.S b/arch/s390/kernel/entry64.S
index 349b7ee..95e9d93 100644
--- a/arch/s390/kernel/entry64.S
+++ b/arch/s390/kernel/entry64.S
@@ -412,6 +412,11 @@
1: UPDATE_VTIME %r14,__LC_SYNC_ENTER_TIMER
LAST_BREAK %r14
lg %r15,__LC_KERNEL_STACK
+ lg %r14,__TI_task(%r12)
+ lghi %r13,__LC_PGM_TDB
+ tm __LC_PGM_ILC+2,0x02 # check for transaction abort
+ jz 2f
+ mvc __THREAD_trap_tdb(256,%r14),0(%r13)
2: aghi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
la %r11,STACK_FRAME_OVERHEAD(%r15)
stmg %r0,%r7,__PT_R0(%r11)
@@ -422,13 +427,12 @@
stg %r10,__PT_ARGS(%r11)
tm __LC_PGM_ILC+3,0x80 # check for per exception
jz 0f
- lg %r1,__TI_task(%r12)
tmhh %r8,0x0001 # kernel per event ?
jz pgm_kprobe
oi __TI_flags+7(%r12),_TIF_PER_TRAP
- mvc __THREAD_per_address(8,%r1),__LC_PER_ADDRESS
- mvc __THREAD_per_cause(2,%r1),__LC_PER_CAUSE
- mvc __THREAD_per_paid(1,%r1),__LC_PER_PAID
+ mvc __THREAD_per_address(8,%r14),__LC_PER_ADDRESS
+ mvc __THREAD_per_cause(2,%r14),__LC_PER_CAUSE
+ mvc __THREAD_per_paid(1,%r14),__LC_PER_PAID
0: REENABLE_IRQS
xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
larl %r1,pgm_check_table
diff --git a/arch/s390/kernel/processor.c b/arch/s390/kernel/processor.c
index 572d4c9..22bf8f0 100644
--- a/arch/s390/kernel/processor.c
+++ b/arch/s390/kernel/processor.c
@@ -39,9 +39,9 @@
*/
static int show_cpuinfo(struct seq_file *m, void *v)
{
- static const char *hwcap_str[10] = {
+ static const char *hwcap_str[11] = {
"esan3", "zarch", "stfle", "msa", "ldisp", "eimm", "dfp",
- "edat", "etf3eh", "highgprs"
+ "edat", "etf3eh", "highgprs", "te"
};
unsigned long n = (unsigned long) v - 1;
int i;
@@ -54,7 +54,7 @@
num_online_cpus(), loops_per_jiffy/(500000/HZ),
(loops_per_jiffy/(5000/HZ))%100);
seq_puts(m, "features\t: ");
- for (i = 0; i < 10; i++)
+ for (i = 0; i < 11; i++)
if (hwcap_str[i] && (elf_hwcap & (1UL << i)))
seq_printf(m, "%s ", hwcap_str[i]);
seq_puts(m, "\n");
diff --git a/arch/s390/kernel/ptrace.c b/arch/s390/kernel/ptrace.c
index e4be113..b817cc5 100644
--- a/arch/s390/kernel/ptrace.c
+++ b/arch/s390/kernel/ptrace.c
@@ -42,6 +42,7 @@
REGSET_GENERAL,
REGSET_FP,
REGSET_LAST_BREAK,
+ REGSET_TDB,
REGSET_SYSTEM_CALL,
REGSET_GENERAL_EXTENDED,
};
@@ -52,6 +53,21 @@
struct thread_struct *thread = &task->thread;
struct per_regs old, new;
+ /* Take care of the enable/disable of transactional execution. */
+ if (MACHINE_HAS_TE) {
+ unsigned long cr0, cr0_new;
+
+ __ctl_store(cr0, 0, 0);
+ /* set or clear transaction execution bits 8 and 9. */
+ if (task->thread.per_flags & PER_FLAG_NO_TE)
+ cr0_new = cr0 & ~(3UL << 54);
+ else
+ cr0_new = cr0 | (3UL << 54);
+ /* Only load control register 0 if necessary. */
+ if (cr0 != cr0_new)
+ __ctl_load(cr0_new, 0, 0);
+ }
+
/* Copy user specified PER registers */
new.control = thread->per_user.control;
new.start = thread->per_user.start;
@@ -60,6 +76,10 @@
/* merge TIF_SINGLE_STEP into user specified PER registers. */
if (test_tsk_thread_flag(task, TIF_SINGLE_STEP)) {
new.control |= PER_EVENT_IFETCH;
+#ifdef CONFIG_64BIT
+ new.control |= PER_CONTROL_SUSPENSION;
+ new.control |= PER_EVENT_TRANSACTION_END;
+#endif
new.start = 0;
new.end = PSW_ADDR_INSN;
}
@@ -100,6 +120,7 @@
memset(&task->thread.per_event, 0, sizeof(task->thread.per_event));
clear_tsk_thread_flag(task, TIF_SINGLE_STEP);
clear_tsk_thread_flag(task, TIF_PER_TRAP);
+ task->thread.per_flags = 0;
}
#ifndef CONFIG_64BIT
@@ -416,6 +437,16 @@
put_user(task_thread_info(child)->last_break,
(unsigned long __user *) data);
return 0;
+ case PTRACE_ENABLE_TE:
+ if (!MACHINE_HAS_TE)
+ return -EIO;
+ child->thread.per_flags &= ~PER_FLAG_NO_TE;
+ return 0;
+ case PTRACE_DISABLE_TE:
+ if (!MACHINE_HAS_TE)
+ return -EIO;
+ child->thread.per_flags |= PER_FLAG_NO_TE;
+ return 0;
default:
/* Removing high order bit from addr (only for 31 bit). */
addr &= PSW_ADDR_INSN;
@@ -903,6 +934,28 @@
return 0;
}
+static int s390_tdb_get(struct task_struct *target,
+ const struct user_regset *regset,
+ unsigned int pos, unsigned int count,
+ void *kbuf, void __user *ubuf)
+{
+ struct pt_regs *regs = task_pt_regs(target);
+ unsigned char *data;
+
+ if (!(regs->int_code & 0x200))
+ return -ENODATA;
+ data = target->thread.trap_tdb;
+ return user_regset_copyout(&pos, &count, &kbuf, &ubuf, data, 0, 256);
+}
+
+static int s390_tdb_set(struct task_struct *target,
+ const struct user_regset *regset,
+ unsigned int pos, unsigned int count,
+ const void *kbuf, const void __user *ubuf)
+{
+ return 0;
+}
+
#endif
static int s390_system_call_get(struct task_struct *target,
@@ -951,6 +1004,14 @@
.get = s390_last_break_get,
.set = s390_last_break_set,
},
+ [REGSET_TDB] = {
+ .core_note_type = NT_S390_TDB,
+ .n = 1,
+ .size = 256,
+ .align = 1,
+ .get = s390_tdb_get,
+ .set = s390_tdb_set,
+ },
#endif
[REGSET_SYSTEM_CALL] = {
.core_note_type = NT_S390_SYSTEM_CALL,
@@ -1148,6 +1209,14 @@
.get = s390_compat_last_break_get,
.set = s390_compat_last_break_set,
},
+ [REGSET_TDB] = {
+ .core_note_type = NT_S390_TDB,
+ .n = 1,
+ .size = 256,
+ .align = 1,
+ .get = s390_tdb_get,
+ .set = s390_tdb_set,
+ },
[REGSET_SYSTEM_CALL] = {
.core_note_type = NT_S390_SYSTEM_CALL,
.n = 1,
diff --git a/arch/s390/kernel/setup.c b/arch/s390/kernel/setup.c
index 40b5769..7b59fff 100644
--- a/arch/s390/kernel/setup.c
+++ b/arch/s390/kernel/setup.c
@@ -980,6 +980,12 @@
* HWCAP_S390_HIGH_GPRS is bit 9.
*/
elf_hwcap |= HWCAP_S390_HIGH_GPRS;
+
+ /*
+ * Transactional execution support HWCAP_S390_TE is bit 10.
+ */
+ if (test_facility(50) && test_facility(73))
+ elf_hwcap |= HWCAP_S390_TE;
#endif
get_cpu_id(&cpu_id);
diff --git a/arch/s390/kernel/traps.c b/arch/s390/kernel/traps.c
index 01775c0..29af526 100644
--- a/arch/s390/kernel/traps.c
+++ b/arch/s390/kernel/traps.c
@@ -57,6 +57,23 @@
static int kstack_depth_to_print = 20;
#endif /* CONFIG_64BIT */
+static inline void __user *get_trap_ip(struct pt_regs *regs)
+{
+#ifdef CONFIG_64BIT
+ unsigned long address;
+
+ if (regs->int_code & 0x200)
+ address = *(unsigned long *)(current->thread.trap_tdb + 24);
+ else
+ address = regs->psw.addr;
+ return (void __user *)
+ ((address - (regs->int_code >> 16)) & PSW_ADDR_INSN);
+#else
+ return (void __user *)
+ ((regs->psw.addr - (regs->int_code >> 16)) & PSW_ADDR_INSN);
+#endif
+}
+
/*
* For show_trace we have tree different stack to consider:
* - the panic stack which is used if the kernel stack has overflown
@@ -285,12 +302,6 @@
return 1;
}
-static inline void __user *get_psw_address(struct pt_regs *regs)
-{
- return (void __user *)
- ((regs->psw.addr - (regs->int_code >> 16)) & PSW_ADDR_INSN);
-}
-
static void __kprobes do_trap(struct pt_regs *regs,
int si_signo, int si_code, char *str)
{
@@ -304,7 +315,7 @@
info.si_signo = si_signo;
info.si_errno = 0;
info.si_code = si_code;
- info.si_addr = get_psw_address(regs);
+ info.si_addr = get_trap_ip(regs);
force_sig_info(si_signo, &info, current);
report_user_fault(regs, si_signo);
} else {
@@ -381,6 +392,11 @@
DO_ERROR_INFO(translation_exception, SIGILL, ILL_ILLOPN,
"translation exception")
+#ifdef CONFIG_64BIT
+DO_ERROR_INFO(transaction_exception, SIGILL, ILL_ILLOPN,
+ "transaction constraint exception")
+#endif
+
static inline void do_fp_trap(struct pt_regs *regs, int fpc)
{
int si_code = 0;
@@ -408,7 +424,7 @@
__u16 __user *location;
int signal = 0;
- location = get_psw_address(regs);
+ location = get_trap_ip(regs);
if (user_mode(regs)) {
if (get_user(*((__u16 *) opcode), (__u16 __user *) location))
@@ -476,7 +492,7 @@
__u16 __user *location = NULL;
int signal = 0;
- location = (__u16 __user *) get_psw_address(regs);
+ location = (__u16 __user *) get_trap_ip(regs);
if (user_mode(regs)) {
get_user(*((__u16 *) opcode), location);
@@ -525,7 +541,7 @@
__u16 __user *location;
int signal = 0;
- location = get_psw_address(regs);
+ location = get_trap_ip(regs);
if (MACHINE_HAS_IEEE)
asm volatile("stfpc %0" : "=m" (current->thread.fp_regs.fpc));
@@ -641,6 +657,7 @@
pgm_check_table[0x12] = &translation_exception;
pgm_check_table[0x13] = &special_op_exception;
#ifdef CONFIG_64BIT
+ pgm_check_table[0x18] = &transaction_exception;
pgm_check_table[0x38] = &do_asce_exception;
pgm_check_table[0x39] = &do_dat_exception;
pgm_check_table[0x3A] = &do_dat_exception;
diff --git a/include/linux/elf.h b/include/linux/elf.h
index 999b4f5..f930b1a 100644
--- a/include/linux/elf.h
+++ b/include/linux/elf.h
@@ -387,6 +387,7 @@
#define NT_S390_PREFIX 0x305 /* s390 prefix register */
#define NT_S390_LAST_BREAK 0x306 /* s390 breaking event address */
#define NT_S390_SYSTEM_CALL 0x307 /* s390 system call restart data */
+#define NT_S390_TDB 0x308 /* s390 transaction diagnostic block */
#define NT_ARM_VFP 0x400 /* ARM VFP/NEON registers */
