powerpc/atomics.c - kvm-unit-tests - Git at Google

 // SPDX-License-Identifier: GPL-2.0-only
 /*
  * Test some powerpc instructions
  *
  * Copyright 2024 Nicholas Piggin, IBM Corp.
  */
 #include <stdint.h>
 #include <libcflat.h>
 #include <migrate.h>
 #include <asm/processor.h>
 #include <asm/time.h>
 #include <asm/atomic.h>
 #include <asm/setup.h>
 #include <asm/barrier.h>
 #include <asm/smp.h>

 static bool do_migrate;
 static bool do_record;

 #define RSV_SIZE 128

 static uint8_t granule[RSV_SIZE] __attribute((__aligned__(RSV_SIZE)));

 static void spin_lock(unsigned int *lock)
 {
 	unsigned int old;

 	asm volatile ("1:"
 		      "lwarx	%0,0,%2;"
 		      "cmpwi	%0,0;"
 		      "bne	1b;"
 		      "stwcx.	%1,0,%2;"
 		      "bne-	1b;"
 		      "lwsync;"
 		      : "=&r"(old) : "r"(1), "r"(lock) : "cr0", "memory");
 }

 static void spin_unlock(unsigned int *lock)
 {
 	asm volatile("lwsync;"
 		     "stw	%1,%0;"
 		     : "+m"(*lock) : "r"(0) : "memory");
 }

 static volatile bool got_interrupt;
 static volatile struct pt_regs recorded_regs;

 static void interrupt_handler(struct pt_regs *regs, void *opaque)
 {
 	assert(!got_interrupt);
 	got_interrupt = true;
 	memcpy((void *)&recorded_regs, regs, sizeof(struct pt_regs));
 	regs_advance_insn(regs);
 }

 static void test_lwarx_stwcx(int argc, char *argv[])
 {
 	unsigned int *var = (unsigned int *)granule;
 	unsigned int old;
 	unsigned int result;

 	*var = 0;
 	asm volatile ("1:"
 		      "lwarx	%0,0,%2;"
 		      "stwcx.	%1,0,%2;"
 		      "bne-	1b;"
 		      : "=&r"(old) : "r"(1), "r"(var) : "cr0", "memory");
 	report(old == 0 && *var == 1, "simple update");

 	*var = 0;
 	asm volatile ("li	%0,0;"
 		      "stwcx.	%1,0,%2;"
 		      "stwcx.	%1,0,%2;"
 		      "bne-	1f;"
 		      "li	%0,1;"
 		      "1:"
 		      : "=&r"(result)
 		      : "r"(1), "r"(var) : "cr0", "memory");
 	report(result == 0 && *var == 0, "failed stwcx. (no reservation)");

 	*var = 0;
 	asm volatile ("li	%0,0;"
 		      "lwarx	%1,0,%4;"
 		      "stw	%3,0(%4);"
 		      "stwcx.	%2,0,%4;"
 		      "bne-	1f;"
 		      "li	%0,1;"
 		      "1:"
 		      : "=&r"(result), "=&r"(old)
 		      : "r"(1), "r"(2), "r"(var) : "cr0", "memory");
 	/* This is implementation specific, so don't fail */
 	if (result == 0 && *var == 2)
 		report(true, "failed stwcx. (intervening store)");
 	else
 		report(true, "succeeded stwcx. (intervening store)");

 	handle_exception(0x600, interrupt_handler, NULL);
 	handle_exception(0x700, interrupt_handler, NULL);

 	/* Implementations may not necessarily invoke the alignment interrupt */
 	old = 10;
 	*var = 0;
 	asm volatile (
 		      "lwarx	%0,0,%1;"
 		      : "+&r"(old) : "r"((char *)var + 1));
 	report(old == 10 && got_interrupt && recorded_regs.trap == 0x600,
 	       "unaligned lwarx causes fault");
 	got_interrupt = false;

 	/*
 	 * Unaligned stwcx. is more difficult to test, at least under QEMU,
 	 * the store does not proceed if there is no matching reservation, so
 	 * the alignment handler does not get invoked. This is okay according
 	 * to the Power ISA (unalignment does not necessarily invoke the
 	 * alignment interrupt). But POWER CPUs do cause alignment interrupt.
 	 */
 	*var = 0;
 	asm volatile (
 		      "lwarx	%0,0,%2;"
 		      "stwcx.	%1,0,%3;"
 		      : "=&r"(old) : "r"(1), "r"(var), "r"((char *)var+1)
 		      : "cr0", "memory");
 	/*
 	 * An unaligned larx/stcx. is not required by the ISA to cause an
 	 * exception, and in TCG the stcx does not though it does on POWER CPUs.
 	 */
 	report_kfail(host_is_tcg, old == 0 && *var == 0 &&
 				  got_interrupt && recorded_regs.trap == 0x600,
 		     "unaligned stwcx. causes fault");
 	got_interrupt = false;

 	handle_exception(0x600, NULL, NULL);

 }

 static void test_lqarx_stqcx(int argc, char *argv[])
 {
 	union {
 		__int128_t var;
 		struct {
 #if  __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
 			unsigned long var1;
 			unsigned long var2;
 #else
 			unsigned long var2;
 			unsigned long var1;
 #endif
 		};
 	} var __attribute__((aligned(16)));
 	register unsigned long new1 asm("r8");
 	register unsigned long new2 asm("r9");
 	register unsigned long old1 asm("r10");
 	register unsigned long old2 asm("r11");
 	unsigned int result;

 	var.var1 = 1;
 	var.var2 = 2;

 	(void)new2;
 	(void)old2;

 	old1 = 0;
 	old2 = 0;
 	new1 = 3;
 	new2 = 4;
 	asm volatile ("1:"
 		      "lqarx	%0,0,%4;"
 		      "stqcx.	%2,0,%4;"
 		      "bne-	1b;"
 		      : "=&r"(old1), "=&r"(old2)
 		      : "r"(new1), "r"(new2), "r"(&var)
 		      : "cr0", "memory");

 	report(old1 == 2 && old2 == 1 && var.var1 == 4 && var.var2 == 3,
 	       "simple update");

 	var.var1 = 1;
 	var.var2 = 2;
 	new1 = 3;
 	new2 = 4;
 	asm volatile ("li	%0,0;"
 		      "stqcx.	%1,0,%3;"
 		      "stqcx.	%1,0,%3;"
 		      "bne-	1f;"
 		      "li	%0,1;"
 		      "1:"
 		      : "=&r"(result)
 		      : "r"(new1), "r"(new2), "r"(&var)
 		      : "cr0", "memory");
 	report(result == 0 && var.var1 == 1 && var.var2 == 2,
 	       "failed stqcx. (no reservation)");

 	var.var1 = 1;
 	var.var2 = 2;
 	new1 = 3;
 	new2 = 4;
 	asm volatile ("li	%0,0;"
 		      "lqarx	%1,0,%6;"
 		      "std	%5,0(%6);"
 		      "stqcx.	%3,0,%6;"
 		      "bne-	1f;"
 		      "li	%0,1;"
 		      "1:"
 		      : "=&r"(result), "=&r"(old1), "=&r"(old2)
 		      : "r"(new1), "r"(new2), "r"(0), "r"(&var)
 		      : "cr0", "memory");
 	/* This is implementation specific, so don't fail */
 	if (result == 0 && (var.var1 == 0 || var.var2 == 0))
 		report(true, "failed stqcx. (intervening store)");
 	else
 		report(true, "succeeded stqcx. (intervening store)");
 }

 static void test_migrate_reserve(int argc, char *argv[])
 {
 	unsigned int *var = (unsigned int *)granule;
 	unsigned int old;
 	int i;
 	int succeed = 0;

 	if (!do_migrate)
 		return;

 	for (i = 0; i < 10; i++) {
 		*var = 0x12345;
 		asm volatile ("lwarx	%0,0,%1" : "=&r"(old) : "r"(var) : "memory");
 		migrate_quiet();
 		asm volatile ("stwcx.	%0,0,%1" : : "r"(0xf00d), "r"(var) : "cr0", "memory");
 		if (*var == 0xf00d)
 			succeed++;
 	}

 	if (do_record) {
 		/*
 		 * Running under TCG record-replay, reservations must not
 		 * be lost by migration
 		 */
 		report(succeed > 0, "migrated reservation is not lost");
 	} else {
 		report(succeed == 0, "migrated reservation is lost");
 	}

 	report_prefix_pop();
 }

 #define ITERS 10000000
 static int test_counter = 0;
 static void test_inc_perf(int argc, char *argv[])
 {
 	int i;
 	uint64_t tb1, tb2;

 	tb1 = get_tb();
 	for (i = 0; i < ITERS; i++)
 		__atomic_fetch_add(&test_counter, 1, __ATOMIC_RELAXED);
 	tb2 = get_tb();
 	report(true, "atomic add takes %ldns",
 		    (tb2 - tb1) * 1000000000 / ITERS / tb_hz);

 	tb1 = get_tb();
 	for (i = 0; i < ITERS; i++)
 		__atomic_fetch_add(&test_counter, 1, __ATOMIC_SEQ_CST);
 	tb2 = get_tb();
 	report(true, "sequentially conssistent atomic add takes %ldns",
 	       (tb2 - tb1) * 1000000000 / ITERS / tb_hz);
 }

 static long smp_inc_counter = 0;
 static int smp_inc_started;

 static void smp_inc_fn(int cpu_id)
 {
 	long i;

 	atomic_fetch_inc(&smp_inc_started);
 	while (smp_inc_started < nr_cpus_present)
 		cpu_relax();

 	for (i = 0; i < ITERS; i++)
 		atomic_fetch_inc(&smp_inc_counter);
 	atomic_fetch_dec(&smp_inc_started);
 }

 static void test_smp_inc(int argc, char **argv)
 {
 	if (nr_cpus_present < 2)
 		return;

 	if (!start_all_cpus(smp_inc_fn))
 		report_abort("Failed to start secondary cpus");

 	while (smp_inc_started < nr_cpus_present - 1)
 		cpu_relax();
 	smp_inc_fn(smp_processor_id());
 	while (smp_inc_started > 0)
 		cpu_relax();

 	stop_all_cpus();

 	report(smp_inc_counter == nr_cpus_present * ITERS,
 	       "counter lost no increments");
 }

 static long smp_lock_counter __attribute__((aligned(128))) = 0;
 static unsigned int smp_lock __attribute__((aligned(128)));
 static int smp_lock_started;

 static void smp_lock_fn(int cpu_id)
 {
 	long i;

 	atomic_fetch_inc(&smp_lock_started);
 	while (smp_lock_started < nr_cpus_present)
 		cpu_relax();

 	for (i = 0; i < ITERS; i++) {
 		spin_lock(&smp_lock);
 		smp_lock_counter++;
 		spin_unlock(&smp_lock);
 	}
 	atomic_fetch_dec(&smp_lock_started);
 }

 static void test_smp_lock(int argc, char **argv)
 {
 	if (nr_cpus_present < 2)
 		return;

 	if (!start_all_cpus(smp_lock_fn))
 		report_abort("Failed to start secondary cpus");

 	while (smp_lock_started < nr_cpus_present - 1)
 		cpu_relax();
 	smp_lock_fn(smp_processor_id());
 	while (smp_lock_started > 0)
 		cpu_relax();

 	stop_all_cpus();

 	report(smp_lock_counter == nr_cpus_present * ITERS,
 	       "counter lost no increments");
 }

 struct {
 	const char *name;
 	void (*func)(int argc, char **argv);
 } hctests[] = {
 	{ "lwarx/stwcx", test_lwarx_stwcx },
 	{ "lqarx/stqcx", test_lqarx_stqcx },
 	{ "migration", test_migrate_reserve },
 	{ "performance", test_inc_perf },
 	{ "SMP-atomic", test_smp_inc },
 	{ "SMP-lock", test_smp_lock },
 	{ NULL, NULL }
 };

 int main(int argc, char **argv)
 {
 	int i;
 	int all;

 	all = argc == 1 || !strcmp(argv[1], "all");

 	for (i = 1; i < argc; i++) {
 		if (strcmp(argv[i], "-r") == 0) {
 			do_record = true;
 		}
 		if (strcmp(argv[i], "-m") == 0) {
 			do_migrate = true;
 		}
 	}

 	report_prefix_push("atomics");

 	for (i = 0; hctests[i].name != NULL; i++) {
 		if (all || strcmp(argv[1], hctests[i].name) == 0) {
 			report_prefix_push(hctests[i].name);
 			hctests[i].func(argc, argv);
 			report_prefix_pop();
 		}
 	}

 	report_prefix_pop();

 	return report_summary();
 }
	// SPDX-License-Identifier: GPL-2.0-only
	/*
	* Test some powerpc instructions
	*
	* Copyright 2024 Nicholas Piggin, IBM Corp.
	*/
	#include <stdint.h>
	#include <libcflat.h>
	#include <migrate.h>
	#include <asm/processor.h>
	#include <asm/time.h>
	#include <asm/atomic.h>
	#include <asm/setup.h>
	#include <asm/barrier.h>
	#include <asm/smp.h>

	static bool do_migrate;
	static bool do_record;

	#define RSV_SIZE 128

	static uint8_t granule[RSV_SIZE] __attribute((__aligned__(RSV_SIZE)));

	static void spin_lock(unsigned int *lock)
	{
	unsigned int old;

	asm volatile ("1:"
	"lwarx %0,0,%2;"
	"cmpwi %0,0;"
	"bne 1b;"
	"stwcx. %1,0,%2;"
	"bne- 1b;"
	"lwsync;"
	: "=&r"(old) : "r"(1), "r"(lock) : "cr0", "memory");
	}

	static void spin_unlock(unsigned int *lock)
	{
	asm volatile("lwsync;"
	"stw %1,%0;"
	: "+m"(*lock) : "r"(0) : "memory");
	}

	static volatile bool got_interrupt;
	static volatile struct pt_regs recorded_regs;

	static void interrupt_handler(struct pt_regs regs, void opaque)
	{
	assert(!got_interrupt);
	got_interrupt = true;
	memcpy((void *)&recorded_regs, regs, sizeof(struct pt_regs));
	regs_advance_insn(regs);
	}

	static void test_lwarx_stwcx(int argc, char *argv[])
	{
	unsigned int var = (unsigned int )granule;
	unsigned int old;
	unsigned int result;

	*var = 0;
	asm volatile ("1:"
	"lwarx %0,0,%2;"
	"stwcx. %1,0,%2;"
	"bne- 1b;"
	: "=&r"(old) : "r"(1), "r"(var) : "cr0", "memory");
	report(old == 0 && *var == 1, "simple update");

	*var = 0;
	asm volatile ("li %0,0;"
	"stwcx. %1,0,%2;"
	"stwcx. %1,0,%2;"
	"bne- 1f;"
	"li %0,1;"
	"1:"
	: "=&r"(result)
	: "r"(1), "r"(var) : "cr0", "memory");
	report(result == 0 && *var == 0, "failed stwcx. (no reservation)");

	*var = 0;
	asm volatile ("li %0,0;"
	"lwarx %1,0,%4;"
	"stw %3,0(%4);"
	"stwcx. %2,0,%4;"
	"bne- 1f;"
	"li %0,1;"
	"1:"
	: "=&r"(result), "=&r"(old)
	: "r"(1), "r"(2), "r"(var) : "cr0", "memory");
	/* This is implementation specific, so don't fail */
	if (result == 0 && *var == 2)
	report(true, "failed stwcx. (intervening store)");
	else
	report(true, "succeeded stwcx. (intervening store)");

	handle_exception(0x600, interrupt_handler, NULL);
	handle_exception(0x700, interrupt_handler, NULL);

	/* Implementations may not necessarily invoke the alignment interrupt */
	old = 10;
	*var = 0;
	asm volatile (
	"lwarx %0,0,%1;"
	: "+&r"(old) : "r"((char *)var + 1));
	report(old == 10 && got_interrupt && recorded_regs.trap == 0x600,
	"unaligned lwarx causes fault");
	got_interrupt = false;

	/*
	* Unaligned stwcx. is more difficult to test, at least under QEMU,
	* the store does not proceed if there is no matching reservation, so
	* the alignment handler does not get invoked. This is okay according
	* to the Power ISA (unalignment does not necessarily invoke the
	* alignment interrupt). But POWER CPUs do cause alignment interrupt.
	*/
	*var = 0;
	asm volatile (
	"lwarx %0,0,%2;"
	"stwcx. %1,0,%3;"
	: "=&r"(old) : "r"(1), "r"(var), "r"((char *)var+1)
	: "cr0", "memory");
	/*
	* An unaligned larx/stcx. is not required by the ISA to cause an
	* exception, and in TCG the stcx does not though it does on POWER CPUs.
	*/
	report_kfail(host_is_tcg, old == 0 && *var == 0 &&
	got_interrupt && recorded_regs.trap == 0x600,
	"unaligned stwcx. causes fault");
	got_interrupt = false;

	handle_exception(0x600, NULL, NULL);

	}

	static void test_lqarx_stqcx(int argc, char *argv[])
	{
	union {
	__int128_t var;
	struct {
	#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
	unsigned long var1;
	unsigned long var2;
	#else
	unsigned long var2;
	unsigned long var1;
	#endif
	};
	} var __attribute__((aligned(16)));
	register unsigned long new1 asm("r8");
	register unsigned long new2 asm("r9");
	register unsigned long old1 asm("r10");
	register unsigned long old2 asm("r11");
	unsigned int result;

	var.var1 = 1;
	var.var2 = 2;

	(void)new2;
	(void)old2;

	old1 = 0;
	old2 = 0;
	new1 = 3;
	new2 = 4;
	asm volatile ("1:"
	"lqarx %0,0,%4;"
	"stqcx. %2,0,%4;"
	"bne- 1b;"
	: "=&r"(old1), "=&r"(old2)
	: "r"(new1), "r"(new2), "r"(&var)
	: "cr0", "memory");

	report(old1 == 2 && old2 == 1 && var.var1 == 4 && var.var2 == 3,
	"simple update");

	var.var1 = 1;
	var.var2 = 2;
	new1 = 3;
	new2 = 4;
	asm volatile ("li %0,0;"
	"stqcx. %1,0,%3;"
	"stqcx. %1,0,%3;"
	"bne- 1f;"
	"li %0,1;"
	"1:"
	: "=&r"(result)
	: "r"(new1), "r"(new2), "r"(&var)
	: "cr0", "memory");
	report(result == 0 && var.var1 == 1 && var.var2 == 2,
	"failed stqcx. (no reservation)");

	var.var1 = 1;
	var.var2 = 2;
	new1 = 3;
	new2 = 4;
	asm volatile ("li %0,0;"
	"lqarx %1,0,%6;"
	"std %5,0(%6);"
	"stqcx. %3,0,%6;"
	"bne- 1f;"
	"li %0,1;"
	"1:"
	: "=&r"(result), "=&r"(old1), "=&r"(old2)
	: "r"(new1), "r"(new2), "r"(0), "r"(&var)
	: "cr0", "memory");
	/* This is implementation specific, so don't fail */
	if (result == 0 && (var.var1 == 0 \|\| var.var2 == 0))
	report(true, "failed stqcx. (intervening store)");
	else
	report(true, "succeeded stqcx. (intervening store)");
	}

	static void test_migrate_reserve(int argc, char *argv[])
	{
	unsigned int var = (unsigned int )granule;
	unsigned int old;
	int i;
	int succeed = 0;

	if (!do_migrate)
	return;

	for (i = 0; i < 10; i++) {
	*var = 0x12345;
	asm volatile ("lwarx %0,0,%1" : "=&r"(old) : "r"(var) : "memory");
	migrate_quiet();
	asm volatile ("stwcx. %0,0,%1" : : "r"(0xf00d), "r"(var) : "cr0", "memory");
	if (*var == 0xf00d)
	succeed++;
	}

	if (do_record) {
	/*
	* Running under TCG record-replay, reservations must not
	* be lost by migration
	*/
	report(succeed > 0, "migrated reservation is not lost");
	} else {
	report(succeed == 0, "migrated reservation is lost");
	}

	report_prefix_pop();
	}

	#define ITERS 10000000
	static int test_counter = 0;
	static void test_inc_perf(int argc, char *argv[])
	{
	int i;
	uint64_t tb1, tb2;

	tb1 = get_tb();
	for (i = 0; i < ITERS; i++)
	__atomic_fetch_add(&test_counter, 1, __ATOMIC_RELAXED);
	tb2 = get_tb();
	report(true, "atomic add takes %ldns",
	(tb2 - tb1) * 1000000000 / ITERS / tb_hz);

	tb1 = get_tb();
	for (i = 0; i < ITERS; i++)
	__atomic_fetch_add(&test_counter, 1, __ATOMIC_SEQ_CST);
	tb2 = get_tb();
	report(true, "sequentially conssistent atomic add takes %ldns",
	(tb2 - tb1) * 1000000000 / ITERS / tb_hz);
	}

	static long smp_inc_counter = 0;
	static int smp_inc_started;

	static void smp_inc_fn(int cpu_id)
	{
	long i;

	atomic_fetch_inc(&smp_inc_started);
	while (smp_inc_started < nr_cpus_present)
	cpu_relax();

	for (i = 0; i < ITERS; i++)
	atomic_fetch_inc(&smp_inc_counter);
	atomic_fetch_dec(&smp_inc_started);
	}

	static void test_smp_inc(int argc, char **argv)
	{
	if (nr_cpus_present < 2)
	return;

	if (!start_all_cpus(smp_inc_fn))
	report_abort("Failed to start secondary cpus");

	while (smp_inc_started < nr_cpus_present - 1)
	cpu_relax();
	smp_inc_fn(smp_processor_id());
	while (smp_inc_started > 0)
	cpu_relax();

	stop_all_cpus();

	report(smp_inc_counter == nr_cpus_present * ITERS,
	"counter lost no increments");
	}

	static long smp_lock_counter __attribute__((aligned(128))) = 0;
	static unsigned int smp_lock __attribute__((aligned(128)));
	static int smp_lock_started;

	static void smp_lock_fn(int cpu_id)
	{
	long i;

	atomic_fetch_inc(&smp_lock_started);
	while (smp_lock_started < nr_cpus_present)
	cpu_relax();

	for (i = 0; i < ITERS; i++) {
	spin_lock(&smp_lock);
	smp_lock_counter++;
	spin_unlock(&smp_lock);
	}
	atomic_fetch_dec(&smp_lock_started);
	}

	static void test_smp_lock(int argc, char **argv)
	{
	if (nr_cpus_present < 2)
	return;

	if (!start_all_cpus(smp_lock_fn))
	report_abort("Failed to start secondary cpus");

	while (smp_lock_started < nr_cpus_present - 1)
	cpu_relax();
	smp_lock_fn(smp_processor_id());
	while (smp_lock_started > 0)
	cpu_relax();

	stop_all_cpus();

	report(smp_lock_counter == nr_cpus_present * ITERS,
	"counter lost no increments");
	}

	struct {
	const char *name;
	void (func)(int argc, char *argv);
	} hctests[] = {
	{ "lwarx/stwcx", test_lwarx_stwcx },
	{ "lqarx/stqcx", test_lqarx_stqcx },
	{ "migration", test_migrate_reserve },
	{ "performance", test_inc_perf },
	{ "SMP-atomic", test_smp_inc },
	{ "SMP-lock", test_smp_lock },
	{ NULL, NULL }
	};

	int main(int argc, char **argv)
	{
	int i;
	int all;

	all = argc == 1 \|\| !strcmp(argv[1], "all");

	for (i = 1; i < argc; i++) {
	if (strcmp(argv[i], "-r") == 0) {
	do_record = true;
	}
	if (strcmp(argv[i], "-m") == 0) {
	do_migrate = true;
	}
	}

	report_prefix_push("atomics");

	for (i = 0; hctests[i].name != NULL; i++) {
	if (all \|\| strcmp(argv[1], hctests[i].name) == 0) {
	report_prefix_push(hctests[i].name);
	hctests[i].func(argc, argv);
	report_prefix_pop();
	}
	}

	report_prefix_pop();

	return report_summary();
	}