tools/testing/selftests/bpf/prog_tests/uprobe_syscall.c

// SPDX-License-Identifier: GPL-2.0

#include <test_progs.h>

#ifdef __x86_64__

#define _GNU_SOURCE
#include <sched.h>
#include <unistd.h>
#include <asm/ptrace.h>
#include <linux/compiler.h>
#include <linux/stringify.h>
#include <linux/kernel.h>
#include <sys/wait.h>
#include <sys/syscall.h>
#include <sys/prctl.h>
#include <asm/prctl.h>
#include <stdnoreturn.h>
#include "uprobe_syscall.skel.h"
#include "uprobe_syscall_executed.skel.h"
#include "bpf/libbpf_internal.h"

#define USDT_NOP .byte 0x66, 0x2e, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00
#include "usdt.h"

#pragma GCC diagnostic ignored "-Wattributes"

__attribute__((aligned(16)))
__nocf_check __weak __naked unsigned long uprobe_regs_trigger(void)
{
	asm volatile (
		".byte 0x66, 0x2e, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00\n" /* nop10 */
		"movq $0xdeadbeef, %rax\n"
		"ret\n"
	);
}

__naked void uprobe_regs(struct pt_regs *before, struct pt_regs *after)
{
	asm volatile (
		"movq %r15,   0(%rdi)\n"
		"movq %r14,   8(%rdi)\n"
		"movq %r13,  16(%rdi)\n"
		"movq %r12,  24(%rdi)\n"
		"movq %rbp,  32(%rdi)\n"
		"movq %rbx,  40(%rdi)\n"
		"movq %r11,  48(%rdi)\n"
		"movq %r10,  56(%rdi)\n"
		"movq  %r9,  64(%rdi)\n"
		"movq  %r8,  72(%rdi)\n"
		"movq %rax,  80(%rdi)\n"
		"movq %rcx,  88(%rdi)\n"
		"movq %rdx,  96(%rdi)\n"
		"movq %rsi, 104(%rdi)\n"
		"movq %rdi, 112(%rdi)\n"
		"movq   $0, 120(%rdi)\n" /* orig_rax */
		"movq   $0, 128(%rdi)\n" /* rip      */
		"movq   $0, 136(%rdi)\n" /* cs       */
		"pushq %rax\n"
		"pushf\n"
		"pop %rax\n"
		"movq %rax, 144(%rdi)\n" /* eflags   */
		"pop %rax\n"
		"movq %rsp, 152(%rdi)\n" /* rsp      */
		"movq   $0, 160(%rdi)\n" /* ss       */

		/* save 2nd argument */
		"pushq %rsi\n"
		"call uprobe_regs_trigger\n"

		/* save  return value and load 2nd argument pointer to rax */
		"pushq %rax\n"
		"movq 8(%rsp), %rax\n"

		"movq %r15,   0(%rax)\n"
		"movq %r14,   8(%rax)\n"
		"movq %r13,  16(%rax)\n"
		"movq %r12,  24(%rax)\n"
		"movq %rbp,  32(%rax)\n"
		"movq %rbx,  40(%rax)\n"
		"movq %r11,  48(%rax)\n"
		"movq %r10,  56(%rax)\n"
		"movq  %r9,  64(%rax)\n"
		"movq  %r8,  72(%rax)\n"
		"movq %rcx,  88(%rax)\n"
		"movq %rdx,  96(%rax)\n"
		"movq %rsi, 104(%rax)\n"
		"movq %rdi, 112(%rax)\n"
		"movq   $0, 120(%rax)\n" /* orig_rax */
		"movq   $0, 128(%rax)\n" /* rip      */
		"movq   $0, 136(%rax)\n" /* cs       */

		/* restore return value and 2nd argument */
		"pop %rax\n"
		"pop %rsi\n"

		"movq %rax,  80(%rsi)\n"

		"pushf\n"
		"pop %rax\n"

		"movq %rax, 144(%rsi)\n" /* eflags   */
		"movq %rsp, 152(%rsi)\n" /* rsp      */
		"movq   $0, 160(%rsi)\n" /* ss       */
		"ret\n"
);
}

static void test_uprobe_regs_equal(bool retprobe)
{
	LIBBPF_OPTS(bpf_uprobe_opts, opts,
		.retprobe = retprobe,
	);
	struct uprobe_syscall *skel = NULL;
	struct pt_regs before = {}, after = {};
	unsigned long *pb = (unsigned long *) &before;
	unsigned long *pa = (unsigned long *) &after;
	unsigned long *pp;
	unsigned long offset;
	unsigned int i, cnt;

	offset = get_uprobe_offset(&uprobe_regs_trigger);
	if (!ASSERT_GE(offset, 0, "get_uprobe_offset"))
		return;

	skel = uprobe_syscall__open_and_load();
	if (!ASSERT_OK_PTR(skel, "uprobe_syscall__open_and_load"))
		goto cleanup;

	skel->links.probe = bpf_program__attach_uprobe_opts(skel->progs.probe,
				0, "/proc/self/exe", offset, &opts);
	if (!ASSERT_OK_PTR(skel->links.probe, "bpf_program__attach_uprobe_opts"))
		goto cleanup;

	/* make sure uprobe gets optimized */
	if (!retprobe)
		uprobe_regs_trigger();

	uprobe_regs(&before, &after);

	pp = (unsigned long *) &skel->bss->regs;
	cnt = sizeof(before)/sizeof(*pb);

	for (i = 0; i < cnt; i++) {
		unsigned int offset = i * sizeof(unsigned long);

		/*
		 * Check register before and after uprobe_regs_trigger call
		 * that triggers the uretprobe.
		 */
		switch (offset) {
		case offsetof(struct pt_regs, rax):
			ASSERT_EQ(pa[i], 0xdeadbeef, "return value");
			break;
		default:
			if (!ASSERT_EQ(pb[i], pa[i], "register before-after value check"))
				fprintf(stdout, "failed register offset %u\n", offset);
		}

		/*
		 * Check register seen from bpf program and register after
		 * uprobe_regs_trigger call (with rax exception, check below).
		 */
		switch (offset) {
		/*
		 * These values will be different (not set in uretprobe_regs),
		 * we don't care.
		 */
		case offsetof(struct pt_regs, orig_rax):
		case offsetof(struct pt_regs, rip):
		case offsetof(struct pt_regs, cs):
		case offsetof(struct pt_regs, rsp):
		case offsetof(struct pt_regs, ss):
			break;
		/*
		 * uprobe does not see return value in rax, it needs to see the
		 * original (before) rax value
		 */
		case offsetof(struct pt_regs, rax):
			if (!retprobe) {
				ASSERT_EQ(pp[i], pb[i], "uprobe rax prog-before value check");
				break;
			}
		default:
			if (!ASSERT_EQ(pp[i], pa[i], "register prog-after value check"))
				fprintf(stdout, "failed register offset %u\n", offset);
		}
	}

cleanup:
	uprobe_syscall__destroy(skel);
}

#define BPF_TESTMOD_UPROBE_TEST_FILE "/sys/kernel/bpf_testmod_uprobe"

static int write_bpf_testmod_uprobe(unsigned long offset)
{
	size_t n, ret;
	char buf[30];
	int fd;

	n = sprintf(buf, "%lu", offset);

	fd = open(BPF_TESTMOD_UPROBE_TEST_FILE, O_WRONLY);
	if (fd < 0)
		return -errno;

	ret = write(fd, buf, n);
	close(fd);
	return ret != n ? (int) ret : 0;
}

static void test_regs_change(void)
{
	struct pt_regs before = {}, after = {};
	unsigned long *pb = (unsigned long *) &before;
	unsigned long *pa = (unsigned long *) &after;
	unsigned long cnt = sizeof(before)/sizeof(*pb);
	unsigned int i, err, offset;

	offset = get_uprobe_offset(uprobe_regs_trigger);

	err = write_bpf_testmod_uprobe(offset);
	if (!ASSERT_OK(err, "register_uprobe"))
		return;

	/* make sure uprobe gets optimized */
	uprobe_regs_trigger();

	uprobe_regs(&before, &after);

	err = write_bpf_testmod_uprobe(0);
	if (!ASSERT_OK(err, "unregister_uprobe"))
		return;

	for (i = 0; i < cnt; i++) {
		unsigned int offset = i * sizeof(unsigned long);

		switch (offset) {
		case offsetof(struct pt_regs, rax):
			ASSERT_EQ(pa[i], 0x12345678deadbeef, "rax");
			break;
		case offsetof(struct pt_regs, rcx):
			ASSERT_EQ(pa[i], 0x87654321feebdaed, "rcx");
			break;
		case offsetof(struct pt_regs, r11):
			ASSERT_EQ(pa[i], (__u64) -1, "r11");
			break;
		default:
			if (!ASSERT_EQ(pa[i], pb[i], "register before-after value check"))
				fprintf(stdout, "failed register offset %u\n", offset);
		}
	}
}

#ifndef __NR_uretprobe
#define __NR_uretprobe 335
#endif

__naked unsigned long uretprobe_syscall_call_1(void)
{
	/*
	 * Pretend we are uretprobe trampoline to trigger the return
	 * probe invocation in order to verify we get SIGILL.
	 */
	asm volatile (
		"pushq %rax\n"
		"pushq %rcx\n"
		"pushq %r11\n"
		"movq $" __stringify(__NR_uretprobe) ", %rax\n"
		"syscall\n"
		"popq %r11\n"
		"popq %rcx\n"
		"retq\n"
	);
}

__naked unsigned long uretprobe_syscall_call(void)
{
	asm volatile (
		"call uretprobe_syscall_call_1\n"
		"retq\n"
	);
}

static void test_uretprobe_syscall_call(void)
{
	LIBBPF_OPTS(bpf_uprobe_multi_opts, opts,
		.retprobe = true,
	);
	struct uprobe_syscall_executed *skel;
	int pid, status, err, go[2], c = 0;
	struct bpf_link *link;

	if (!ASSERT_OK(pipe(go), "pipe"))
		return;

	skel = uprobe_syscall_executed__open_and_load();
	if (!ASSERT_OK_PTR(skel, "uprobe_syscall_executed__open_and_load"))
		goto cleanup;

	pid = fork();
	if (!ASSERT_GE(pid, 0, "fork"))
		goto cleanup;

	/* child */
	if (pid == 0) {
		close(go[1]);

		/* wait for parent's kick */
		err = read(go[0], &c, 1);
		if (err != 1)
			exit(-1);

		uretprobe_syscall_call();
		_exit(0);
	}

	skel->bss->pid = pid;

	link = bpf_program__attach_uprobe_multi(skel->progs.test_uretprobe_multi,
						pid, "/proc/self/exe",
						"uretprobe_syscall_call", &opts);
	if (!ASSERT_OK_PTR(link, "bpf_program__attach_uprobe_multi"))
		goto cleanup;
	skel->links.test_uretprobe_multi = link;

	/* kick the child */
	write(go[1], &c, 1);
	err = waitpid(pid, &status, 0);
	ASSERT_EQ(err, pid, "waitpid");

	/* verify the child got killed with SIGILL */
	ASSERT_EQ(WIFSIGNALED(status), 1, "WIFSIGNALED");
	ASSERT_EQ(WTERMSIG(status), SIGILL, "WTERMSIG");

	/* verify the uretprobe program wasn't called */
	ASSERT_EQ(skel->bss->executed, 0, "executed");

cleanup:
	uprobe_syscall_executed__destroy(skel);
	close(go[1]);
	close(go[0]);
}

#define TRAMP "[uprobes-trampoline]"

__attribute__((aligned(16)))
__nocf_check __weak __naked void uprobe_test(void)
{
	asm volatile (
		".byte 0x66, 0x2e, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00\n" /* nop10 */
		"ret\n"
	);
}

__attribute__((aligned(16)))
__nocf_check __weak void usdt_test(void)
{
	USDT(optimized_uprobe, usdt);
}

/*
 * Assembly-level red zone clobbering test. Stores known values in the
 * red zone (below RSP), executes a nop10 (uprobe site), and checks that
 * the values survived. Returns 0 if intact, 1 if clobbered.
 *
 * The nop5 optimization used CALL (which pushes a return address to
 * [rsp-8]), the value at -8(%rsp) was overwritten. The nop10 optimization
 * should escape that by moving stackpointer below the redzone before
 * doing the CALL.
 *
 * Align the code at 64 bytes, to make sure nop10 is not on page boundary.
 */
__attribute__((aligned(64)))
__nocf_check __weak __naked unsigned long uprobe_red_zone_test(void)
{
	asm volatile (
		"movabs $0x1111111111111111, %%rax\n"
		"movq   %%rax, -8(%%rsp)\n"
		"movabs $0x2222222222222222, %%rax\n"
		"movq   %%rax, -16(%%rsp)\n"
		"movabs $0x3333333333333333, %%rax\n"
		"movq   %%rax, -24(%%rsp)\n"

		".byte 0x66, 0x2e, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00\n" /* nop10: uprobe site */

		"movabs $0x1111111111111111, %%rax\n"
		"cmpq   %%rax, -8(%%rsp)\n"
		"jne    1f\n"
		"movabs $0x2222222222222222, %%rax\n"
		"cmpq   %%rax, -16(%%rsp)\n"
		"jne    1f\n"
		"movabs $0x3333333333333333, %%rax\n"
		"cmpq   %%rax, -24(%%rsp)\n"
		"jne    1f\n"

		"xorl   %%eax, %%eax\n"
		"retq\n"
		"1:\n"
		"movl   $1, %%eax\n"
		"retq\n"
		::: "rax", "memory"
	);
}

static int find_uprobes_trampoline(void *tramp_addr)
{
	void *start, *end;
	char line[128];
	int ret = -1;
	FILE *maps;

	maps = fopen("/proc/self/maps", "r");
	if (!maps) {
		fprintf(stderr, "cannot open maps\n");
		return -1;
	}

	while (fgets(line, sizeof(line), maps)) {
		int m = -1;

		/* We care only about private r-x mappings. */
		if (sscanf(line, "%p-%p r-xp %*x %*x:%*x %*u %n", &start, &end, &m) != 2)
			continue;
		if (m < 0)
			continue;
		if (!strncmp(&line[m], TRAMP, sizeof(TRAMP)-1) && (start == tramp_addr)) {
			ret = 0;
			break;
		}
	}

	fclose(maps);
	return ret;
}

static unsigned char nop10[10]  = { 0x66, 0x2e, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00 };
static unsigned char lea_rsp[5] = { 0x48, 0x8d, 0x64, 0x24, 0x80 };

static void *find_nop10(void *fn)
{
	int i;

	for (i = 0; i < 128; i++) {
		if (!memcmp(nop10, fn + i, 10))
			return fn + i;
	}
	return NULL;
}

typedef void (__attribute__((nocf_check)) *trigger_t)(void);

static void check_attach_notrigger(struct uprobe_syscall_executed *skel,
				   void *addr, int executed)
{
	unsigned char *op = addr;

	/* Make sure bpf program was not executed. */
	ASSERT_EQ(skel->bss->executed, executed, "executed");
	ASSERT_EQ(*op, 0xcc, "int3");
}

static void *check_attach(struct uprobe_syscall_executed *skel, trigger_t trigger,
			  void *addr, int executed)
{
	struct __arch_relative_insn {
		__u8 op;
		__s32 raddr;
	} __packed *call;
	void *tramp = NULL;

	/* Uprobe gets optimized after first trigger, so let's press twice. */
	trigger();
	trigger();

	/* Make sure bpf program got executed.. */
	ASSERT_EQ(skel->bss->executed, executed, "executed");

	/* .. and check the trampoline is as expected. */
	ASSERT_OK(memcmp(addr, lea_rsp, 5), "lea_rsp");
	call = (struct __arch_relative_insn *)(addr + 5);
	tramp = (void *) (call + 1) + call->raddr;
	ASSERT_EQ(call->op, 0xe8, "call");
	ASSERT_OK(find_uprobes_trampoline(tramp), "uprobes_trampoline");

	return tramp;
}

static bool check_detach(void *addr, void *tramp)
{
	static const unsigned char nop10_prefix[] = { 0x66, 0x2e, 0x0f, 0x1f, 0x84 };
	bool ok = true;

	/* [uprobes_trampoline] stays after detach */
	ok &= ASSERT_OK(find_uprobes_trampoline(tramp), "uprobes_trampoline");
	ok &= ASSERT_OK(memcmp(addr, nop10_prefix, 5), "nop10_prefix");
	return ok;
}

static void *check(struct uprobe_syscall_executed *skel, struct bpf_link *link,
		   trigger_t trigger, void *addr, int executed)
{
	void *tramp;

	tramp = check_attach(skel, trigger, addr, executed);
	bpf_link__destroy(link);
	check_detach(addr, tramp);
	return tramp;
}

static void test_uprobe_legacy(void)
{
	struct uprobe_syscall_executed *skel = NULL;
	LIBBPF_OPTS(bpf_uprobe_opts, opts,
		.retprobe = true,
	);
	struct bpf_link *link;
	unsigned long offset;
	void *tramp;

	offset = get_uprobe_offset(&uprobe_test);
	if (!ASSERT_GE(offset, 0, "get_uprobe_offset"))
		goto cleanup;

	/* uprobe */
	skel = uprobe_syscall_executed__open_and_load();
	if (!ASSERT_OK_PTR(skel, "uprobe_syscall_executed__open_and_load"))
		return;

	skel->bss->pid = getpid();

	link = bpf_program__attach_uprobe_opts(skel->progs.test_uprobe,
				0, "/proc/self/exe", offset, NULL);
	if (!ASSERT_OK_PTR(link, "bpf_program__attach_uprobe_opts"))
		goto cleanup;

	tramp = check(skel, link, uprobe_test, uprobe_test, 2);

	/* reattach and detach without triggering optimization */
	link = bpf_program__attach_uprobe_opts(skel->progs.test_uprobe,
					       0, "/proc/self/exe", offset, NULL);
	if (!ASSERT_OK_PTR(link, "bpf_program__attach_uprobe_opts"))
		goto cleanup;

	check_attach_notrigger(skel, uprobe_test, 2);

	bpf_link__destroy(link);
	if (!check_detach(uprobe_test, tramp))
		goto cleanup;

	uprobe_test();
	ASSERT_EQ(skel->bss->executed, 2, "executed_no_probe");

	/* reattach with triggering optimization */
	link = bpf_program__attach_uprobe_opts(skel->progs.test_uprobe,
				0, "/proc/self/exe", offset, NULL);
	if (!ASSERT_OK_PTR(link, "bpf_program__attach_uprobe_opts"))
		goto cleanup;

	check(skel, link, uprobe_test, uprobe_test, 4);

	/* uretprobe */
	skel->bss->executed = 0;

	link = bpf_program__attach_uprobe_opts(skel->progs.test_uretprobe,
				0, "/proc/self/exe", offset, &opts);
	if (!ASSERT_OK_PTR(link, "bpf_program__attach_uprobe_opts"))
		goto cleanup;

	check(skel, link, uprobe_test, uprobe_test, 2);

cleanup:
	uprobe_syscall_executed__destroy(skel);
}

static void test_uprobe_multi(void)
{
	struct uprobe_syscall_executed *skel = NULL;
	LIBBPF_OPTS(bpf_uprobe_multi_opts, opts);
	struct bpf_link *link;
	unsigned long offset;
	void *tramp;

	offset = get_uprobe_offset(&uprobe_test);
	if (!ASSERT_GE(offset, 0, "get_uprobe_offset"))
		goto cleanup;

	opts.offsets = &offset;
	opts.cnt = 1;

	skel = uprobe_syscall_executed__open_and_load();
	if (!ASSERT_OK_PTR(skel, "uprobe_syscall_executed__open_and_load"))
		return;

	skel->bss->pid = getpid();

	/* uprobe.multi */
	link = bpf_program__attach_uprobe_multi(skel->progs.test_uprobe_multi,
				0, "/proc/self/exe", NULL, &opts);
	if (!ASSERT_OK_PTR(link, "bpf_program__attach_uprobe_multi"))
		goto cleanup;

	tramp = check(skel, link, uprobe_test, uprobe_test, 2);

	/* reattach and detach without triggering optimization */
	link = bpf_program__attach_uprobe_multi(skel->progs.test_uprobe_multi,
				0, "/proc/self/exe", NULL, &opts);
	if (!ASSERT_OK_PTR(link, "bpf_program__attach_uprobe_multi"))
		goto cleanup;

	check_attach_notrigger(skel, uprobe_test, 2);

	bpf_link__destroy(link);
	if (!check_detach(uprobe_test, tramp))
		goto cleanup;

	uprobe_test();
	ASSERT_EQ(skel->bss->executed, 2, "executed_no_probe");

	/* reattach with triggering optimization */
	link = bpf_program__attach_uprobe_multi(skel->progs.test_uprobe_multi,
				0, "/proc/self/exe", NULL, &opts);
	if (!ASSERT_OK_PTR(link, "bpf_program__attach_uprobe_multi"))
		goto cleanup;

	check(skel, link, uprobe_test, uprobe_test, 4);

	/* uretprobe.multi */
	skel->bss->executed = 0;
	opts.retprobe = true;
	link = bpf_program__attach_uprobe_multi(skel->progs.test_uretprobe_multi,
				0, "/proc/self/exe", NULL, &opts);
	if (!ASSERT_OK_PTR(link, "bpf_program__attach_uprobe_multi"))
		goto cleanup;

	check(skel, link, uprobe_test, uprobe_test, 2);

cleanup:
	uprobe_syscall_executed__destroy(skel);
}

static void test_uprobe_session(void)
{
	struct uprobe_syscall_executed *skel = NULL;
	LIBBPF_OPTS(bpf_uprobe_multi_opts, opts,
		.session = true,
	);
	struct bpf_link *link;
	unsigned long offset;
	void *tramp;

	offset = get_uprobe_offset(&uprobe_test);
	if (!ASSERT_GE(offset, 0, "get_uprobe_offset"))
		goto cleanup;

	opts.offsets = &offset;
	opts.cnt = 1;

	skel = uprobe_syscall_executed__open_and_load();
	if (!ASSERT_OK_PTR(skel, "uprobe_syscall_executed__open_and_load"))
		return;

	skel->bss->pid = getpid();

	link = bpf_program__attach_uprobe_multi(skel->progs.test_uprobe_session,
				0, "/proc/self/exe", NULL, &opts);
	if (!ASSERT_OK_PTR(link, "bpf_program__attach_uprobe_multi"))
		goto cleanup;

	tramp = check(skel, link, uprobe_test, uprobe_test, 4);

	/* reattach and detach without triggering optimization */
	link = bpf_program__attach_uprobe_multi(skel->progs.test_uprobe_session,
				0, "/proc/self/exe", NULL, &opts);
	if (!ASSERT_OK_PTR(link, "bpf_program__attach_uprobe_multi"))
		goto cleanup;

	check_attach_notrigger(skel, uprobe_test, 4);

	bpf_link__destroy(link);
	if (!check_detach(uprobe_test, tramp))
		goto cleanup;

	uprobe_test();
	ASSERT_EQ(skel->bss->executed, 4, "executed_no_probe");

	/* reattach with triggering optimization */
	link = bpf_program__attach_uprobe_multi(skel->progs.test_uprobe_session,
				0, "/proc/self/exe", NULL, &opts);
	if (!ASSERT_OK_PTR(link, "bpf_program__attach_uprobe_multi"))
		goto cleanup;

	check(skel, link, uprobe_test, uprobe_test, 8);

cleanup:
	uprobe_syscall_executed__destroy(skel);
}

static void test_uprobe_usdt(void)
{
	struct uprobe_syscall_executed *skel;
	struct bpf_link *link;
	void *addr, *tramp;

	errno = 0;
	addr = find_nop10(usdt_test);
	if (!ASSERT_OK_PTR(addr, "find_nop10"))
		return;

	skel = uprobe_syscall_executed__open_and_load();
	if (!ASSERT_OK_PTR(skel, "uprobe_syscall_executed__open_and_load"))
		return;

	skel->bss->pid = getpid();

	link = bpf_program__attach_usdt(skel->progs.test_usdt,
				-1 /* all PIDs */, "/proc/self/exe",
				"optimized_uprobe", "usdt", NULL);
	if (!ASSERT_OK_PTR(link, "bpf_program__attach_usdt"))
		goto cleanup;

	tramp = check(skel, link, usdt_test, addr, 2);

	/* reattach and detach without triggering optimization */
	link = bpf_program__attach_usdt(skel->progs.test_usdt,
				-1 /* all PIDs */, "/proc/self/exe",
				"optimized_uprobe", "usdt", NULL);
	if (!ASSERT_OK_PTR(link, "bpf_program__attach_usdt"))
		goto cleanup;

	check_attach_notrigger(skel, addr, 2);

	bpf_link__destroy(link);
	if (!check_detach(addr, tramp))
		goto cleanup;

	usdt_test();
	ASSERT_EQ(skel->bss->executed, 2, "executed_no_probe");

	/* reattach with triggering optimization */
	link = bpf_program__attach_usdt(skel->progs.test_usdt,
				-1 /* all PIDs */, "/proc/self/exe",
				"optimized_uprobe", "usdt", NULL);
	if (!ASSERT_OK_PTR(link, "bpf_program__attach_usdt"))
		goto cleanup;

	check(skel, link, usdt_test, addr, 4);

cleanup:
	uprobe_syscall_executed__destroy(skel);
}

/*
 * Borrowed from tools/testing/selftests/x86/test_shadow_stack.c.
 *
 * For use in inline enablement of shadow stack.
 *
 * The program can't return from the point where shadow stack gets enabled
 * because there will be no address on the shadow stack. So it can't use
 * syscall() for enablement, since it is a function.
 *
 * Based on code from nolibc.h. Keep a copy here because this can't pull
 * in all of nolibc.h.
 */
#define ARCH_PRCTL(arg1, arg2)					\
({								\
	long _ret;						\
	register long _num  asm("eax") = __NR_arch_prctl;	\
	register long _arg1 asm("rdi") = (long)(arg1);		\
	register long _arg2 asm("rsi") = (long)(arg2);		\
								\
	asm volatile (						\
		"syscall\n"					\
		: "=a"(_ret)					\
		: "r"(_arg1), "r"(_arg2),			\
		  "0"(_num)					\
		: "rcx", "r11", "memory", "cc"			\
	);							\
	_ret;							\
})

#ifndef ARCH_SHSTK_ENABLE
#define ARCH_SHSTK_ENABLE	0x5001
#define ARCH_SHSTK_DISABLE	0x5002
#define ARCH_SHSTK_SHSTK	(1ULL <<  0)
#endif

static void test_uretprobe_shadow_stack(void)
{
	if (ARCH_PRCTL(ARCH_SHSTK_ENABLE, ARCH_SHSTK_SHSTK)) {
		test__skip();
		return;
	}

	/* Run all the tests with shadow stack in place. */

	test_uprobe_regs_equal(false);
	test_uprobe_regs_equal(true);
	test_uretprobe_syscall_call();

	test_uprobe_legacy();
	test_uprobe_multi();
	test_uprobe_session();
	test_uprobe_usdt();

	test_regs_change();

	ARCH_PRCTL(ARCH_SHSTK_DISABLE, ARCH_SHSTK_SHSTK);
}

static volatile bool race_stop;

static USDT_DEFINE_SEMA(race);

static void *worker_trigger(void *arg)
{
	unsigned long rounds = 0;

	while (!race_stop) {
		uprobe_test();
		rounds++;
	}

	printf("tid %ld trigger rounds: %lu\n", sys_gettid(), rounds);
	return NULL;
}

static void *worker_attach(void *arg)
{
	LIBBPF_OPTS(bpf_uprobe_opts, opts);
	struct uprobe_syscall_executed *skel;
	unsigned long rounds = 0, offset;
	const char *sema[2] = {
		__stringify(USDT_SEMA(race)),
		NULL,
	};
	unsigned long *ref;
	int err;

	offset = get_uprobe_offset(&uprobe_test);
	if (!ASSERT_GE(offset, 0, "get_uprobe_offset"))
		return NULL;

	err = elf_resolve_syms_offsets("/proc/self/exe", 1, (const char **) &sema, &ref, STT_OBJECT);
	if (!ASSERT_OK(err, "elf_resolve_syms_offsets_sema"))
		return NULL;

	opts.ref_ctr_offset = *ref;

	skel = uprobe_syscall_executed__open_and_load();
	if (!ASSERT_OK_PTR(skel, "uprobe_syscall_executed__open_and_load"))
		return NULL;

	skel->bss->pid = getpid();

	while (!race_stop) {
		skel->links.test_uprobe = bpf_program__attach_uprobe_opts(skel->progs.test_uprobe,
					0, "/proc/self/exe", offset, &opts);
		if (!ASSERT_OK_PTR(skel->links.test_uprobe, "bpf_program__attach_uprobe_opts"))
			break;

		bpf_link__destroy(skel->links.test_uprobe);
		skel->links.test_uprobe = NULL;
		rounds++;
	}

	printf("tid %ld attach rounds: %lu hits: %d\n", sys_gettid(), rounds, skel->bss->executed);
	uprobe_syscall_executed__destroy(skel);
	free(ref);
	return NULL;
}

static useconds_t race_msec(void)
{
	char *env;

	env = getenv("BPF_SELFTESTS_UPROBE_SYSCALL_RACE_MSEC");
	if (env)
		return atoi(env);

	/* default duration is 500ms */
	return 500;
}

static void test_uprobe_race(void)
{
	int err, i, nr_threads;
	pthread_t *threads;

	nr_threads = libbpf_num_possible_cpus();
	if (!ASSERT_GT(nr_threads, 0, "libbpf_num_possible_cpus"))
		return;
	nr_threads = max(2, nr_threads);

	threads = alloca(sizeof(*threads) * nr_threads);
	if (!ASSERT_OK_PTR(threads, "malloc"))
		return;

	for (i = 0; i < nr_threads; i++) {
		err = pthread_create(&threads[i], NULL, i % 2 ? worker_trigger : worker_attach,
				     NULL);
		if (!ASSERT_OK(err, "pthread_create"))
			goto cleanup;
	}

	usleep(race_msec() * 1000);

cleanup:
	race_stop = true;
	for (nr_threads = i, i = 0; i < nr_threads; i++)
		pthread_join(threads[i], NULL);

	ASSERT_FALSE(USDT_SEMA_IS_ACTIVE(race), "race_semaphore");
}

#ifndef __NR_uprobe
#define __NR_uprobe 336
#endif

static void test_uprobe_red_zone(void)
{
	struct uprobe_syscall_executed *skel;
	struct bpf_link *link;
	void *nop10_addr;
	size_t offset;
	int i;

	nop10_addr = find_nop10(uprobe_red_zone_test);
	if (!ASSERT_NEQ(nop10_addr, NULL, "find_nop10"))
		return;

	skel = uprobe_syscall_executed__open_and_load();
	if (!ASSERT_OK_PTR(skel, "open_and_load"))
		return;

	offset = get_uprobe_offset(nop10_addr);
	link = bpf_program__attach_uprobe_opts(skel->progs.test_uprobe,
			0, "/proc/self/exe", offset, NULL);
	if (!ASSERT_OK_PTR(link, "attach_uprobe"))
		goto cleanup;

	for (i = 0; i < 10; i++)
		ASSERT_EQ(uprobe_red_zone_test(), 0, "red_zone_intact");

	bpf_link__destroy(link);

cleanup:
	uprobe_syscall_executed__destroy(skel);
}

static void test_uprobe_error(void)
{
	long err = syscall(__NR_uprobe);

	ASSERT_EQ(err, -1, "error");
	ASSERT_EQ(errno, EPROTO, "errno");
}

__attribute__((aligned(16)))
__nocf_check __weak __naked void uprobe_fork_test(void)
{
	asm volatile (
		".byte 0x66, 0x2e, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00\n" /* nop10 */
		"ret\n"
	);
}

static noreturn int child_func(void *arg)
{
	struct uprobe_syscall_executed *skel = arg;

	/* Make sure the child's probe is still there and optimized.. */
	if (memcmp(uprobe_fork_test, lea_rsp, sizeof(lea_rsp)))
		_exit(1);

	skel->bss->pid = getpid();

	/* .. and it executes properly. */
	uprobe_fork_test();

	if (skel->bss->executed != 3)
		_exit(2);

	_exit(0);
}

static void test_uprobe_fork_optimized(bool clone_vm)
{
	struct uprobe_syscall_executed *skel = NULL;
	unsigned long offset;
	int pid, status, err;
	char stack[65535];

	offset = get_uprobe_offset(&uprobe_fork_test);
	if (!ASSERT_GE(offset, 0, "get_uprobe_offset"))
		return;

	skel = uprobe_syscall_executed__open_and_load();
	if (!ASSERT_OK_PTR(skel, "open_and_load"))
		goto cleanup;

	skel->links.test_uprobe = bpf_program__attach_uprobe_opts(skel->progs.test_uprobe,
					-1, "/proc/self/exe", offset, NULL);
	if (!ASSERT_OK_PTR(skel->links.test_uprobe, "attach_uprobe"))
		goto cleanup;

	skel->bss->pid = getpid();

	/* Trigger optimization of uprobe in uprobe_fork_test.  */
	uprobe_fork_test();
	uprobe_fork_test();

	/* Make sure it got optimied. */
	if (!ASSERT_OK(memcmp(uprobe_fork_test, lea_rsp, sizeof(lea_rsp)), "optimized"))
		goto cleanup;

	if (clone_vm) {
		pid = clone(child_func, stack + sizeof(stack), CLONE_VM|SIGCHLD, skel);
		if (!ASSERT_GT(pid, 0, "clone"))
			goto cleanup;
	} else {
		pid = fork();
		if (!ASSERT_GE(pid, 0, "fork"))
			goto cleanup;
		if (pid == 0)
			child_func(skel);
	}

	/* Wait for the child and verify it exited properly with 0. */
	err = waitpid(pid, &status, 0);
	if (ASSERT_EQ(err, pid, "waitpid")) {
		ASSERT_EQ(WIFEXITED(status), 1, "child_exited");
		ASSERT_EQ(WEXITSTATUS(status), 0, "child_exit_code");
	}

cleanup:
	uprobe_syscall_executed__destroy(skel);
}

static void __test_uprobe_syscall(void)
{
	if (test__start_subtest("uretprobe_regs_equal"))
		test_uprobe_regs_equal(true);
	if (test__start_subtest("uretprobe_syscall_call"))
		test_uretprobe_syscall_call();
	if (test__start_subtest("uretprobe_shadow_stack"))
		test_uretprobe_shadow_stack();
	if (test__start_subtest("uprobe_legacy"))
		test_uprobe_legacy();
	if (test__start_subtest("uprobe_multi"))
		test_uprobe_multi();
	if (test__start_subtest("uprobe_session"))
		test_uprobe_session();
	if (test__start_subtest("uprobe_usdt"))
		test_uprobe_usdt();
	if (test__start_subtest("uprobe_race"))
		test_uprobe_race();
	if (test__start_subtest("uprobe_red_zone"))
		test_uprobe_red_zone();
	if (test__start_subtest("uprobe_optimized_fork"))
		test_uprobe_fork_optimized(false);
	if (test__start_subtest("uprobe_optimized_clone_vm"))
		test_uprobe_fork_optimized(true);
	if (test__start_subtest("uprobe_error"))
		test_uprobe_error();
	if (test__start_subtest("uprobe_regs_equal"))
		test_uprobe_regs_equal(false);
	if (test__start_subtest("regs_change"))
		test_regs_change();
}
#else
static void __test_uprobe_syscall(void)
{
	test__skip();
}
#endif

void test_uprobe_syscall(void)
{
	__test_uprobe_syscall();
}