tools/perf/builtin-record.c

// SPDX-License-Identifier: GPL-2.0
/*
 * builtin-record.c
 *
 * Builtin record command: Record the profile of a workload
 * (or a CPU, or a PID) into the perf.data output file - for
 * later analysis via perf report.
 */
#include "builtin.h"

#include "util/build-id.h"
#include <subcmd/parse-options.h>
#include <internal/xyarray.h>
#include "util/parse-events.h"
#include "util/config.h"

#include "util/callchain.h"
#include "util/cgroup.h"
#include "util/header.h"
#include "util/event.h"
#include "util/evlist.h"
#include "util/evsel.h"
#include "util/debug.h"
#include "util/mmap.h"
#include "util/mutex.h"
#include "util/target.h"
#include "util/session.h"
#include "util/tool.h"
#include "util/symbol.h"
#include "util/record.h"
#include "util/cpumap.h"
#include "util/thread_map.h"
#include "util/data.h"
#include "util/perf_regs.h"
#include "util/auxtrace.h"
#include "util/tsc.h"
#include "util/parse-branch-options.h"
#include "util/parse-regs-options.h"
#include "util/perf_api_probe.h"
#include "util/trigger.h"
#include "util/perf-hooks.h"
#include "util/cpu-set-sched.h"
#include "util/synthetic-events.h"
#include "util/time-utils.h"
#include "util/units.h"
#include "util/bpf-event.h"
#include "util/util.h"
#include "util/pfm.h"
#include "util/pmu.h"
#include "util/pmus.h"
#include "util/clockid.h"
#include "util/off_cpu.h"
#include "util/bpf-filter.h"
#include "asm/bug.h"
#include "perf.h"
#include "cputopo.h"

#include <errno.h>
#include <inttypes.h>
#include <locale.h>
#include <poll.h>
#include <pthread.h>
#include <unistd.h>
#ifndef HAVE_GETTID
#include <syscall.h>
#endif
#include <sched.h>
#include <signal.h>
#ifdef HAVE_EVENTFD_SUPPORT
#include <sys/eventfd.h>
#endif
#include <sys/mman.h>
#include <sys/wait.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <linux/err.h>
#include <linux/string.h>
#include <linux/time64.h>
#include <linux/zalloc.h>
#include <linux/bitmap.h>
#include <sys/time.h>

struct switch_output {
	bool		 enabled;
	bool		 signal;
	unsigned long	 size;
	unsigned long	 time;
	const char	*str;
	bool		 set;
	char		 **filenames;
	int		 num_files;
	int		 cur_file;
};

struct thread_mask {
	struct mmap_cpu_mask	maps;
	struct mmap_cpu_mask	affinity;
};

struct record_thread {
	pid_t			tid;
	struct thread_mask	*mask;
	struct {
		int		msg[2];
		int		ack[2];
	} pipes;
	struct fdarray		pollfd;
	int			ctlfd_pos;
	int			nr_mmaps;
	struct mmap		**maps;
	struct mmap		**overwrite_maps;
	struct record		*rec;
	unsigned long long	samples;
	unsigned long		waking;
	u64			bytes_written;
	u64			bytes_transferred;
	u64			bytes_compressed;
};

static __thread struct record_thread *thread;

enum thread_msg {
	THREAD_MSG__UNDEFINED = 0,
	THREAD_MSG__READY,
	THREAD_MSG__MAX,
};

static const char *thread_msg_tags[THREAD_MSG__MAX] = {
	"UNDEFINED", "READY"
};

enum thread_spec {
	THREAD_SPEC__UNDEFINED = 0,
	THREAD_SPEC__CPU,
	THREAD_SPEC__CORE,
	THREAD_SPEC__PACKAGE,
	THREAD_SPEC__NUMA,
	THREAD_SPEC__USER,
	THREAD_SPEC__MAX,
};

static const char *thread_spec_tags[THREAD_SPEC__MAX] = {
	"undefined", "cpu", "core", "package", "numa", "user"
};

struct pollfd_index_map {
	int evlist_pollfd_index;
	int thread_pollfd_index;
};

struct record {
	struct perf_tool	tool;
	struct record_opts	opts;
	u64			bytes_written;
	u64			thread_bytes_written;
	struct perf_data	data;
	struct auxtrace_record	*itr;
	struct evlist	*evlist;
	struct perf_session	*session;
	struct evlist		*sb_evlist;
	pthread_t		thread_id;
	int			realtime_prio;
	bool			switch_output_event_set;
	bool			no_buildid;
	bool			no_buildid_set;
	bool			no_buildid_cache;
	bool			no_buildid_cache_set;
	bool			buildid_all;
	bool			buildid_mmap;
	bool			timestamp_filename;
	bool			timestamp_boundary;
	bool			off_cpu;
	struct switch_output	switch_output;
	unsigned long long	samples;
	unsigned long		output_max_size;	/* = 0: unlimited */
	struct perf_debuginfod	debuginfod;
	int			nr_threads;
	struct thread_mask	*thread_masks;
	struct record_thread	*thread_data;
	struct pollfd_index_map	*index_map;
	size_t			index_map_sz;
	size_t			index_map_cnt;
};

static volatile int done;

static volatile int auxtrace_record__snapshot_started;
static DEFINE_TRIGGER(auxtrace_snapshot_trigger);
static DEFINE_TRIGGER(switch_output_trigger);

static const char *affinity_tags[PERF_AFFINITY_MAX] = {
	"SYS", "NODE", "CPU"
};

#ifndef HAVE_GETTID
static inline pid_t gettid(void)
{
	return (pid_t)syscall(__NR_gettid);
}
#endif

static int record__threads_enabled(struct record *rec)
{
	return rec->opts.threads_spec;
}

static bool switch_output_signal(struct record *rec)
{
	return rec->switch_output.signal &&
	       trigger_is_ready(&switch_output_trigger);
}

static bool switch_output_size(struct record *rec)
{
	return rec->switch_output.size &&
	       trigger_is_ready(&switch_output_trigger) &&
	       (rec->bytes_written >= rec->switch_output.size);
}

static bool switch_output_time(struct record *rec)
{
	return rec->switch_output.time &&
	       trigger_is_ready(&switch_output_trigger);
}

static u64 record__bytes_written(struct record *rec)
{
	return rec->bytes_written + rec->thread_bytes_written;
}

static bool record__output_max_size_exceeded(struct record *rec)
{
	return rec->output_max_size &&
	       (record__bytes_written(rec) >= rec->output_max_size);
}

static int record__write(struct record *rec, struct mmap *map __maybe_unused,
			 void *bf, size_t size)
{
	struct perf_data_file *file = &rec->session->data->file;

	if (map && map->file)
		file = map->file;

	if (perf_data_file__write(file, bf, size) < 0) {
		pr_err("failed to write perf data, error: %m\n");
		return -1;
	}

	if (map && map->file) {
		thread->bytes_written += size;
		rec->thread_bytes_written += size;
	} else {
		rec->bytes_written += size;
	}

	if (record__output_max_size_exceeded(rec) && !done) {
		fprintf(stderr, "[ perf record: perf size limit reached (%" PRIu64 " KB),"
				" stopping session ]\n",
				record__bytes_written(rec) >> 10);
		done = 1;
	}

	if (switch_output_size(rec))
		trigger_hit(&switch_output_trigger);

	return 0;
}

static int record__aio_enabled(struct record *rec);
static int record__comp_enabled(struct record *rec);
static ssize_t zstd_compress(struct perf_session *session, struct mmap *map,
			    void *dst, size_t dst_size, void *src, size_t src_size);

#ifdef HAVE_AIO_SUPPORT
static int record__aio_write(struct aiocb *cblock, int trace_fd,
		void *buf, size_t size, off_t off)
{
	int rc;

	cblock->aio_fildes = trace_fd;
	cblock->aio_buf    = buf;
	cblock->aio_nbytes = size;
	cblock->aio_offset = off;
	cblock->aio_sigevent.sigev_notify = SIGEV_NONE;

	do {
		rc = aio_write(cblock);
		if (rc == 0) {
			break;
		} else if (errno != EAGAIN) {
			cblock->aio_fildes = -1;
			pr_err("failed to queue perf data, error: %m\n");
			break;
		}
	} while (1);

	return rc;
}

static int record__aio_complete(struct mmap *md, struct aiocb *cblock)
{
	void *rem_buf;
	off_t rem_off;
	size_t rem_size;
	int rc, aio_errno;
	ssize_t aio_ret, written;

	aio_errno = aio_error(cblock);
	if (aio_errno == EINPROGRESS)
		return 0;

	written = aio_ret = aio_return(cblock);
	if (aio_ret < 0) {
		if (aio_errno != EINTR)
			pr_err("failed to write perf data, error: %m\n");
		written = 0;
	}

	rem_size = cblock->aio_nbytes - written;

	if (rem_size == 0) {
		cblock->aio_fildes = -1;
		/*
		 * md->refcount is incremented in record__aio_pushfn() for
		 * every aio write request started in record__aio_push() so
		 * decrement it because the request is now complete.
		 */
		perf_mmap__put(&md->core);
		rc = 1;
	} else {
		/*
		 * aio write request may require restart with the
		 * remainder if the kernel didn't write whole
		 * chunk at once.
		 */
		rem_off = cblock->aio_offset + written;
		rem_buf = (void *)(cblock->aio_buf + written);
		record__aio_write(cblock, cblock->aio_fildes,
				rem_buf, rem_size, rem_off);
		rc = 0;
	}

	return rc;
}

static int record__aio_sync(struct mmap *md, bool sync_all)
{
	struct aiocb **aiocb = md->aio.aiocb;
	struct aiocb *cblocks = md->aio.cblocks;
	struct timespec timeout = { 0, 1000 * 1000  * 1 }; /* 1ms */
	int i, do_suspend;

	do {
		do_suspend = 0;
		for (i = 0; i < md->aio.nr_cblocks; ++i) {
			if (cblocks[i].aio_fildes == -1 || record__aio_complete(md, &cblocks[i])) {
				if (sync_all)
					aiocb[i] = NULL;
				else
					return i;
			} else {
				/*
				 * Started aio write is not complete yet
				 * so it has to be waited before the
				 * next allocation.
				 */
				aiocb[i] = &cblocks[i];
				do_suspend = 1;
			}
		}
		if (!do_suspend)
			return -1;

		while (aio_suspend((const struct aiocb **)aiocb, md->aio.nr_cblocks, &timeout)) {
			if (!(errno == EAGAIN || errno == EINTR))
				pr_err("failed to sync perf data, error: %m\n");
		}
	} while (1);
}

struct record_aio {
	struct record	*rec;
	void		*data;
	size_t		size;
};

static int record__aio_pushfn(struct mmap *map, void *to, void *buf, size_t size)
{
	struct record_aio *aio = to;

	/*
	 * map->core.base data pointed by buf is copied into free map->aio.data[] buffer
	 * to release space in the kernel buffer as fast as possible, calling
	 * perf_mmap__consume() from perf_mmap__push() function.
	 *
	 * That lets the kernel to proceed with storing more profiling data into
	 * the kernel buffer earlier than other per-cpu kernel buffers are handled.
	 *
	 * Coping can be done in two steps in case the chunk of profiling data
	 * crosses the upper bound of the kernel buffer. In this case we first move
	 * part of data from map->start till the upper bound and then the remainder
	 * from the beginning of the kernel buffer till the end of the data chunk.
	 */

	if (record__comp_enabled(aio->rec)) {
		ssize_t compressed = zstd_compress(aio->rec->session, NULL, aio->data + aio->size,
						   mmap__mmap_len(map) - aio->size,
						   buf, size);
		if (compressed < 0)
			return (int)compressed;

		size = compressed;
	} else {
		memcpy(aio->data + aio->size, buf, size);
	}

	if (!aio->size) {
		/*
		 * Increment map->refcount to guard map->aio.data[] buffer
		 * from premature deallocation because map object can be
		 * released earlier than aio write request started on
		 * map->aio.data[] buffer is complete.
		 *
		 * perf_mmap__put() is done at record__aio_complete()
		 * after started aio request completion or at record__aio_push()
		 * if the request failed to start.
		 */
		perf_mmap__get(&map->core);
	}

	aio->size += size;

	return size;
}

static int record__aio_push(struct record *rec, struct mmap *map, off_t *off)
{
	int ret, idx;
	int trace_fd = rec->session->data->file.fd;
	struct record_aio aio = { .rec = rec, .size = 0 };

	/*
	 * Call record__aio_sync() to wait till map->aio.data[] buffer
	 * becomes available after previous aio write operation.
	 */

	idx = record__aio_sync(map, false);
	aio.data = map->aio.data[idx];
	ret = perf_mmap__push(map, &aio, record__aio_pushfn);
	if (ret != 0) /* ret > 0 - no data, ret < 0 - error */
		return ret;

	rec->samples++;
	ret = record__aio_write(&(map->aio.cblocks[idx]), trace_fd, aio.data, aio.size, *off);
	if (!ret) {
		*off += aio.size;
		rec->bytes_written += aio.size;
		if (switch_output_size(rec))
			trigger_hit(&switch_output_trigger);
	} else {
		/*
		 * Decrement map->refcount incremented in record__aio_pushfn()
		 * back if record__aio_write() operation failed to start, otherwise
		 * map->refcount is decremented in record__aio_complete() after
		 * aio write operation finishes successfully.
		 */
		perf_mmap__put(&map->core);
	}

	return ret;
}

static off_t record__aio_get_pos(int trace_fd)
{
	return lseek(trace_fd, 0, SEEK_CUR);
}

static void record__aio_set_pos(int trace_fd, off_t pos)
{
	lseek(trace_fd, pos, SEEK_SET);
}

static void record__aio_mmap_read_sync(struct record *rec)
{
	int i;
	struct evlist *evlist = rec->evlist;
	struct mmap *maps = evlist->mmap;

	if (!record__aio_enabled(rec))
		return;

	for (i = 0; i < evlist->core.nr_mmaps; i++) {
		struct mmap *map = &maps[i];

		if (map->core.base)
			record__aio_sync(map, true);
	}
}

static int nr_cblocks_default = 1;
static int nr_cblocks_max = 4;

static int record__aio_parse(const struct option *opt,
			     const char *str,
			     int unset)
{
	struct record_opts *opts = (struct record_opts *)opt->value;

	if (unset) {
		opts->nr_cblocks = 0;
	} else {
		if (str)
			opts->nr_cblocks = strtol(str, NULL, 0);
		if (!opts->nr_cblocks)
			opts->nr_cblocks = nr_cblocks_default;
	}

	return 0;
}
#else /* HAVE_AIO_SUPPORT */
static int nr_cblocks_max = 0;

static int record__aio_push(struct record *rec __maybe_unused, struct mmap *map __maybe_unused,
			    off_t *off __maybe_unused)
{
	return -1;
}

static off_t record__aio_get_pos(int trace_fd __maybe_unused)
{
	return -1;
}

static void record__aio_set_pos(int trace_fd __maybe_unused, off_t pos __maybe_unused)
{
}

static void record__aio_mmap_read_sync(struct record *rec __maybe_unused)
{
}
#endif

static int record__aio_enabled(struct record *rec)
{
	return rec->opts.nr_cblocks > 0;
}

#define MMAP_FLUSH_DEFAULT 1
static int record__mmap_flush_parse(const struct option *opt,
				    const char *str,
				    int unset)
{
	int flush_max;
	struct record_opts *opts = (struct record_opts *)opt->value;
	static struct parse_tag tags[] = {
			{ .tag  = 'B', .mult = 1       },
			{ .tag  = 'K', .mult = 1 << 10 },
			{ .tag  = 'M', .mult = 1 << 20 },
			{ .tag  = 'G', .mult = 1 << 30 },
			{ .tag  = 0 },
	};

	if (unset)
		return 0;

	if (str) {
		opts->mmap_flush = parse_tag_value(str, tags);
		if (opts->mmap_flush == (int)-1)
			opts->mmap_flush = strtol(str, NULL, 0);
	}

	if (!opts->mmap_flush)
		opts->mmap_flush = MMAP_FLUSH_DEFAULT;

	flush_max = evlist__mmap_size(opts->mmap_pages);
	flush_max /= 4;
	if (opts->mmap_flush > flush_max)
		opts->mmap_flush = flush_max;

	return 0;
}

#ifdef HAVE_ZSTD_SUPPORT
static unsigned int comp_level_default = 1;

static int record__parse_comp_level(const struct option *opt, const char *str, int unset)
{
	struct record_opts *opts = opt->value;

	if (unset) {
		opts->comp_level = 0;
	} else {
		if (str)
			opts->comp_level = strtol(str, NULL, 0);
		if (!opts->comp_level)
			opts->comp_level = comp_level_default;
	}

	return 0;
}
#endif
static unsigned int comp_level_max = 22;

static int record__comp_enabled(struct record *rec)
{
	return rec->opts.comp_level > 0;
}

static int process_synthesized_event(struct perf_tool *tool,
				     union perf_event *event,
				     struct perf_sample *sample __maybe_unused,
				     struct machine *machine __maybe_unused)
{
	struct record *rec = container_of(tool, struct record, tool);
	return record__write(rec, NULL, event, event->header.size);
}

static struct mutex synth_lock;

static int process_locked_synthesized_event(struct perf_tool *tool,
				     union perf_event *event,
				     struct perf_sample *sample __maybe_unused,
				     struct machine *machine __maybe_unused)
{
	int ret;

	mutex_lock(&synth_lock);
	ret = process_synthesized_event(tool, event, sample, machine);
	mutex_unlock(&synth_lock);
	return ret;
}

static int record__pushfn(struct mmap *map, void *to, void *bf, size_t size)
{
	struct record *rec = to;

	if (record__comp_enabled(rec)) {
		ssize_t compressed = zstd_compress(rec->session, map, map->data,
						   mmap__mmap_len(map), bf, size);

		if (compressed < 0)
			return (int)compressed;

		size = compressed;
		bf   = map->data;
	}

	thread->samples++;
	return record__write(rec, map, bf, size);
}

static volatile sig_atomic_t signr = -1;
static volatile sig_atomic_t child_finished;
#ifdef HAVE_EVENTFD_SUPPORT
static volatile sig_atomic_t done_fd = -1;
#endif

static void sig_handler(int sig)
{
	if (sig == SIGCHLD)
		child_finished = 1;
	else
		signr = sig;

	done = 1;
#ifdef HAVE_EVENTFD_SUPPORT
	if (done_fd >= 0) {
		u64 tmp = 1;
		int orig_errno = errno;

		/*
		 * It is possible for this signal handler to run after done is
		 * checked in the main loop, but before the perf counter fds are
		 * polled. If this happens, the poll() will continue to wait
		 * even though done is set, and will only break out if either
		 * another signal is received, or the counters are ready for
		 * read. To ensure the poll() doesn't sleep when done is set,
		 * use an eventfd (done_fd) to wake up the poll().
		 */
		if (write(done_fd, &tmp, sizeof(tmp)) < 0)
			pr_err("failed to signal wakeup fd, error: %m\n");

		errno = orig_errno;
	}
#endif // HAVE_EVENTFD_SUPPORT
}

static void sigsegv_handler(int sig)
{
	perf_hooks__recover();
	sighandler_dump_stack(sig);
}

static void record__sig_exit(void)
{
	if (signr == -1)
		return;

	signal(signr, SIG_DFL);
	raise(signr);
}

#ifdef HAVE_AUXTRACE_SUPPORT

static int record__process_auxtrace(struct perf_tool *tool,
				    struct mmap *map,
				    union perf_event *event, void *data1,
				    size_t len1, void *data2, size_t len2)
{
	struct record *rec = container_of(tool, struct record, tool);
	struct perf_data *data = &rec->data;
	size_t padding;
	u8 pad[8] = {0};

	if (!perf_data__is_pipe(data) && perf_data__is_single_file(data)) {
		off_t file_offset;
		int fd = perf_data__fd(data);
		int err;

		file_offset = lseek(fd, 0, SEEK_CUR);
		if (file_offset == -1)
			return -1;
		err = auxtrace_index__auxtrace_event(&rec->session->auxtrace_index,
						     event, file_offset);
		if (err)
			return err;
	}

	/* event.auxtrace.size includes padding, see __auxtrace_mmap__read() */
	padding = (len1 + len2) & 7;
	if (padding)
		padding = 8 - padding;

	record__write(rec, map, event, event->header.size);
	record__write(rec, map, data1, len1);
	if (len2)
		record__write(rec, map, data2, len2);
	record__write(rec, map, &pad, padding);

	return 0;
}

static int record__auxtrace_mmap_read(struct record *rec,
				      struct mmap *map)
{
	int ret;

	ret = auxtrace_mmap__read(map, rec->itr, &rec->tool,
				  record__process_auxtrace);
	if (ret < 0)
		return ret;

	if (ret)
		rec->samples++;

	return 0;
}

static int record__auxtrace_mmap_read_snapshot(struct record *rec,
					       struct mmap *map)
{
	int ret;

	ret = auxtrace_mmap__read_snapshot(map, rec->itr, &rec->tool,
					   record__process_auxtrace,
					   rec->opts.auxtrace_snapshot_size);
	if (ret < 0)
		return ret;

	if (ret)
		rec->samples++;

	return 0;
}

static int record__auxtrace_read_snapshot_all(struct record *rec)
{
	int i;
	int rc = 0;

	for (i = 0; i < rec->evlist->core.nr_mmaps; i++) {
		struct mmap *map = &rec->evlist->mmap[i];

		if (!map->auxtrace_mmap.base)
			continue;

		if (record__auxtrace_mmap_read_snapshot(rec, map) != 0) {
			rc = -1;
			goto out;
		}
	}
out:
	return rc;
}

static void record__read_auxtrace_snapshot(struct record *rec, bool on_exit)
{
	pr_debug("Recording AUX area tracing snapshot\n");
	if (record__auxtrace_read_snapshot_all(rec) < 0) {
		trigger_error(&auxtrace_snapshot_trigger);
	} else {
		if (auxtrace_record__snapshot_finish(rec->itr, on_exit))
			trigger_error(&auxtrace_snapshot_trigger);
		else
			trigger_ready(&auxtrace_snapshot_trigger);
	}
}

static int record__auxtrace_snapshot_exit(struct record *rec)
{
	if (trigger_is_error(&auxtrace_snapshot_trigger))
		return 0;

	if (!auxtrace_record__snapshot_started &&
	    auxtrace_record__snapshot_start(rec->itr))
		return -1;

	record__read_auxtrace_snapshot(rec, true);
	if (trigger_is_error(&auxtrace_snapshot_trigger))
		return -1;

	return 0;
}

static int record__auxtrace_init(struct record *rec)
{
	int err;

	if ((rec->opts.auxtrace_snapshot_opts || rec->opts.auxtrace_sample_opts)
	    && record__threads_enabled(rec)) {
		pr_err("AUX area tracing options are not available in parallel streaming mode.\n");
		return -EINVAL;
	}

	if (!rec->itr) {
		rec->itr = auxtrace_record__init(rec->evlist, &err);
		if (err)
			return err;
	}

	err = auxtrace_parse_snapshot_options(rec->itr, &rec->opts,
					      rec->opts.auxtrace_snapshot_opts);
	if (err)
		return err;

	err = auxtrace_parse_sample_options(rec->itr, rec->evlist, &rec->opts,
					    rec->opts.auxtrace_sample_opts);
	if (err)
		return err;

	auxtrace_regroup_aux_output(rec->evlist);

	return auxtrace_parse_filters(rec->evlist);
}

#else

static inline
int record__auxtrace_mmap_read(struct record *rec __maybe_unused,
			       struct mmap *map __maybe_unused)
{
	return 0;
}

static inline
void record__read_auxtrace_snapshot(struct record *rec __maybe_unused,
				    bool on_exit __maybe_unused)
{
}

static inline
int auxtrace_record__snapshot_start(struct auxtrace_record *itr __maybe_unused)
{
	return 0;
}

static inline
int record__auxtrace_snapshot_exit(struct record *rec __maybe_unused)
{
	return 0;
}

static int record__auxtrace_init(struct record *rec __maybe_unused)
{
	return 0;
}

#endif

static int record__config_text_poke(struct evlist *evlist)
{
	struct evsel *evsel;

	/* Nothing to do if text poke is already configured */
	evlist__for_each_entry(evlist, evsel) {
		if (evsel->core.attr.text_poke)
			return 0;
	}

	evsel = evlist__add_dummy_on_all_cpus(evlist);
	if (!evsel)
		return -ENOMEM;

	evsel->core.attr.text_poke = 1;
	evsel->core.attr.ksymbol = 1;
	evsel->immediate = true;
	evsel__set_sample_bit(evsel, TIME);

	return 0;
}

static int record__config_off_cpu(struct record *rec)
{
	return off_cpu_prepare(rec->evlist, &rec->opts.target, &rec->opts);
}

static bool record__tracking_system_wide(struct record *rec)
{
	struct evlist *evlist = rec->evlist;
	struct evsel *evsel;

	/*
	 * If non-dummy evsel exists, system_wide sideband is need to
	 * help parse sample information.
	 * For example, PERF_EVENT_MMAP event to help parse symbol,
	 * and PERF_EVENT_COMM event to help parse task executable name.
	 */
	evlist__for_each_entry(evlist, evsel) {
		if (!evsel__is_dummy_event(evsel))
			return true;
	}

	return false;
}

static int record__config_tracking_events(struct record *rec)
{
	struct record_opts *opts = &rec->opts;
	struct evlist *evlist = rec->evlist;
	bool system_wide = false;
	struct evsel *evsel;

	/*
	 * For initial_delay, system wide or a hybrid system, we need to add
	 * tracking event so that we can track PERF_RECORD_MMAP to cover the
	 * delay of waiting or event synthesis.
	 */
	if (opts->target.initial_delay || target__has_cpu(&opts->target) ||
	    perf_pmus__num_core_pmus() > 1) {

		/*
		 * User space tasks can migrate between CPUs, so when tracing
		 * selected CPUs, sideband for all CPUs is still needed.
		 */
		if (!!opts->target.cpu_list && record__tracking_system_wide(rec))
			system_wide = true;

		evsel = evlist__findnew_tracking_event(evlist, system_wide);
		if (!evsel)
			return -ENOMEM;

		/*
		 * Enable the tracking event when the process is forked for
		 * initial_delay, immediately for system wide.
		 */
		if (opts->target.initial_delay && !evsel->immediate &&
		    !target__has_cpu(&opts->target))
			evsel->core.attr.enable_on_exec = 1;
		else
			evsel->immediate = 1;
	}

	return 0;
}

static bool record__kcore_readable(struct machine *machine)
{
	char kcore[PATH_MAX];
	int fd;

	scnprintf(kcore, sizeof(kcore), "%s/proc/kcore", machine->root_dir);

	fd = open(kcore, O_RDONLY);
	if (fd < 0)
		return false;

	close(fd);

	return true;
}

static int record__kcore_copy(struct machine *machine, struct perf_data *data)
{
	char from_dir[PATH_MAX];
	char kcore_dir[PATH_MAX];
	int ret;

	snprintf(from_dir, sizeof(from_dir), "%s/proc", machine->root_dir);

	ret = perf_data__make_kcore_dir(data, kcore_dir, sizeof(kcore_dir));
	if (ret)
		return ret;

	return kcore_copy(from_dir, kcore_dir);
}

static void record__thread_data_init_pipes(struct record_thread *thread_data)
{
	thread_data->pipes.msg[0] = -1;
	thread_data->pipes.msg[1] = -1;
	thread_data->pipes.ack[0] = -1;
	thread_data->pipes.ack[1] = -1;
}

static int record__thread_data_open_pipes(struct record_thread *thread_data)
{
	if (pipe(thread_data->pipes.msg))
		return -EINVAL;

	if (pipe(thread_data->pipes.ack)) {
		close(thread_data->pipes.msg[0]);
		thread_data->pipes.msg[0] = -1;
		close(thread_data->pipes.msg[1]);
		thread_data->pipes.msg[1] = -1;
		return -EINVAL;
	}

	pr_debug2("thread_data[%p]: msg=[%d,%d], ack=[%d,%d]\n", thread_data,
		 thread_data->pipes.msg[0], thread_data->pipes.msg[1],
		 thread_data->pipes.ack[0], thread_data->pipes.ack[1]);

	return 0;
}

static void record__thread_data_close_pipes(struct record_thread *thread_data)
{
	if (thread_data->pipes.msg[0] != -1) {
		close(thread_data->pipes.msg[0]);
		thread_data->pipes.msg[0] = -1;
	}
	if (thread_data->pipes.msg[1] != -1) {
		close(thread_data->pipes.msg[1]);
		thread_data->pipes.msg[1] = -1;
	}
	if (thread_data->pipes.ack[0] != -1) {
		close(thread_data->pipes.ack[0]);
		thread_data->pipes.ack[0] = -1;
	}
	if (thread_data->pipes.ack[1] != -1) {
		close(thread_data->pipes.ack[1]);
		thread_data->pipes.ack[1] = -1;
	}
}

static bool evlist__per_thread(struct evlist *evlist)
{
	return cpu_map__is_dummy(evlist->core.user_requested_cpus);
}

static int record__thread_data_init_maps(struct record_thread *thread_data, struct evlist *evlist)
{
	int m, tm, nr_mmaps = evlist->core.nr_mmaps;
	struct mmap *mmap = evlist->mmap;
	struct mmap *overwrite_mmap = evlist->overwrite_mmap;
	struct perf_cpu_map *cpus = evlist->core.all_cpus;
	bool per_thread = evlist__per_thread(evlist);

	if (per_thread)
		thread_data->nr_mmaps = nr_mmaps;
	else
		thread_data->nr_mmaps = bitmap_weight(thread_data->mask->maps.bits,
						      thread_data->mask->maps.nbits);
	if (mmap) {
		thread_data->maps = zalloc(thread_data->nr_mmaps * sizeof(struct mmap *));
		if (!thread_data->maps)
			return -ENOMEM;
	}
	if (overwrite_mmap) {
		thread_data->overwrite_maps = zalloc(thread_data->nr_mmaps * sizeof(struct mmap *));
		if (!thread_data->overwrite_maps) {
			zfree(&thread_data->maps);
			return -ENOMEM;
		}
	}
	pr_debug2("thread_data[%p]: nr_mmaps=%d, maps=%p, ow_maps=%p\n", thread_data,
		 thread_data->nr_mmaps, thread_data->maps, thread_data->overwrite_maps);

	for (m = 0, tm = 0; m < nr_mmaps && tm < thread_data->nr_mmaps; m++) {
		if (per_thread ||
		    test_bit(perf_cpu_map__cpu(cpus, m).cpu, thread_data->mask->maps.bits)) {
			if (thread_data->maps) {
				thread_data->maps[tm] = &mmap[m];
				pr_debug2("thread_data[%p]: cpu%d: maps[%d] -> mmap[%d]\n",
					  thread_data, perf_cpu_map__cpu(cpus, m).cpu, tm, m);
			}
			if (thread_data->overwrite_maps) {
				thread_data->overwrite_maps[tm] = &overwrite_mmap[m];
				pr_debug2("thread_data[%p]: cpu%d: ow_maps[%d] -> ow_mmap[%d]\n",
					  thread_data, perf_cpu_map__cpu(cpus, m).cpu, tm, m);
			}
			tm++;
		}
	}

	return 0;
}

static int record__thread_data_init_pollfd(struct record_thread *thread_data, struct evlist *evlist)
{
	int f, tm, pos;
	struct mmap *map, *overwrite_map;

	fdarray__init(&thread_data->pollfd, 64);

	for (tm = 0; tm < thread_data->nr_mmaps; tm++) {
		map = thread_data->maps ? thread_data->maps[tm] : NULL;
		overwrite_map = thread_data->overwrite_maps ?
				thread_data->overwrite_maps[tm] : NULL;

		for (f = 0; f < evlist->core.pollfd.nr; f++) {
			void *ptr = evlist->core.pollfd.priv[f].ptr;

			if ((map && ptr == map) || (overwrite_map && ptr == overwrite_map)) {
				pos = fdarray__dup_entry_from(&thread_data->pollfd, f,
							      &evlist->core.pollfd);
				if (pos < 0)
					return pos;
				pr_debug2("thread_data[%p]: pollfd[%d] <- event_fd=%d\n",
					 thread_data, pos, evlist->core.pollfd.entries[f].fd);
			}
		}
	}

	return 0;
}

static void record__free_thread_data(struct record *rec)
{
	int t;
	struct record_thread *thread_data = rec->thread_data;

	if (thread_data == NULL)
		return;

	for (t = 0; t < rec->nr_threads; t++) {
		record__thread_data_close_pipes(&thread_data[t]);
		zfree(&thread_data[t].maps);
		zfree(&thread_data[t].overwrite_maps);
		fdarray__exit(&thread_data[t].pollfd);
	}

	zfree(&rec->thread_data);
}

static int record__map_thread_evlist_pollfd_indexes(struct record *rec,
						    int evlist_pollfd_index,
						    int thread_pollfd_index)
{
	size_t x = rec->index_map_cnt;

	if (realloc_array_as_needed(rec->index_map, rec->index_map_sz, x, NULL))
		return -ENOMEM;
	rec->index_map[x].evlist_pollfd_index = evlist_pollfd_index;
	rec->index_map[x].thread_pollfd_index = thread_pollfd_index;
	rec->index_map_cnt += 1;
	return 0;
}

static int record__update_evlist_pollfd_from_thread(struct record *rec,
						    struct evlist *evlist,
						    struct record_thread *thread_data)
{
	struct pollfd *e_entries = evlist->core.pollfd.entries;
	struct pollfd *t_entries = thread_data->pollfd.entries;
	int err = 0;
	size_t i;

	for (i = 0; i < rec->index_map_cnt; i++) {
		int e_pos = rec->index_map[i].evlist_pollfd_index;
		int t_pos = rec->index_map[i].thread_pollfd_index;

		if (e_entries[e_pos].fd != t_entries[t_pos].fd ||
		    e_entries[e_pos].events != t_entries[t_pos].events) {
			pr_err("Thread and evlist pollfd index mismatch\n");
			err = -EINVAL;
			continue;
		}
		e_entries[e_pos].revents = t_entries[t_pos].revents;
	}
	return err;
}

static int record__dup_non_perf_events(struct record *rec,
				       struct evlist *evlist,
				       struct record_thread *thread_data)
{
	struct fdarray *fda = &evlist->core.pollfd;
	int i, ret;

	for (i = 0; i < fda->nr; i++) {
		if (!(fda->priv[i].flags & fdarray_flag__non_perf_event))
			continue;
		ret = fdarray__dup_entry_from(&thread_data->pollfd, i, fda);
		if (ret < 0) {
			pr_err("Failed to duplicate descriptor in main thread pollfd\n");
			return ret;
		}
		pr_debug2("thread_data[%p]: pollfd[%d] <- non_perf_event fd=%d\n",
			  thread_data, ret, fda->entries[i].fd);
		ret = record__map_thread_evlist_pollfd_indexes(rec, i, ret);
		if (ret < 0) {
			pr_err("Failed to map thread and evlist pollfd indexes\n");
			return ret;
		}
	}
	return 0;
}

static int record__alloc_thread_data(struct record *rec, struct evlist *evlist)
{
	int t, ret;
	struct record_thread *thread_data;

	rec->thread_data = zalloc(rec->nr_threads * sizeof(*(rec->thread_data)));
	if (!rec->thread_data) {
		pr_err("Failed to allocate thread data\n");
		return -ENOMEM;
	}
	thread_data = rec->thread_data;

	for (t = 0; t < rec->nr_threads; t++)
		record__thread_data_init_pipes(&thread_data[t]);

	for (t = 0; t < rec->nr_threads; t++) {
		thread_data[t].rec = rec;
		thread_data[t].mask = &rec->thread_masks[t];
		ret = record__thread_data_init_maps(&thread_data[t], evlist);
		if (ret) {
			pr_err("Failed to initialize thread[%d] maps\n", t);
			goto out_free;
		}
		ret = record__thread_data_init_pollfd(&thread_data[t], evlist);
		if (ret) {
			pr_err("Failed to initialize thread[%d] pollfd\n", t);
			goto out_free;
		}
		if (t) {
			thread_data[t].tid = -1;
			ret = record__thread_data_open_pipes(&thread_data[t]);
			if (ret) {
				pr_err("Failed to open thread[%d] communication pipes\n", t);
				goto out_free;
			}
			ret = fdarray__add(&thread_data[t].pollfd, thread_data[t].pipes.msg[0],
					   POLLIN | POLLERR | POLLHUP, fdarray_flag__nonfilterable);
			if (ret < 0) {
				pr_err("Failed to add descriptor to thread[%d] pollfd\n", t);
				goto out_free;
			}
			thread_data[t].ctlfd_pos = ret;
			pr_debug2("thread_data[%p]: pollfd[%d] <- ctl_fd=%d\n",
				 thread_data, thread_data[t].ctlfd_pos,
				 thread_data[t].pipes.msg[0]);
		} else {
			thread_data[t].tid = gettid();

			ret = record__dup_non_perf_events(rec, evlist, &thread_data[t]);
			if (ret < 0)
				goto out_free;

			thread_data[t].ctlfd_pos = -1; /* Not used */
		}
	}

	return 0;

out_free:
	record__free_thread_data(rec);

	return ret;
}

static int record__mmap_evlist(struct record *rec,
			       struct evlist *evlist)
{
	int i, ret;
	struct record_opts *opts = &rec->opts;
	bool auxtrace_overwrite = opts->auxtrace_snapshot_mode ||
				  opts->auxtrace_sample_mode;
	char msg[512];

	if (opts->affinity != PERF_AFFINITY_SYS)
		cpu__setup_cpunode_map();

	if (evlist__mmap_ex(evlist, opts->mmap_pages,
				 opts->auxtrace_mmap_pages,
				 auxtrace_overwrite,
				 opts->nr_cblocks, opts->affinity,
				 opts->mmap_flush, opts->comp_level) < 0) {
		if (errno == EPERM) {
			pr_err("Permission error mapping pages.\n"
			       "Consider increasing "
			       "/proc/sys/kernel/perf_event_mlock_kb,\n"
			       "or try again with a smaller value of -m/--mmap_pages.\n"
			       "(current value: %u,%u)\n",
			       opts->mmap_pages, opts->auxtrace_mmap_pages);
			return -errno;
		} else {
			pr_err("failed to mmap with %d (%s)\n", errno,
				str_error_r(errno, msg, sizeof(msg)));
			if (errno)
				return -errno;
			else
				return -EINVAL;
		}
	}

	if (evlist__initialize_ctlfd(evlist, opts->ctl_fd, opts->ctl_fd_ack))
		return -1;

	ret = record__alloc_thread_data(rec, evlist);
	if (ret)
		return ret;

	if (record__threads_enabled(rec)) {
		ret = perf_data__create_dir(&rec->data, evlist->core.nr_mmaps);
		if (ret) {
			pr_err("Failed to create data directory: %s\n", strerror(-ret));
			return ret;
		}
		for (i = 0; i < evlist->core.nr_mmaps; i++) {
			if (evlist->mmap)
				evlist->mmap[i].file = &rec->data.dir.files[i];
			if (evlist->overwrite_mmap)
				evlist->overwrite_mmap[i].file = &rec->data.dir.files[i];
		}
	}

	return 0;
}

static int record__mmap(struct record *rec)
{
	return record__mmap_evlist(rec, rec->evlist);
}

static int record__open(struct record *rec)
{
	char msg[BUFSIZ];
	struct evsel *pos;
	struct evlist *evlist = rec->evlist;
	struct perf_session *session = rec->session;
	struct record_opts *opts = &rec->opts;
	int rc = 0;

	evlist__for_each_entry(evlist, pos) {
try_again:
		if (evsel__open(pos, pos->core.cpus, pos->core.threads) < 0) {
			if (evsel__fallback(pos, &opts->target, errno, msg, sizeof(msg))) {
				if (verbose > 0)
					ui__warning("%s\n", msg);
				goto try_again;
			}
			if ((errno == EINVAL || errno == EBADF) &&
			    pos->core.leader != &pos->core &&
			    pos->weak_group) {
			        pos = evlist__reset_weak_group(evlist, pos, true);
				goto try_again;
			}
			rc = -errno;
			evsel__open_strerror(pos, &opts->target, errno, msg, sizeof(msg));
			ui__error("%s\n", msg);
			goto out;
		}

		pos->supported = true;
	}

	if (symbol_conf.kptr_restrict && !evlist__exclude_kernel(evlist)) {
		pr_warning(
"WARNING: Kernel address maps (/proc/{kallsyms,modules}) are restricted,\n"
"check /proc/sys/kernel/kptr_restrict and /proc/sys/kernel/perf_event_paranoid.\n\n"
"Samples in kernel functions may not be resolved if a suitable vmlinux\n"
"file is not found in the buildid cache or in the vmlinux path.\n\n"
"Samples in kernel modules won't be resolved at all.\n\n"
"If some relocation was applied (e.g. kexec) symbols may be misresolved\n"
"even with a suitable vmlinux or kallsyms file.\n\n");
	}

	if (evlist__apply_filters(evlist, &pos)) {
		pr_err("failed to set filter \"%s\" on event %s with %d (%s)\n",
			pos->filter ?: "BPF", evsel__name(pos), errno,
			str_error_r(errno, msg, sizeof(msg)));
		rc = -1;
		goto out;
	}

	rc = record__mmap(rec);
	if (rc)
		goto out;

	session->evlist = evlist;
	perf_session__set_id_hdr_size(session);
out:
	return rc;
}

static void set_timestamp_boundary(struct record *rec, u64 sample_time)
{
	if (rec->evlist->first_sample_time == 0)
		rec->evlist->first_sample_time = sample_time;

	if (sample_time)
		rec->evlist->last_sample_time = sample_time;
}

static int process_sample_event(struct perf_tool *tool,
				union perf_event *event,
				struct perf_sample *sample,
				struct evsel *evsel,
				struct machine *machine)
{
	struct record *rec = container_of(tool, struct record, tool);

	set_timestamp_boundary(rec, sample->time);

	if (rec->buildid_all)
		return 0;

	rec->samples++;
	return build_id__mark_dso_hit(tool, event, sample, evsel, machine);
}

static int process_buildids(struct record *rec)
{
	struct perf_session *session = rec->session;

	if (perf_data__size(&rec->data) == 0)
		return 0;

	/*
	 * During this process, it'll load kernel map and replace the
	 * dso->long_name to a real pathname it found.  In this case
	 * we prefer the vmlinux path like
	 *   /lib/modules/3.16.4/build/vmlinux
	 *
	 * rather than build-id path (in debug directory).
	 *   $HOME/.debug/.build-id/f0/6e17aa50adf4d00b88925e03775de107611551
	 */
	symbol_conf.ignore_vmlinux_buildid = true;

	/*
	 * If --buildid-all is given, it marks all DSO regardless of hits,
	 * so no need to process samples. But if timestamp_boundary is enabled,
	 * it still needs to walk on all samples to get the timestamps of
	 * first/last samples.
	 */
	if (rec->buildid_all && !rec->timestamp_boundary)
		rec->tool.sample = NULL;

	return perf_session__process_events(session);
}

static void perf_event__synthesize_guest_os(struct machine *machine, void *data)
{
	int err;
	struct perf_tool *tool = data;
	/*
	 *As for guest kernel when processing subcommand record&report,
	 *we arrange module mmap prior to guest kernel mmap and trigger
	 *a preload dso because default guest module symbols are loaded
	 *from guest kallsyms instead of /lib/modules/XXX/XXX. This
	 *method is used to avoid symbol missing when the first addr is
	 *in module instead of in guest kernel.
	 */
	err = perf_event__synthesize_modules(tool, process_synthesized_event,
					     machine);
	if (err < 0)
		pr_err("Couldn't record guest kernel [%d]'s reference"
		       " relocation symbol.\n", machine->pid);

	/*
	 * We use _stext for guest kernel because guest kernel's /proc/kallsyms
	 * have no _text sometimes.
	 */
	err = perf_event__synthesize_kernel_mmap(tool, process_synthesized_event,
						 machine);
	if (err < 0)
		pr_err("Couldn't record guest kernel [%d]'s reference"
		       " relocation symbol.\n", machine->pid);
}

static struct perf_event_header finished_round_event = {
	.size = sizeof(struct perf_event_header),
	.type = PERF_RECORD_FINISHED_ROUND,
};

static struct perf_event_header finished_init_event = {
	.size = sizeof(struct perf_event_header),
	.type = PERF_RECORD_FINISHED_INIT,
};

static void record__adjust_affinity(struct record *rec, struct mmap *map)
{
	if (rec->opts.affinity != PERF_AFFINITY_SYS &&
	    !bitmap_equal(thread->mask->affinity.bits, map->affinity_mask.bits,
			  thread->mask->affinity.nbits)) {
		bitmap_zero(thread->mask->affinity.bits, thread->mask->affinity.nbits);
		bitmap_or(thread->mask->affinity.bits, thread->mask->affinity.bits,
			  map->affinity_mask.bits, thread->mask->affinity.nbits);
		sched_setaffinity(0, MMAP_CPU_MASK_BYTES(&thread->mask->affinity),
					(cpu_set_t *)thread->mask->affinity.bits);
		if (verbose == 2) {
			pr_debug("threads[%d]: running on cpu%d: ", thread->tid, sched_getcpu());
			mmap_cpu_mask__scnprintf(&thread->mask->affinity, "affinity");
		}
	}
}

static size_t process_comp_header(void *record, size_t increment)
{
	struct perf_record_compressed *event = record;
	size_t size = sizeof(*event);

	if (increment) {
		event->header.size += increment;
		return increment;
	}

	event->header.type = PERF_RECORD_COMPRESSED;
	event->header.size = size;

	return size;
}

static ssize_t zstd_compress(struct perf_session *session, struct mmap *map,
			    void *dst, size_t dst_size, void *src, size_t src_size)
{
	ssize_t compressed;
	size_t max_record_size = PERF_SAMPLE_MAX_SIZE - sizeof(struct perf_record_compressed) - 1;
	struct zstd_data *zstd_data = &session->zstd_data;

	if (map && map->file)
		zstd_data = &map->zstd_data;

	compressed = zstd_compress_stream_to_records(zstd_data, dst, dst_size, src, src_size,
						     max_record_size, process_comp_header);
	if (compressed < 0)
		return compressed;

	if (map && map->file) {
		thread->bytes_transferred += src_size;
		thread->bytes_compressed  += compressed;
	} else {
		session->bytes_transferred += src_size;
		session->bytes_compressed  += compressed;
	}

	return compressed;
}

static int record__mmap_read_evlist(struct record *rec, struct evlist *evlist,
				    bool overwrite, bool synch)
{
	u64 bytes_written = rec->bytes_written;
	int i;
	int rc = 0;
	int nr_mmaps;
	struct mmap **maps;
	int trace_fd = rec->data.file.fd;
	off_t off = 0;

	if (!evlist)
		return 0;

	nr_mmaps = thread->nr_mmaps;
	maps = overwrite ? thread->overwrite_maps : thread->maps;

	if (!maps)
		return 0;

	if (overwrite && evlist->bkw_mmap_state != BKW_MMAP_DATA_PENDING)
		return 0;

	if (record__aio_enabled(rec))
		off = record__aio_get_pos(trace_fd);

	for (i = 0; i < nr_mmaps; i++) {
		u64 flush = 0;
		struct mmap *map = maps[i];

		if (map->core.base) {
			record__adjust_affinity(rec, map);
			if (synch) {
				flush = map->core.flush;
				map->core.flush = 1;
			}
			if (!record__aio_enabled(rec)) {
				if (perf_mmap__push(map, rec, record__pushfn) < 0) {
					if (synch)
						map->core.flush = flush;
					rc = -1;
					goto out;
				}
			} else {
				if (record__aio_push(rec, map, &off) < 0) {
					record__aio_set_pos(trace_fd, off);
					if (synch)
						map->core.flush = flush;
					rc = -1;
					goto out;
				}
			}
			if (synch)
				map->core.flush = flush;
		}

		if (map->auxtrace_mmap.base && !rec->opts.auxtrace_snapshot_mode &&
		    !rec->opts.auxtrace_sample_mode &&
		    record__auxtrace_mmap_read(rec, map) != 0) {
			rc = -1;
			goto out;
		}
	}

	if (record__aio_enabled(rec))
		record__aio_set_pos(trace_fd, off);

	/*
	 * Mark the round finished in case we wrote
	 * at least one event.
	 *
	 * No need for round events in directory mode,
	 * because per-cpu maps and files have data
	 * sorted by kernel.
	 */
	if (!record__threads_enabled(rec) && bytes_written != rec->bytes_written)
		rc = record__write(rec, NULL, &finished_round_event, sizeof(finished_round_event));

	if (overwrite)
		evlist__toggle_bkw_mmap(evlist, BKW_MMAP_EMPTY);
out:
	return rc;
}

static int record__mmap_read_all(struct record *rec, bool synch)
{
	int err;

	err = record__mmap_read_evlist(rec, rec->evlist, false, synch);
	if (err)
		return err;

	return record__mmap_read_evlist(rec, rec->evlist, true, synch);
}

static void record__thread_munmap_filtered(struct fdarray *fda, int fd,
					   void *arg __maybe_unused)
{
	struct perf_mmap *map = fda->priv[fd].ptr;

	if (map)
		perf_mmap__put(map);
}

static void *record__thread(void *arg)
{
	enum thread_msg msg = THREAD_MSG__READY;
	bool terminate = false;
	struct fdarray *pollfd;
	int err, ctlfd_pos;

	thread = arg;
	thread->tid = gettid();

	err = write(thread->pipes.ack[1], &msg, sizeof(msg));
	if (err == -1)
		pr_warning("threads[%d]: failed to notify on start: %s\n",
			   thread->tid, strerror(errno));

	pr_debug("threads[%d]: started on cpu%d\n", thread->tid, sched_getcpu());

	pollfd = &thread->pollfd;
	ctlfd_pos = thread->ctlfd_pos;

	for (;;) {
		unsigned long long hits = thread->samples;

		if (record__mmap_read_all(thread->rec, false) < 0 || terminate)
			break;

		if (hits == thread->samples) {

			err = fdarray__poll(pollfd, -1);
			/*
			 * Propagate error, only if there's any. Ignore positive
			 * number of returned events and interrupt error.
			 */
			if (err > 0 || (err < 0 && errno == EINTR))
				err = 0;
			thread->waking++;

			if (fdarray__filter(pollfd, POLLERR | POLLHUP,
					    record__thread_munmap_filtered, NULL) == 0)
				break;
		}

		if (pollfd->entries[ctlfd_pos].revents & POLLHUP) {
			terminate = true;
			close(thread->pipes.msg[0]);
			thread->pipes.msg[0] = -1;
			pollfd->entries[ctlfd_pos].fd = -1;
			pollfd->entries[ctlfd_pos].events = 0;
		}

		pollfd->entries[ctlfd_pos].revents = 0;
	}
	record__mmap_read_all(thread->rec, true);

	err = write(thread->pipes.ack[1], &msg, sizeof(msg));
	if (err == -1)
		pr_warning("threads[%d]: failed to notify on termination: %s\n",
			   thread->tid, strerror(errno));

	return NULL;
}

static void record__init_features(struct record *rec)
{
	struct perf_session *session = rec->session;
	int feat;

	for (feat = HEADER_FIRST_FEATURE; feat < HEADER_LAST_FEATURE; feat++)
		perf_header__set_feat(&session->header, feat);

	if (rec->no_buildid)
		perf_header__clear_feat(&session->header, HEADER_BUILD_ID);

#ifdef HAVE_LIBTRACEEVENT
	if (!have_tracepoints(&rec->evlist->core.entries))
		perf_header__clear_feat(&session->header, HEADER_TRACING_DATA);
#endif

	if (!rec->opts.branch_stack)
		perf_header__clear_feat(&session->header, HEADER_BRANCH_STACK);

	if (!rec->opts.full_auxtrace)
		perf_header__clear_feat(&session->header, HEADER_AUXTRACE);

	if (!(rec->opts.use_clockid && rec->opts.clockid_res_ns))
		perf_header__clear_feat(&session->header, HEADER_CLOCKID);

	if (!rec->opts.use_clockid)
		perf_header__clear_feat(&session->header, HEADER_CLOCK_DATA);

	if (!record__threads_enabled(rec))
		perf_header__clear_feat(&session->header, HEADER_DIR_FORMAT);

	if (!record__comp_enabled(rec))
		perf_header__clear_feat(&session->header, HEADER_COMPRESSED);

	perf_header__clear_feat(&session->header, HEADER_STAT);
}

static void
record__finish_output(struct record *rec)
{
	int i;
	struct perf_data *data = &rec->data;
	int fd = perf_data__fd(data);

	if (data->is_pipe) {
		/* Just to display approx. size */
		data->file.size = rec->bytes_written;
		return;
	}

	rec->session->header.data_size += rec->bytes_written;
	data->file.size = lseek(perf_data__fd(data), 0, SEEK_CUR);
	if (record__threads_enabled(rec)) {
		for (i = 0; i < data->dir.nr; i++)
			data->dir.files[i].size = lseek(data->dir.files[i].fd, 0, SEEK_CUR);
	}

	if (!rec->no_buildid) {
		process_buildids(rec);

		if (rec->buildid_all)
			perf_session__dsos_hit_all(rec->session);
	}
	perf_session__write_header(rec->session, rec->evlist, fd, true);

	return;
}

static int record__synthesize_workload(struct record *rec, bool tail)
{
	int err;
	struct perf_thread_map *thread_map;
	bool needs_mmap = rec->opts.synth & PERF_SYNTH_MMAP;

	if (rec->opts.tail_synthesize != tail)
		return 0;

	thread_map = thread_map__new_by_tid(rec->evlist->workload.pid);
	if (thread_map == NULL)
		return -1;

	err = perf_event__synthesize_thread_map(&rec->tool, thread_map,
						 process_synthesized_event,
						 &rec->session->machines.host,
						 needs_mmap,
						 rec->opts.sample_address);
	perf_thread_map__put(thread_map);
	return err;
}

static int write_finished_init(struct record *rec, bool tail)
{
	if (rec->opts.tail_synthesize != tail)
		return 0;

	return record__write(rec, NULL, &finished_init_event, sizeof(finished_init_event));
}

static int record__synthesize(struct record *rec, bool tail);

static int
record__switch_output(struct record *rec, bool at_exit)
{
	struct perf_data *data = &rec->data;
	char *new_filename = NULL;
	int fd, err;

	/* Same Size:      "2015122520103046"*/
	char timestamp[] = "InvalidTimestamp";

	record__aio_mmap_read_sync(rec);

	write_finished_init(rec, true);

	record__synthesize(rec, true);
	if (target__none(&rec->opts.target))
		record__synthesize_workload(rec, true);

	rec->samples = 0;
	record__finish_output(rec);
	err = fetch_current_timestamp(timestamp, sizeof(timestamp));
	if (err) {
		pr_err("Failed to get current timestamp\n");
		return -EINVAL;
	}

	fd = perf_data__switch(data, timestamp,
			       rec->session->header.data_offset,
			       at_exit, &new_filename);
	if (fd >= 0 && !at_exit) {
		rec->bytes_written = 0;
		rec->session->header.data_size = 0;
	}

	if (!quiet) {
		fprintf(stderr, "[ perf record: Dump %s.%s ]\n",
			data->path, timestamp);
	}

	if (rec->switch_output.num_files) {
		int n = rec->switch_output.cur_file + 1;

		if (n >= rec->switch_output.num_files)
			n = 0;
		rec->switch_output.cur_file = n;
		if (rec->switch_output.filenames[n]) {
			remove(rec->switch_output.filenames[n]);
			zfree(&rec->switch_output.filenames[n]);
		}
		rec->switch_output.filenames[n] = new_filename;
	} else {
		free(new_filename);
	}

	/* Output tracking events */
	if (!at_exit) {
		record__synthesize(rec, false);

		/*
		 * In 'perf record --switch-output' without -a,
		 * record__synthesize() in record__switch_output() won't
		 * generate tracking events because there's no thread_map
		 * in evlist. Which causes newly created perf.data doesn't
		 * contain map and comm information.
		 * Create a fake thread_map and directly call
		 * perf_event__synthesize_thread_map() for those events.
		 */
		if (target__none(&rec->opts.target))
			record__synthesize_workload(rec, false);
		write_finished_init(rec, false);
	}
	return fd;
}

static void __record__save_lost_samples(struct record *rec, struct evsel *evsel,
					struct perf_record_lost_samples *lost,
					int cpu_idx, int thread_idx, u64 lost_count,
					u16 misc_flag)
{
	struct perf_sample_id *sid;
	struct perf_sample sample = {};
	int id_hdr_size;

	lost->lost = lost_count;
	if (evsel->core.ids) {
		sid = xyarray__entry(evsel->core.sample_id, cpu_idx, thread_idx);
		sample.id = sid->id;
	}

	id_hdr_size = perf_event__synthesize_id_sample((void *)(lost + 1),
						       evsel->core.attr.sample_type, &sample);
	lost->header.size = sizeof(*lost) + id_hdr_size;
	lost->header.misc = misc_flag;
	record__write(rec, NULL, lost, lost->header.size);
}

static void record__read_lost_samples(struct record *rec)
{
	struct perf_session *session = rec->session;
	struct perf_record_lost_samples_and_ids lost;
	struct evsel *evsel;

	/* there was an error during record__open */
	if (session->evlist == NULL)
		return;

	evlist__for_each_entry(session->evlist, evsel) {
		struct xyarray *xy = evsel->core.sample_id;
		u64 lost_count;

		if (xy == NULL || evsel->core.fd == NULL)
			continue;
		if (xyarray__max_x(evsel->core.fd) != xyarray__max_x(xy) ||
		    xyarray__max_y(evsel->core.fd) != xyarray__max_y(xy)) {
			pr_debug("Unmatched FD vs. sample ID: skip reading LOST count\n");
			continue;
		}

		for (int x = 0; x < xyarray__max_x(xy); x++) {
			for (int y = 0; y < xyarray__max_y(xy); y++) {
				struct perf_counts_values count;

				if (perf_evsel__read(&evsel->core, x, y, &count) < 0) {
					pr_debug("read LOST count failed\n");
					return;
				}

				if (count.lost) {
					memset(&lost, 0, sizeof(lost));
					lost.lost.header.type = PERF_RECORD_LOST_SAMPLES;
					__record__save_lost_samples(rec, evsel, &lost.lost,
								    x, y, count.lost, 0);
				}
			}
		}

		lost_count = perf_bpf_filter__lost_count(evsel);
		if (lost_count) {
			memset(&lost, 0, sizeof(lost));
			lost.lost.header.type = PERF_RECORD_LOST_SAMPLES;
			__record__save_lost_samples(rec, evsel, &lost.lost, 0, 0, lost_count,
						    PERF_RECORD_MISC_LOST_SAMPLES_BPF);
		}
	}
}

static volatile sig_atomic_t workload_exec_errno;

/*
 * evlist__prepare_workload will send a SIGUSR1
 * if the fork fails, since we asked by setting its
 * want_signal to true.
 */
static void workload_exec_failed_signal(int signo __maybe_unused,
					siginfo_t *info,
					void *ucontext __maybe_unused)
{
	workload_exec_errno = info->si_value.sival_int;
	done = 1;
	child_finished = 1;
}

static void snapshot_sig_handler(int sig);
static void alarm_sig_handler(int sig);

static const struct perf_event_mmap_page *evlist__pick_pc(struct evlist *evlist)
{
	if (evlist) {
		if (evlist->mmap && evlist->mmap[0].core.base)
			return evlist->mmap[0].core.base;
		if (evlist->overwrite_mmap && evlist->overwrite_mmap[0].core.base)
			return evlist->overwrite_mmap[0].core.base;
	}
	return NULL;
}

static const struct perf_event_mmap_page *record__pick_pc(struct record *rec)
{
	const struct perf_event_mmap_page *pc = evlist__pick_pc(rec->evlist);
	if (pc)
		return pc;
	return NULL;
}

static int record__synthesize(struct record *rec, bool tail)
{
	struct perf_session *session = rec->session;
	struct machine *machine = &session->machines.host;
	struct perf_data *data = &rec->data;
	struct record_opts *opts = &rec->opts;
	struct perf_tool *tool = &rec->tool;
	int err = 0;
	event_op f = process_synthesized_event;

	if (rec->opts.tail_synthesize != tail)
		return 0;

	if (data->is_pipe) {
		err = perf_event__synthesize_for_pipe(tool, session, data,
						      process_synthesized_event);
		if (err < 0)
			goto out;

		rec->bytes_written += err;
	}

	err = perf_event__synth_time_conv(record__pick_pc(rec), tool,
					  process_synthesized_event, machine);
	if (err)
		goto out;

	/* Synthesize id_index before auxtrace_info */
	err = perf_event__synthesize_id_index(tool,
					      process_synthesized_event,
					      session->evlist, machine);
	if (err)
		goto out;

	if (rec->opts.full_auxtrace) {
		err = perf_event__synthesize_auxtrace_info(rec->itr, tool,
					session, process_synthesized_event);
		if (err)
			goto out;
	}

	if (!evlist__exclude_kernel(rec->evlist)) {
		err = perf_event__synthesize_kernel_mmap(tool, process_synthesized_event,
							 machine);
		WARN_ONCE(err < 0, "Couldn't record kernel reference relocation symbol\n"
				   "Symbol resolution may be skewed if relocation was used (e.g. kexec).\n"
				   "Check /proc/kallsyms permission or run as root.\n");

		err = perf_event__synthesize_modules(tool, process_synthesized_event,
						     machine);
		WARN_ONCE(err < 0, "Couldn't record kernel module information.\n"
				   "Symbol resolution may be skewed if relocation was used (e.g. kexec).\n"
				   "Check /proc/modules permission or run as root.\n");
	}

	if (perf_guest) {
		machines__process_guests(&session->machines,
					 perf_event__synthesize_guest_os, tool);
	}

	err = perf_event__synthesize_extra_attr(&rec->tool,
						rec->evlist,
						process_synthesized_event,
						data->is_pipe);
	if (err)
		goto out;

	err = perf_event__synthesize_thread_map2(&rec->tool, rec->evlist->core.threads,
						 process_synthesized_event,
						NULL);
	if (err < 0) {
		pr_err("Couldn't synthesize thread map.\n");
		return err;
	}

	err = perf_event__synthesize_cpu_map(&rec->tool, rec->evlist->core.all_cpus,
					     process_synthesized_event, NULL);
	if (err < 0) {
		pr_err("Couldn't synthesize cpu map.\n");
		return err;
	}

	err = perf_event__synthesize_bpf_events(session, process_synthesized_event,
						machine, opts);
	if (err < 0) {
		pr_warning("Couldn't synthesize bpf events.\n");
		err = 0;
	}

	if (rec->opts.synth & PERF_SYNTH_CGROUP) {
		err = perf_event__synthesize_cgroups(tool, process_synthesized_event,
						     machine);
		if (err < 0) {
			pr_warning("Couldn't synthesize cgroup events.\n");
			err = 0;
		}
	}

	if (rec->opts.nr_threads_synthesize > 1) {
		mutex_init(&synth_lock);
		perf_set_multithreaded();
		f = process_locked_synthesized_event;
	}

	if (rec->opts.synth & PERF_SYNTH_TASK) {
		bool needs_mmap = rec->opts.synth & PERF_SYNTH_MMAP;

		err = __machine__synthesize_threads(machine, tool, &opts->target,
						    rec->evlist->core.threads,
						    f, needs_mmap, opts->sample_address,
						    rec->opts.nr_threads_synthesize);
	}

	if (rec->opts.nr_threads_synthesize > 1) {
		perf_set_singlethreaded();
		mutex_destroy(&synth_lock);
	}

out:
	return err;
}

static int record__process_signal_event(union perf_event *event __maybe_unused, void *data)
{
	struct record *rec = data;
	pthread_kill(rec->thread_id, SIGUSR2);
	return 0;
}

static int record__setup_sb_evlist(struct record *rec)
{
	struct record_opts *opts = &rec->opts;

	if (rec->sb_evlist != NULL) {
		/*
		 * We get here if --switch-output-event populated the
		 * sb_evlist, so associate a callback that will send a SIGUSR2
		 * to the main thread.
		 */
		evlist__set_cb(rec->sb_evlist, record__process_signal_event, rec);
		rec->thread_id = pthread_self();
	}
#ifdef HAVE_LIBBPF_SUPPORT
	if (!opts->no_bpf_event) {
		if (rec->sb_evlist == NULL) {
			rec->sb_evlist = evlist__new();

			if (rec->sb_evlist == NULL) {
				pr_err("Couldn't create side band evlist.\n.");
				return -1;
			}
		}

		if (evlist__add_bpf_sb_event(rec->sb_evlist, &rec->session->header.env)) {
			pr_err("Couldn't ask for PERF_RECORD_BPF_EVENT side band events.\n.");
			return -1;
		}
	}
#endif
	if (evlist__start_sb_thread(rec->sb_evlist, &rec->opts.target)) {
		pr_debug("Couldn't start the BPF side band thread:\nBPF programs starting from now on won't be annotatable\n");
		opts->no_bpf_event = true;
	}

	return 0;
}

static int record__init_clock(struct record *rec)
{
	struct perf_session *session = rec->session;
	struct timespec ref_clockid;
	struct timeval ref_tod;
	u64 ref;

	if (!rec->opts.use_clockid)
		return 0;

	if (rec->opts.use_clockid && rec->opts.clockid_res_ns)
		session->header.env.clock.clockid_res_ns = rec->opts.clockid_res_ns;

	session->header.env.clock.clockid = rec->opts.clockid;

	if (gettimeofday(&ref_tod, NULL) != 0) {
		pr_err("gettimeofday failed, cannot set reference time.\n");
		return -1;
	}

	if (clock_gettime(rec->opts.clockid, &ref_clockid)) {
		pr_err("clock_gettime failed, cannot set reference time.\n");
		return -1;
	}

	ref = (u64) ref_tod.tv_sec * NSEC_PER_SEC +
	      (u64) ref_tod.tv_usec * NSEC_PER_USEC;

	session->header.env.clock.tod_ns = ref;

	ref = (u64) ref_clockid.tv_sec * NSEC_PER_SEC +
	      (u64) ref_clockid.tv_nsec;

	session->header.env.clock.clockid_ns = ref;
	return 0;
}

static void hit_auxtrace_snapshot_trigger(struct record *rec)
{
	if (trigger_is_ready(&auxtrace_snapshot_trigger)) {
		trigger_hit(&auxtrace_snapshot_trigger);
		auxtrace_record__snapshot_started = 1;
		if (auxtrace_record__snapshot_start(rec->itr))
			trigger_error(&auxtrace_snapshot_trigger);
	}
}

static int record__terminate_thread(struct record_thread *thread_data)
{
	int err;
	enum thread_msg ack = THREAD_MSG__UNDEFINED;
	pid_t tid = thread_data->tid;

	close(thread_data->pipes.msg[1]);
	thread_data->pipes.msg[1] = -1;
	err = read(thread_data->pipes.ack[0], &ack, sizeof(ack));
	if (err > 0)
		pr_debug2("threads[%d]: sent %s\n", tid, thread_msg_tags[ack]);
	else
		pr_warning("threads[%d]: failed to receive termination notification from %d\n",
			   thread->tid, tid);

	return 0;
}

static int record__start_threads(struct record *rec)
{
	int t, tt, err, ret = 0, nr_threads = rec->nr_threads;
	struct record_thread *thread_data = rec->thread_data;
	sigset_t full, mask;
	pthread_t handle;
	pthread_attr_t attrs;

	thread = &thread_data[0];

	if (!record__threads_enabled(rec))
		return 0;

	sigfillset(&full);
	if (sigprocmask(SIG_SETMASK, &full, &mask)) {
		pr_err("Failed to block signals on threads start: %s\n", strerror(errno));
		return -1;
	}

	pthread_attr_init(&attrs);
	pthread_attr_setdetachstate(&attrs, PTHREAD_CREATE_DETACHED);

	for (t = 1; t < nr_threads; t++) {
		enum thread_msg msg = THREAD_MSG__UNDEFINED;

#ifdef HAVE_PTHREAD_ATTR_SETAFFINITY_NP
		pthread_attr_setaffinity_np(&attrs,
					    MMAP_CPU_MASK_BYTES(&(thread_data[t].mask->affinity)),
					    (cpu_set_t *)(thread_data[t].mask->affinity.bits));
#endif
		if (pthread_create(&handle, &attrs, record__thread, &thread_data[t])) {
			for (tt = 1; tt < t; tt++)
				record__terminate_thread(&thread_data[t]);
			pr_err("Failed to start threads: %s\n", strerror(errno));
			ret = -1;
			goto out_err;
		}

		err = read(thread_data[t].pipes.ack[0], &msg, sizeof(msg));
		if (err > 0)
			pr_debug2("threads[%d]: sent %s\n", rec->thread_data[t].tid,
				  thread_msg_tags[msg]);
		else
			pr_warning("threads[%d]: failed to receive start notification from %d\n",
				   thread->tid, rec->thread_data[t].tid);
	}

	sched_setaffinity(0, MMAP_CPU_MASK_BYTES(&thread->mask->affinity),
			(cpu_set_t *)thread->mask->affinity.bits);

	pr_debug("threads[%d]: started on cpu%d\n", thread->tid, sched_getcpu());

out_err:
	pthread_attr_destroy(&attrs);

	if (sigprocmask(SIG_SETMASK, &mask, NULL)) {
		pr_err("Failed to unblock signals on threads start: %s\n", strerror(errno));
		ret = -1;
	}

	return ret;
}

static int record__stop_threads(struct record *rec)
{
	int t;
	struct record_thread *thread_data = rec->thread_data;

	for (t = 1; t < rec->nr_threads; t++)
		record__terminate_thread(&thread_data[t]);

	for (t = 0; t < rec->nr_threads; t++) {
		rec->samples += thread_data[t].samples;
		if (!record__threads_enabled(rec))
			continue;
		rec->session->bytes_transferred += thread_data[t].bytes_transferred;
		rec->session->bytes_compressed += thread_data[t].bytes_compressed;
		pr_debug("threads[%d]: samples=%lld, wakes=%ld, ", thread_data[t].tid,
			 thread_data[t].samples, thread_data[t].waking);
		if (thread_data[t].bytes_transferred && thread_data[t].bytes_compressed)
			pr_debug("transferred=%" PRIu64 ", compressed=%" PRIu64 "\n",
				 thread_data[t].bytes_transferred, thread_data[t].bytes_compressed);
		else
			pr_debug("written=%" PRIu64 "\n", thread_data[t].bytes_written);
	}

	return 0;
}

static unsigned long record__waking(struct record *rec)
{
	int t;
	unsigned long waking = 0;
	struct record_thread *thread_data = rec->thread_data;

	for (t = 0; t < rec->nr_threads; t++)
		waking += thread_data[t].waking;

	return waking;
}

static int __cmd_record(struct record *rec, int argc, const char **argv)
{
	int err;
	int status = 0;
	const bool forks = argc > 0;
	struct perf_tool *tool = &rec->tool;
	struct record_opts *opts = &rec->opts;
	struct perf_data *data = &rec->data;
	struct perf_session *session;
	bool disabled = false, draining = false;
	int fd;
	float ratio = 0;
	enum evlist_ctl_cmd cmd = EVLIST_CTL_CMD_UNSUPPORTED;

	atexit(record__sig_exit);
	signal(SIGCHLD, sig_handler);
	signal(SIGINT, sig_handler);
	signal(SIGTERM, sig_handler);
	signal(SIGSEGV, sigsegv_handler);

	if (rec->opts.record_namespaces)
		tool->namespace_events = true;

	if (rec->opts.record_cgroup) {
#ifdef HAVE_FILE_HANDLE
		tool->cgroup_events = true;
#else
		pr_err("cgroup tracking is not supported\n");
		return -1;
#endif
	}

	if (rec->opts.auxtrace_snapshot_mode || rec->switch_output.enabled) {
		signal(SIGUSR2, snapshot_sig_handler);
		if (rec->opts.auxtrace_snapshot_mode)
			trigger_on(&auxtrace_snapshot_trigger);
		if (rec->switch_output.enabled)
			trigger_on(&switch_output_trigger);
	} else {
		signal(SIGUSR2, SIG_IGN);
	}

	session = perf_session__new(data, tool);
	if (IS_ERR(session)) {
		pr_err("Perf session creation failed.\n");
		return PTR_ERR(session);
	}

	if (record__threads_enabled(rec)) {
		if (perf_data__is_pipe(&rec->data)) {
			pr_err("Parallel trace streaming is not available in pipe mode.\n");
			return -1;
		}
		if (rec->opts.full_auxtrace) {
			pr_err("Parallel trace streaming is not available in AUX area tracing mode.\n");
			return -1;
		}
	}

	fd = perf_data__fd(data);
	rec->session = session;

	if (zstd_init(&session->zstd_data, rec->opts.comp_level) < 0) {
		pr_err("Compression initialization failed.\n");
		return -1;
	}
#ifdef HAVE_EVENTFD_SUPPORT
	done_fd = eventfd(0, EFD_NONBLOCK);
	if (done_fd < 0) {
		pr_err("Failed to create wakeup eventfd, error: %m\n");
		status = -1;
		goto out_delete_session;
	}
	err = evlist__add_wakeup_eventfd(rec->evlist, done_fd);
	if (err < 0) {
		pr_err("Failed to add wakeup eventfd to poll list\n");
		status = err;
		goto out_delete_session;
	}
#endif // HAVE_EVENTFD_SUPPORT

	session->header.env.comp_type  = PERF_COMP_ZSTD;
	session->header.env.comp_level = rec->opts.comp_level;

	if (rec->opts.kcore &&
	    !record__kcore_readable(&session->machines.host)) {
		pr_err("ERROR: kcore is not readable.\n");
		return -1;
	}

	if (record__init_clock(rec))
		return -1;

	record__init_features(rec);

	if (forks) {
		err = evlist__prepare_workload(rec->evlist, &opts->target, argv, data->is_pipe,
					       workload_exec_failed_signal);
		if (err < 0) {
			pr_err("Couldn't run the workload!\n");
			status = err;
			goto out_delete_session;
		}
	}

	/*
	 * If we have just single event and are sending data
	 * through pipe, we need to force the ids allocation,
	 * because we synthesize event name through the pipe
	 * and need the id for that.
	 */
	if (data->is_pipe && rec->evlist->core.nr_entries == 1)
		rec->opts.sample_id = true;

	if (rec->timestamp_filename && perf_data__is_pipe(data)) {
		rec->timestamp_filename = false;
		pr_warning("WARNING: --timestamp-filename option is not available in pipe mode.\n");
	}

	evlist__uniquify_name(rec->evlist);

	evlist__config(rec->evlist, opts, &callchain_param);

	/* Debug message used by test scripts */
	pr_debug3("perf record opening and mmapping events\n");
	if (record__open(rec) != 0) {
		err = -1;
		goto out_free_threads;
	}
	/* Debug message used by test scripts */
	pr_debug3("perf record done opening and mmapping events\n");
	session->header.env.comp_mmap_len = session->evlist->core.mmap_len;

	if (rec->opts.kcore) {
		err = record__kcore_copy(&session->machines.host, data);
		if (err) {
			pr_err("ERROR: Failed to copy kcore\n");
			goto out_free_threads;
		}
	}

	/*
	 * Normally perf_session__new would do this, but it doesn't have the
	 * evlist.
	 */
	if (rec->tool.ordered_events && !evlist__sample_id_all(rec->evlist)) {
		pr_warning("WARNING: No sample_id_all support, falling back to unordered processing\n");
		rec->tool.ordered_events = false;
	}

	if (evlist__nr_groups(rec->evlist) == 0)
		perf_header__clear_feat(&session->header, HEADER_GROUP_DESC);

	if (data->is_pipe) {
		err = perf_header__write_pipe(fd);
		if (err < 0)
			goto out_free_threads;
	} else {
		err = perf_session__write_header(session, rec->evlist, fd, false);
		if (err < 0)
			goto out_free_threads;
	}

	err = -1;
	if (!rec->no_buildid
	    && !perf_header__has_feat(&session->header, HEADER_BUILD_ID)) {
		pr_err("Couldn't generate buildids. "
		       "Use --no-buildid to profile anyway.\n");
		goto out_free_threads;
	}

	err = record__setup_sb_evlist(rec);
	if (err)
		goto out_free_threads;

	err = record__synthesize(rec, false);
	if (err < 0)
		goto out_free_threads;

	if (rec->realtime_prio) {
		struct sched_param param;

		param.sched_priority = rec->realtime_prio;
		if (sched_setscheduler(0, SCHED_FIFO, &param)) {
			pr_err("Could not set realtime priority.\n");
			err = -1;
			goto out_free_threads;
		}
	}

	if (record__start_threads(rec))
		goto out_free_threads;

	/*
	 * When perf is starting the traced process, all the events
	 * (apart from group members) have enable_on_exec=1 set,
	 * so don't spoil it by prematurely enabling them.
	 */
	if (!target__none(&opts->target) && !opts->target.initial_delay)
		evlist__enable(rec->evlist);

	/*
	 * Let the child rip
	 */
	if (forks) {
		struct machine *machine = &session->machines.host;
		union perf_event *event;
		pid_t tgid;

		event = malloc(sizeof(event->comm) + machine->id_hdr_size);
		if (event == NULL) {
			err = -ENOMEM;
			goto out_child;
		}

		/*
		 * Some H/W events are generated before COMM event
		 * which is emitted during exec(), so perf script
		 * cannot see a correct process name for those events.
		 * Synthesize COMM event to prevent it.
		 */
		tgid = perf_event__synthesize_comm(tool, event,
						   rec->evlist->workload.pid,
						   process_synthesized_event,
						   machine);
		free(event);

		if (tgid == -1)
			goto out_child;

		event = malloc(sizeof(event->namespaces) +
			       (NR_NAMESPACES * sizeof(struct perf_ns_link_info)) +
			       machine->id_hdr_size);
		if (event == NULL) {
			err = -ENOMEM;
			goto out_child;
		}

		/*
		 * Synthesize NAMESPACES event for the command specified.
		 */
		perf_event__synthesize_namespaces(tool, event,
						  rec->evlist->workload.pid,
						  tgid, process_synthesized_event,
						  machine);
		free(event);

		evlist__start_workload(rec->evlist);
	}

	if (opts->target.initial_delay) {
		pr_info(EVLIST_DISABLED_MSG);
		if (opts->target.initial_delay > 0) {
			usleep(opts->target.initial_delay * USEC_PER_MSEC);
			evlist__enable(rec->evlist);
			pr_info(EVLIST_ENABLED_MSG);
		}
	}

	err = event_enable_timer__start(rec->evlist->eet);
	if (err)
		goto out_child;

	/* Debug message used by test scripts */
	pr_debug3("perf record has started\n");
	fflush(stderr);

	trigger_ready(&auxtrace_snapshot_trigger);
	trigger_ready(&switch_output_trigger);
	perf_hooks__invoke_record_start();

	/*
	 * Must write FINISHED_INIT so it will be seen after all other
	 * synthesized user events, but before any regular events.
	 */
	err = write_finished_init(rec, false);
	if (err < 0)
		goto out_child;

	for (;;) {
		unsigned long long hits = thread->samples;

		/*
		 * rec->evlist->bkw_mmap_state is possible to be
		 * BKW_MMAP_EMPTY here: when done == true and
		 * hits != rec->samples in previous round.
		 *
		 * evlist__toggle_bkw_mmap ensure we never
		 * convert BKW_MMAP_EMPTY to BKW_MMAP_DATA_PENDING.
		 */
		if (trigger_is_hit(&switch_output_trigger) || done || draining)
			evlist__toggle_bkw_mmap(rec->evlist, BKW_MMAP_DATA_PENDING);

		if (record__mmap_read_all(rec, false) < 0) {
			trigger_error(&auxtrace_snapshot_trigger);
			trigger_error(&switch_output_trigger);
			err = -1;
			goto out_child;
		}

		if (auxtrace_record__snapshot_started) {
			auxtrace_record__snapshot_started = 0;
			if (!trigger_is_error(&auxtrace_snapshot_trigger))
				record__read_auxtrace_snapshot(rec, false);
			if (trigger_is_error(&auxtrace_snapshot_trigger)) {
				pr_err("AUX area tracing snapshot failed\n");
				err = -1;
				goto out_child;
			}
		}

		if (trigger_is_hit(&switch_output_trigger)) {
			/*
			 * If switch_output_trigger is hit, the data in
			 * overwritable ring buffer should have been collected,
			 * so bkw_mmap_state should be set to BKW_MMAP_EMPTY.
			 *
			 * If SIGUSR2 raise after or during record__mmap_read_all(),
			 * record__mmap_read_all() didn't collect data from
			 * overwritable ring buffer. Read again.
			 */
			if (rec->evlist->bkw_mmap_state == BKW_MMAP_RUNNING)
				continue;
			trigger_ready(&switch_output_trigger);

			/*
			 * Reenable events in overwrite ring buffer after
			 * record__mmap_read_all(): we should have collected
			 * data from it.
			 */
			evlist__toggle_bkw_mmap(rec->evlist, BKW_MMAP_RUNNING);

			if (!quiet)
				fprintf(stderr, "[ perf record: dump data: Woken up %ld times ]\n",
					record__waking(rec));
			thread->waking = 0;
			fd = record__switch_output(rec, false);
			if (fd < 0) {
				pr_err("Failed to switch to new file\n");
				trigger_error(&switch_output_trigger);
				err = fd;
				goto out_child;
			}

			/* re-arm the alarm */
			if (rec->switch_output.time)
				alarm(rec->switch_output.time);
		}

		if (hits == thread->samples) {
			if (done || draining)
				break;
			err = fdarray__poll(&thread->pollfd, -1);
			/*
			 * Propagate error, only if there's any. Ignore positive
			 * number of returned events and interrupt error.
			 */
			if (err > 0 || (err < 0 && errno == EINTR))
				err = 0;
			thread->waking++;

			if (fdarray__filter(&thread->pollfd, POLLERR | POLLHUP,
					    record__thread_munmap_filtered, NULL) == 0)
				draining = true;

			err = record__update_evlist_pollfd_from_thread(rec, rec->evlist, thread);
			if (err)
				goto out_child;
		}

		if (evlist__ctlfd_process(rec->evlist, &cmd) > 0) {
			switch (cmd) {
			case EVLIST_CTL_CMD_SNAPSHOT:
				hit_auxtrace_snapshot_trigger(rec);
				evlist__ctlfd_ack(rec->evlist);
				break;
			case EVLIST_CTL_CMD_STOP:
				done = 1;
				break;
			case EVLIST_CTL_CMD_ACK:
			case EVLIST_CTL_CMD_UNSUPPORTED:
			case EVLIST_CTL_CMD_ENABLE:
			case EVLIST_CTL_CMD_DISABLE:
			case EVLIST_CTL_CMD_EVLIST:
			case EVLIST_CTL_CMD_PING:
			default:
				break;
			}
		}

		err = event_enable_timer__process(rec->evlist->eet);
		if (err < 0)
			goto out_child;
		if (err) {
			err = 0;
			done = 1;
		}

		/*
		 * When perf is starting the traced process, at the end events
		 * die with the process and we wait for that. Thus no need to
		 * disable events in this case.
		 */
		if (done && !disabled && !target__none(&opts->target)) {
			trigger_off(&auxtrace_snapshot_trigger);
			evlist__disable(rec->evlist);
			disabled = true;
		}
	}

	trigger_off(&auxtrace_snapshot_trigger);
	trigger_off(&switch_output_trigger);

	if (opts->auxtrace_snapshot_on_exit)
		record__auxtrace_snapshot_exit(rec);

	if (forks && workload_exec_errno) {
		char msg[STRERR_BUFSIZE], strevsels[2048];
		const char *emsg = str_error_r(workload_exec_errno, msg, sizeof(msg));

		evlist__scnprintf_evsels(rec->evlist, sizeof(strevsels), strevsels);

		pr_err("Failed to collect '%s' for the '%s' workload: %s\n",
			strevsels, argv[0], emsg);
		err = -1;
		goto out_child;
	}

	if (!quiet)
		fprintf(stderr, "[ perf record: Woken up %ld times to write data ]\n",
			record__waking(rec));

	write_finished_init(rec, true);

	if (target__none(&rec->opts.target))
		record__synthesize_workload(rec, true);

out_child:
	record__stop_threads(rec);
	record__mmap_read_all(rec, true);
out_free_threads:
	record__free_thread_data(rec);
	evlist__finalize_ctlfd(rec->evlist);
	record__aio_mmap_read_sync(rec);

	if (rec->session->bytes_transferred && rec->session->bytes_compressed) {
		ratio = (float)rec->session->bytes_transferred/(float)rec->session->bytes_compressed;
		session->header.env.comp_ratio = ratio + 0.5;
	}

	if (forks) {
		int exit_status;

		if (!child_finished)
			kill(rec->evlist->workload.pid, SIGTERM);

		wait(&exit_status);

		if (err < 0)
			status = err;
		else if (WIFEXITED(exit_status))
			status = WEXITSTATUS(exit_status);
		else if (WIFSIGNALED(exit_status))
			signr = WTERMSIG(exit_status);
	} else
		status = err;

	if (rec->off_cpu)
		rec->bytes_written += off_cpu_write(rec->session);

	record__read_lost_samples(rec);
	record__synthesize(rec, true);
	/* this will be recalculated during process_buildids() */
	rec->samples = 0;

	if (!err) {
		if (!rec->timestamp_filename) {
			record__finish_output(rec);
		} else {
			fd = record__switch_output(rec, true);
			if (fd < 0) {
				status = fd;
				goto out_delete_session;
			}
		}
	}

	perf_hooks__invoke_record_end();

	if (!err && !quiet) {
		char samples[128];
		const char *postfix = rec->timestamp_filename ?
					".<timestamp>" : "";

		if (rec->samples && !rec->opts.full_auxtrace)
			scnprintf(samples, sizeof(samples),
				  " (%" PRIu64 " samples)", rec->samples);
		else
			samples[0] = '\0';

		fprintf(stderr,	"[ perf record: Captured and wrote %.3f MB %s%s%s",
			perf_data__size(data) / 1024.0 / 1024.0,
			data->path, postfix, samples);
		if (ratio) {
			fprintf(stderr,	", compressed (original %.3f MB, ratio is %.3f)",
					rec->session->bytes_transferred / 1024.0 / 1024.0,
					ratio);
		}
		fprintf(stderr, " ]\n");
	}

out_delete_session:
#ifdef HAVE_EVENTFD_SUPPORT
	if (done_fd >= 0) {
		fd = done_fd;
		done_fd = -1;

		close(fd);
	}
#endif
	zstd_fini(&session->zstd_data);
	if (!opts->no_bpf_event)
		evlist__stop_sb_thread(rec->sb_evlist);

	perf_session__delete(session);
	return status;
}

static void callchain_debug(struct callchain_param *callchain)
{
	static const char *str[CALLCHAIN_MAX] = { "NONE", "FP", "DWARF", "LBR" };

	pr_debug("callchain: type %s\n", str[callchain->record_mode]);

	if (callchain->record_mode == CALLCHAIN_DWARF)
		pr_debug("callchain: stack dump size %d\n",
			 callchain->dump_size);
}

int record_opts__parse_callchain(struct record_opts *record,
				 struct callchain_param *callchain,
				 const char *arg, bool unset)
{
	int ret;
	callchain->enabled = !unset;

	/* --no-call-graph */
	if (unset) {
		callchain->record_mode = CALLCHAIN_NONE;
		pr_debug("callchain: disabled\n");
		return 0;
	}

	ret = parse_callchain_record_opt(arg, callchain);
	if (!ret) {
		/* Enable data address sampling for DWARF unwind. */
		if (callchain->record_mode == CALLCHAIN_DWARF)
			record->sample_address = true;
		callchain_debug(callchain);
	}

	return ret;
}

int record_parse_callchain_opt(const struct option *opt,
			       const char *arg,
			       int unset)
{
	return record_opts__parse_callchain(opt->value, &callchain_param, arg, unset);
}

int record_callchain_opt(const struct option *opt,
			 const char *arg __maybe_unused,
			 int unset __maybe_unused)
{
	struct callchain_param *callchain = opt->value;

	callchain->enabled = true;

	if (callchain->record_mode == CALLCHAIN_NONE)
		callchain->record_mode = CALLCHAIN_FP;

	callchain_debug(callchain);
	return 0;
}

static int perf_record_config(const char *var, const char *value, void *cb)
{
	struct record *rec = cb;

	if (!strcmp(var, "record.build-id")) {
		if (!strcmp(value, "cache"))
			rec->no_buildid_cache = false;
		else if (!strcmp(value, "no-cache"))
			rec->no_buildid_cache = true;
		else if (!strcmp(value, "skip"))
			rec->no_buildid = true;
		else if (!strcmp(value, "mmap"))
			rec->buildid_mmap = true;
		else
			return -1;
		return 0;
	}
	if (!strcmp(var, "record.call-graph")) {
		var = "call-graph.record-mode";
		return perf_default_config(var, value, cb);
	}
#ifdef HAVE_AIO_SUPPORT
	if (!strcmp(var, "record.aio")) {
		rec->opts.nr_cblocks = strtol(value, NULL, 0);
		if (!rec->opts.nr_cblocks)
			rec->opts.nr_cblocks = nr_cblocks_default;
	}
#endif
	if (!strcmp(var, "record.debuginfod")) {
		rec->debuginfod.urls = strdup(value);
		if (!rec->debuginfod.urls)
			return -ENOMEM;
		rec->debuginfod.set = true;
	}

	return 0;
}

static int record__parse_event_enable_time(const struct option *opt, const char *str, int unset)
{
	struct record *rec = (struct record *)opt->value;

	return evlist__parse_event_enable_time(rec->evlist, &rec->opts, str, unset);
}

static int record__parse_affinity(const struct option *opt, const char *str, int unset)
{
	struct record_opts *opts = (struct record_opts *)opt->value;

	if (unset || !str)
		return 0;

	if (!strcasecmp(str, "node"))
		opts->affinity = PERF_AFFINITY_NODE;
	else if (!strcasecmp(str, "cpu"))
		opts->affinity = PERF_AFFINITY_CPU;

	return 0;
}

static int record__mmap_cpu_mask_alloc(struct mmap_cpu_mask *mask, int nr_bits)
{
	mask->nbits = nr_bits;