cregit-Linux how code gets into the kernel

Release 4.10 tools/perf/builtin-record.c

Directory: tools/perf
/*
 * builtin-record.c
 *
 * Builtin record command: Record the profile of a workload
 * (or a CPU, or a PID) into the perf.data output file - for
 * later analysis via perf report.
 */
#include "builtin.h"

#include "perf.h"

#include "util/build-id.h"
#include "util/util.h"
#include <subcmd/parse-options.h>
#include "util/parse-events.h"
#include "util/config.h"

#include "util/callchain.h"
#include "util/cgroup.h"
#include "util/header.h"
#include "util/event.h"
#include "util/evlist.h"
#include "util/evsel.h"
#include "util/debug.h"
#include "util/drv_configs.h"
#include "util/session.h"
#include "util/tool.h"
#include "util/symbol.h"
#include "util/cpumap.h"
#include "util/thread_map.h"
#include "util/data.h"
#include "util/perf_regs.h"
#include "util/auxtrace.h"
#include "util/tsc.h"
#include "util/parse-branch-options.h"
#include "util/parse-regs-options.h"
#include "util/llvm-utils.h"
#include "util/bpf-loader.h"
#include "util/trigger.h"
#include "util/perf-hooks.h"
#include "asm/bug.h"

#include <unistd.h>
#include <sched.h>
#include <sys/mman.h>
#include <asm/bug.h>
#include <linux/time64.h>


struct record {
	
struct perf_tool	tool;
	
struct record_opts	opts;
	
u64			bytes_written;
	
struct perf_data_file	file;
	
struct auxtrace_record	*itr;
	
struct perf_evlist	*evlist;
	
struct perf_session	*session;
	
const char		*progname;
	
int			realtime_prio;
	
bool			no_buildid;
	
bool			no_buildid_set;
	
bool			no_buildid_cache;
	
bool			no_buildid_cache_set;
	
bool			buildid_all;
	
bool			timestamp_filename;
	
bool			switch_output;
	
unsigned long long	samples;
};


static int record__write(struct record *rec, void *bf, size_t size) { if (perf_data_file__write(rec->session->file, bf, size) < 0) { pr_err("failed to write perf data, error: %m\n"); return -1; } rec->bytes_written += size; return 0; }

Contributors

PersonTokensPropCommitsCommitProp
peter zijlstrapeter zijlstra2443.64%114.29%
arnaldo carvalho de meloarnaldo carvalho de melo1730.91%342.86%
david aherndavid ahern1018.18%114.29%
jiri olsajiri olsa35.45%114.29%
adrian hunteradrian hunter11.82%114.29%
Total55100.00%7100.00%


static int process_synthesized_event(struct perf_tool *tool, union perf_event *event, struct perf_sample *sample __maybe_unused, struct machine *machine __maybe_unused) { struct record *rec = container_of(tool, struct record, tool); return record__write(rec, event, event->header.size); }

Contributors

PersonTokensPropCommitsCommitProp
arnaldo carvalho de meloarnaldo carvalho de melo5494.74%981.82%
irina tirdeairina tirdea23.51%19.09%
jiri olsajiri olsa11.75%19.09%
Total57100.00%11100.00%


static int backward_rb_find_range(void *buf, int mask, u64 head, u64 *start, u64 *end) { struct perf_event_header *pheader; u64 evt_head = head; int size = mask + 1; pr_debug2("backward_rb_find_range: buf=%p, head=%"PRIx64"\n", buf, head); pheader = (struct perf_event_header *)(buf + (head & mask)); *start = head; while (true) { if (evt_head - head >= (unsigned int)size) { pr_debug("Finished reading backward ring buffer: rewind\n"); if (evt_head - head > (unsigned int)size) evt_head -= pheader->size; *end = evt_head; return 0; } pheader = (struct perf_event_header *)(buf + (evt_head & mask)); if (pheader->size == 0) { pr_debug("Finished reading backward ring buffer: get start\n"); *end = evt_head; return 0; } evt_head += pheader->size; pr_debug3("move evt_head: %"PRIx64"\n", evt_head); } WARN_ONCE(1, "Shouldn't get here\n"); return -1; }

Contributors

PersonTokensPropCommitsCommitProp
wang nanwang nan18998.95%150.00%
colin kingcolin king21.05%150.00%
Total191100.00%2100.00%


static int rb_find_range(void *data, int mask, u64 head, u64 old, u64 *start, u64 *end, bool backward) { if (!backward) { *start = old; *end = head; return 0; } return backward_rb_find_range(data, mask, head, start, end); }

Contributors

PersonTokensPropCommitsCommitProp
wang nanwang nan64100.00%2100.00%
Total64100.00%2100.00%


static int record__mmap_read(struct record *rec, struct perf_mmap *md, bool overwrite, bool backward) { u64 head = perf_mmap__read_head(md); u64 old = md->prev; u64 end = head, start = old; unsigned char *data = md->base + page_size; unsigned long size; void *buf; int rc = 0; if (rb_find_range(data, md->mask, head, old, &start, &end, backward)) return -1; if (start == end) return 0; rec->samples++; size = end - start; if (size > (unsigned long)(md->mask) + 1) { WARN_ONCE(1, "failed to keep up with mmap data. (warn only once)\n"); md->prev = head; perf_mmap__consume(md, overwrite || backward); return 0; } if ((start & md->mask) + size != (end & md->mask)) { buf = &data[start & md->mask]; size = md->mask + 1 - (start & md->mask); start += size; if (record__write(rec, buf, size) < 0) { rc = -1; goto out; } } buf = &data[start & md->mask]; size = end - start; start += size; if (record__write(rec, buf, size) < 0) { rc = -1; goto out; } md->prev = head; perf_mmap__consume(md, overwrite || backward); out: return rc; }

Contributors

PersonTokensPropCommitsCommitProp
wang nanwang nan10835.76%529.41%
arnaldo carvalho de meloarnaldo carvalho de melo8829.14%741.18%
ingo molnaringo molnar4916.23%15.88%
david aherndavid ahern4414.57%211.76%
peter zijlstrapeter zijlstra134.30%211.76%
Total302100.00%17100.00%

static volatile int done; static volatile int signr = -1; static volatile int child_finished; static volatile int auxtrace_record__snapshot_started; static DEFINE_TRIGGER(auxtrace_snapshot_trigger); static DEFINE_TRIGGER(switch_output_trigger);
static void sig_handler(int sig) { if (sig == SIGCHLD) child_finished = 1; else signr = sig; done = 1; }

Contributors

PersonTokensPropCommitsCommitProp
adrian hunteradrian hunter28100.00%1100.00%
Total28100.00%1100.00%


static void sigsegv_handler(int sig) { perf_hooks__recover(); sighandler_dump_stack(sig); }

Contributors

PersonTokensPropCommitsCommitProp
wang nanwang nan17100.00%1100.00%
Total17100.00%1100.00%


static void record__sig_exit(void) { if (signr == -1) return; signal(signr, SIG_DFL); raise(signr); }

Contributors

PersonTokensPropCommitsCommitProp
adrian hunteradrian hunter28100.00%1100.00%
Total28100.00%1100.00%

#ifdef HAVE_AUXTRACE_SUPPORT
static int record__process_auxtrace(struct perf_tool *tool, union perf_event *event, void *data1, size_t len1, void *data2, size_t len2) { struct record *rec = container_of(tool, struct record, tool); struct perf_data_file *file = &rec->file; size_t padding; u8 pad[8] = {0}; if (!perf_data_file__is_pipe(file)) { off_t file_offset; int fd = perf_data_file__fd(file); int err; file_offset = lseek(fd, 0, SEEK_CUR); if (file_offset == -1) return -1; err = auxtrace_index__auxtrace_event(&rec->session->auxtrace_index, event, file_offset); if (err) return err; } /* event.auxtrace.size includes padding, see __auxtrace_mmap__read() */ padding = (len1 + len2) & 7; if (padding) padding = 8 - padding; record__write(rec, event, event->header.size); record__write(rec, data1, len1); if (len2) record__write(rec, data2, len2); record__write(rec, &pad, padding); return 0; }

Contributors

PersonTokensPropCommitsCommitProp
adrian hunteradrian hunter206100.00%2100.00%
Total206100.00%2100.00%


static int record__auxtrace_mmap_read(struct record *rec, struct auxtrace_mmap *mm) { int ret; ret = auxtrace_mmap__read(mm, rec->itr, &rec->tool, record__process_auxtrace); if (ret < 0) return ret; if (ret) rec->samples++; return 0; }

Contributors

PersonTokensPropCommitsCommitProp
adrian hunteradrian hunter58100.00%2100.00%
Total58100.00%2100.00%


static int record__auxtrace_mmap_read_snapshot(struct record *rec, struct auxtrace_mmap *mm) { int ret; ret = auxtrace_mmap__read_snapshot(mm, rec->itr, &rec->tool, record__process_auxtrace, rec->opts.auxtrace_snapshot_size); if (ret < 0) return ret; if (ret) rec->samples++; return 0; }

Contributors

PersonTokensPropCommitsCommitProp
adrian hunteradrian hunter64100.00%1100.00%
Total64100.00%1100.00%


static int record__auxtrace_read_snapshot_all(struct record *rec) { int i; int rc = 0; for (i = 0; i < rec->evlist->nr_mmaps; i++) { struct auxtrace_mmap *mm = &rec->evlist->mmap[i].auxtrace_mmap; if (!mm->base) continue; if (record__auxtrace_mmap_read_snapshot(rec, mm) != 0) { rc = -1; goto out; } } out: return rc; }

Contributors

PersonTokensPropCommitsCommitProp
adrian hunteradrian hunter89100.00%1100.00%
Total89100.00%1100.00%


static void record__read_auxtrace_snapshot(struct record *rec) { pr_debug("Recording AUX area tracing snapshot\n"); if (record__auxtrace_read_snapshot_all(rec) < 0) { trigger_error(&auxtrace_snapshot_trigger); } else { if (auxtrace_record__snapshot_finish(rec->itr)) trigger_error(&auxtrace_snapshot_trigger); else trigger_ready(&auxtrace_snapshot_trigger); } }

Contributors

PersonTokensPropCommitsCommitProp
adrian hunteradrian hunter3967.24%266.67%
wang nanwang nan1932.76%133.33%
Total58100.00%3100.00%

#else
static inline int record__auxtrace_mmap_read(struct record *rec __maybe_unused, struct auxtrace_mmap *mm __maybe_unused) { return 0; }

Contributors

PersonTokensPropCommitsCommitProp
adrian hunteradrian hunter22100.00%1100.00%
Total22100.00%1100.00%


static inline void record__read_auxtrace_snapshot(struct record *rec __maybe_unused) { }

Contributors

PersonTokensPropCommitsCommitProp
adrian hunteradrian hunter866.67%150.00%
peter zijlstrapeter zijlstra433.33%150.00%
Total12100.00%2100.00%


static inline int auxtrace_record__snapshot_start(struct auxtrace_record *itr __maybe_unused) { return 0; }

Contributors

PersonTokensPropCommitsCommitProp
adrian hunteradrian hunter1168.75%133.33%
arnaldo carvalho de meloarnaldo carvalho de melo425.00%133.33%
peter zijlstrapeter zijlstra16.25%133.33%
Total16100.00%3100.00%

#endif
static int record__mmap_evlist(struct record *rec, struct perf_evlist *evlist) { struct record_opts *opts = &rec->opts; char msg[512]; if (perf_evlist__mmap_ex(evlist, opts->mmap_pages, false, opts->auxtrace_mmap_pages, opts->auxtrace_snapshot_mode) < 0) { if (errno == EPERM) { pr_err("Permission error mapping pages.\n" "Consider increasing " "/proc/sys/kernel/perf_event_mlock_kb,\n" "or try again with a smaller value of -m/--mmap_pages.\n" "(current value: %u,%u)\n", opts->mmap_pages, opts->auxtrace_mmap_pages); return -errno; } else { pr_err("failed to mmap with %d (%s)\n", errno, str_error_r(errno, msg, sizeof(msg))); if (errno) return -errno; else return -EINVAL; } } return 0; }

Contributors

PersonTokensPropCommitsCommitProp
wang nanwang nan12399.19%150.00%
arnaldo carvalho de meloarnaldo carvalho de melo10.81%150.00%
Total124100.00%2100.00%


static int record__mmap(struct record *rec) { return record__mmap_evlist(rec, rec->evlist); }

Contributors

PersonTokensPropCommitsCommitProp
wang nanwang nan21100.00%1100.00%
Total21100.00%1100.00%


static int record__open(struct record *rec) { char msg[512]; struct perf_evsel *pos; struct perf_evlist *evlist = rec->evlist; struct perf_session *session = rec->session; struct record_opts *opts = &rec->opts; struct perf_evsel_config_term *err_term; int rc = 0; perf_evlist__config(evlist, opts, &callchain_param); evlist__for_each_entry(evlist, pos) { try_again: if (perf_evsel__open(pos, pos->cpus, pos->threads) < 0) { if (perf_evsel__fallback(pos, errno, msg, sizeof(msg))) { if (verbose) ui__warning("%s\n", msg); goto try_again; } rc = -errno; perf_evsel__open_strerror(pos, &opts->target, errno, msg, sizeof(msg)); ui__error("%s\n", msg); goto out; } } if (perf_evlist__apply_filters(evlist, &pos)) { error("failed to set filter \"%s\" on event %s with %d (%s)\n", pos->filter, perf_evsel__name(pos), errno, str_error_r(errno, msg, sizeof(msg))); rc = -1; goto out; } if (perf_evlist__apply_drv_configs(evlist, &pos, &err_term)) { error("failed to set config \"%s\" on event %s with %d (%s)\n", err_term->val.drv_cfg, perf_evsel__name(pos), errno, str_error_r(errno, msg, sizeof(msg))); rc = -1; goto out; } rc = record__mmap(rec); if (rc) goto out; session->evlist = evlist; perf_session__set_id_hdr_size(session); out: return rc; }

Contributors

PersonTokensPropCommitsCommitProp
arnaldo carvalho de meloarnaldo carvalho de melo15351.69%1954.29%
mathieu j. poiriermathieu j. poirier5819.59%12.86%
david aherndavid ahern3110.47%38.57%
frederic weisbeckerfrederic weisbecker196.42%12.86%
masami hiramatsumasami hiramatsu72.36%12.86%
wang nanwang nan62.03%25.71%
jiri olsajiri olsa62.03%12.86%
li zefanli zefan51.69%12.86%
peter zijlstrapeter zijlstra41.35%25.71%
ingo molnaringo molnar31.01%12.86%
kan liangkan liang20.68%12.86%
stephane eranianstephane eranian10.34%12.86%
yanmin zhangyanmin zhang10.34%12.86%
Total296100.00%35100.00%


static int process_sample_event(struct perf_tool *tool, union perf_event *event, struct perf_sample *sample, struct perf_evsel *evsel, struct machine *machine) { struct record *rec = container_of(tool, struct record, tool); rec->samples++; return build_id__mark_dso_hit(tool, event, sample, evsel, machine); }

Contributors

PersonTokensPropCommitsCommitProp
namhyung kimnamhyung kim65100.00%1100.00%
Total65100.00%1100.00%


static int process_buildids(struct record *rec) { struct perf_data_file *file = &rec->file; struct perf_session *session = rec->session; if (file->size == 0) return 0; /* * During this process, it'll load kernel map and replace the * dso->long_name to a real pathname it found. In this case * we prefer the vmlinux path like * /lib/modules/3.16.4/build/vmlinux * * rather than build-id path (in debug directory). * $HOME/.debug/.build-id/f0/6e17aa50adf4d00b88925e03775de107611551 */ symbol_conf.ignore_vmlinux_buildid = true; /* * If --buildid-all is given, it marks all DSO regardless of hits, * so no need to process samples. */ if (rec->buildid_all) rec->tool.sample = NULL; return perf_session__process_events(session); }

Contributors

PersonTokensPropCommitsCommitProp
arnaldo carvalho de meloarnaldo carvalho de melo2536.23%444.44%
namhyung kimnamhyung kim2333.33%333.33%
jiri olsajiri olsa1927.54%111.11%
he kuanghe kuang22.90%111.11%
Total69100.00%9100.00%


static void perf_event__synthesize_guest_os(struct machine *machine, void *data) { int err; struct perf_tool *tool = data; /* *As for guest kernel when processing subcommand record&report, *we arrange module mmap prior to guest kernel mmap and trigger *a preload dso because default guest module symbols are loaded *from guest kallsyms instead of /lib/modules/XXX/XXX. This *method is used to avoid symbol missing when the first addr is *in module instead of in guest kernel. */ err = perf_event__synthesize_modules(tool, process_synthesized_event, machine); if (err < 0) pr_err("Couldn't record guest kernel [%d]'s reference" " relocation symbol.\n", machine->pid); /* * We use _stext for guest kernel because guest kernel's /proc/kallsyms * have no _text sometimes. */ err = perf_event__synthesize_kernel_mmap(tool, process_synthesized_event, machine); if (err < 0) pr_err("Couldn't record guest kernel [%d]'s reference" " relocation symbol.\n", machine->pid); }

Contributors

PersonTokensPropCommitsCommitProp
yanmin zhangyanmin zhang5567.90%120.00%
arnaldo carvalho de meloarnaldo carvalho de melo2632.10%480.00%
Total81100.00%5100.00%

static struct perf_event_header finished_round_event = { .size = sizeof(struct perf_event_header), .type = PERF_RECORD_FINISHED_ROUND, };
static int record__mmap_read_evlist(struct record *rec, struct perf_evlist *evlist, bool backward) { u64 bytes_written = rec->bytes_written; int i; int rc = 0; struct perf_mmap *maps; if (!evlist) return 0; maps = backward ? evlist->backward_mmap : evlist->mmap; if (!maps) return 0; if (backward && evlist->bkw_mmap_state != BKW_MMAP_DATA_PENDING) return 0; for (i = 0; i < evlist->nr_mmaps; i++) { struct auxtrace_mmap *mm = &maps[i].auxtrace_mmap; if (maps[i].base) { if (record__mmap_read(rec, &maps[i], evlist->overwrite, backward) != 0) { rc = -1; goto out; } } if (mm->base && !rec->opts.auxtrace_snapshot_mode && record__auxtrace_mmap_read(rec, mm) != 0) { rc = -1; goto out; } } /* * Mark the round finished in case we wrote * at least one event. */ if (bytes_written != rec->bytes_written) rc = record__write(rec, &finished_round_event, sizeof(finished_round_event)); if (backward) perf_evlist__toggle_bkw_mmap(evlist, BKW_MMAP_EMPTY); out: return rc; }

Contributors

PersonTokensPropCommitsCommitProp
wang nanwang nan7935.27%430.77%
adrian hunteradrian hunter4419.64%215.38%
frederic weisbeckerfrederic weisbecker4319.20%17.69%
david aherndavid ahern2912.95%17.69%
jiri olsajiri olsa167.14%17.69%
arnaldo carvalho de meloarnaldo carvalho de melo135.80%430.77%
Total224100.00%13100.00%


static int record__mmap_read_all(struct record *rec) { int err; err = record__mmap_read_evlist(rec, rec->evlist, false); if (err) return err; return record__mmap_read_evlist(rec, rec->evlist, true); }

Contributors

PersonTokensPropCommitsCommitProp
wang nanwang nan46100.00%3100.00%
Total46100.00%3100.00%


static void record__init_features(struct record *rec) { struct perf_session *session = rec->session; int feat; for (feat = HEADER_FIRST_FEATURE; feat < HEADER_LAST_FEATURE; feat++) perf_header__set_feat(&session->header, feat); if (rec->no_buildid) perf_header__clear_feat(&session->header, HEADER_BUILD_ID); if (!have_tracepoints(&rec->evlist->entries)) perf_header__clear_feat(&session->header, HEADER_TRACING_DATA); if (!rec->opts.branch_stack) perf_header__clear_feat(&session->header, HEADER_BRANCH_STACK); if (!rec->opts.full_auxtrace) perf_header__clear_feat(&session->header, HEADER_AUXTRACE); perf_header__clear_feat(&session->header, HEADER_STAT); }

Contributors

PersonTokensPropCommitsCommitProp
david aherndavid ahern9974.44%120.00%
adrian hunteradrian hunter1914.29%120.00%
jiri olsajiri olsa107.52%120.00%
arnaldo carvalho de meloarnaldo carvalho de melo53.76%240.00%
Total133100.00%5100.00%


static void record__finish_output(struct record *rec) { struct perf_data_file *file = &rec->file; int fd = perf_data_file__fd(file); if (file->is_pipe) return; rec->session->header.data_size += rec->bytes_written; file->size = lseek(perf_data_file__fd(file), 0, SEEK_CUR); if (!rec->no_buildid) { process_buildids(rec); if (rec->buildid_all) dsos__hit_all(rec->session); } perf_session__write_header(rec->session, rec->evlist, fd, true); return; }

Contributors

PersonTokensPropCommitsCommitProp
wang nanwang nan107100.00%1100.00%
Total107100.00%1100.00%


static int record__synthesize_workload(struct record *rec, bool tail) { struct { struct thread_map map; struct thread_map_data map_data; } thread_map; if (rec->opts.tail_synthesize != tail) return 0; thread_map.map.nr = 1; thread_map.map.map[0].pid = rec->evlist->workload.pid; thread_map.map.map[0].comm = NULL; return perf_event__synthesize_thread_map(&rec->tool, &thread_map.map, process_synthesized_event, &rec->session->machines.host, rec->opts.sample_address, rec->opts.proc_map_timeout); }

Contributors

PersonTokensPropCommitsCommitProp
wang nanwang nan117100.00%2100.00%
Total117100.00%2100.00%

static int record__synthesize(struct record *rec, bool tail);
static int record__switch_output(struct record *rec, bool at_exit) { struct perf_data_file *file = &rec->file; int fd