Release 4.15 kernel/trace/trace_event_perf.c

/*
 * trace event based perf event profiling/tracing
 *
 * Copyright (C) 2009 Red Hat Inc, Peter Zijlstra
 * Copyright (C) 2009-2010 Frederic Weisbecker <fweisbec@gmail.com>
 */

#include <linux/module.h>
#include <linux/kprobes.h>
#include "trace.h"


static char __percpu *perf_trace_buf[PERF_NR_CONTEXTS];

/*
 * Force it to be aligned to unsigned long to avoid misaligned accesses
 * suprises
 */
typedef typeof(unsigned long [PERF_MAX_TRACE_SIZE / sizeof(unsigned long)])
	
perf_trace_t;

/* Count the events in use (per event id, not per instance) */

static int	total_ref_count;



static int perf_trace_event_perm(struct trace_event_call *tp_event,
				 struct perf_event *p_event)
{
	if (tp_event->perf_perm) {
		int ret = tp_event->perf_perm(tp_event, p_event);
		if (ret)
			return ret;
	}

	/*
         * We checked and allowed to create parent,
         * allow children without checking.
         */
	if (p_event->parent)
		return 0;

	/*
         * It's ok to check current process (owner) permissions in here,
         * because code below is called only via perf_event_open syscall.
         */

	/* The ftrace function trace is allowed only for root. */
	if (ftrace_event_is_function(tp_event)) {
		if (perf_paranoid_tracepoint_raw() && !capable(CAP_SYS_ADMIN))
			return -EPERM;

		if (!is_sampling_event(p_event))
			return 0;

		/*
                 * We don't allow user space callchains for  function trace
                 * event, due to issues with page faults while tracing page
                 * fault handler and its overall trickiness nature.
                 */
		if (!p_event->attr.exclude_callchain_user)
			return -EINVAL;

		/*
                 * Same reason to disable user stack dump as for user space
                 * callchains above.
                 */
		if (p_event->attr.sample_type & PERF_SAMPLE_STACK_USER)
			return -EINVAL;
	}

	/* No tracing, just counting, so no obvious leak */
	if (!(p_event->attr.sample_type & PERF_SAMPLE_RAW))
		return 0;

	/* Some events are ok to be traced by non-root users... */
	if (p_event->attach_state == PERF_ATTACH_TASK) {
		if (tp_event->flags & TRACE_EVENT_FL_CAP_ANY)
			return 0;
	}

	/*
         * ...otherwise raw tracepoint data can be a severe data leak,
         * only allow root to have these.
         */
	if (perf_paranoid_tracepoint_raw() && !capable(CAP_SYS_ADMIN))
		return -EPERM;

	return 0;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Jiri Olsa
75
42.37%
5
55.56%
Frédéric Weisbecker
73
41.24%
1
11.11%
Peter Zijlstra
27
15.25%
1
11.11%
Steven Rostedt
2
1.13%
2
22.22%
Total
177
100.00%
9
100.00%



static int perf_trace_event_reg(struct trace_event_call *tp_event,
				struct perf_event *p_event)
{
	struct hlist_head __percpu *list;
	int ret = -ENOMEM;
	int cpu;

	p_event->tp_event = tp_event;
	if (tp_event->perf_refcount++ > 0)
		return 0;

	list = alloc_percpu(struct hlist_head);
	if (!list)
		goto fail;

	for_each_possible_cpu(cpu)
		INIT_HLIST_HEAD(per_cpu_ptr(list, cpu));

	tp_event->perf_events = list;

	if (!total_ref_count) {
		char __percpu *buf;
		int i;

		for (i = 0; i < PERF_NR_CONTEXTS; i++) {
			buf = (char __percpu *)alloc_percpu(perf_trace_t);
			if (!buf)
				goto fail;

			perf_trace_buf[i] = buf;
		}
	}

	ret = tp_event->class->reg(tp_event, TRACE_REG_PERF_REGISTER, NULL);
	if (ret)
		goto fail;

	total_ref_count++;
	return 0;

fail:
	if (!total_ref_count) {
		int i;

		for (i = 0; i < PERF_NR_CONTEXTS; i++) {
			free_percpu(perf_trace_buf[i]);
			perf_trace_buf[i] = NULL;
		}
	}

	if (!--tp_event->perf_refcount) {
		free_percpu(tp_event->perf_events);
		tp_event->perf_events = NULL;
	}

	return ret;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Peter Zijlstra
114
47.90%
3
18.75%
Frédéric Weisbecker
103
43.28%
8
50.00%
Steven Rostedt
9
3.78%
2
12.50%
Jiri Olsa
6
2.52%
1
6.25%
Namhyung Kim
3
1.26%
1
6.25%
Li Zefan
3
1.26%
1
6.25%
Total
238
100.00%
16
100.00%



static void perf_trace_event_unreg(struct perf_event *p_event)
{
	struct trace_event_call *tp_event = p_event->tp_event;
	int i;

	if (--tp_event->perf_refcount > 0)
		goto out;

	tp_event->class->reg(tp_event, TRACE_REG_PERF_UNREGISTER, NULL);

	/*
         * Ensure our callback won't be called anymore. The buffers
         * will be freed after that.
         */
	tracepoint_synchronize_unregister();

	free_percpu(tp_event->perf_events);
	tp_event->perf_events = NULL;

	if (!--total_ref_count) {
		for (i = 0; i < PERF_NR_CONTEXTS; i++) {
			free_percpu(perf_trace_buf[i]);
			perf_trace_buf[i] = NULL;
		}
	}
out:
	module_put(tp_event->mod);
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Jiri Olsa
111
99.11%
1
50.00%
Steven Rostedt
1
0.89%
1
50.00%
Total
112
100.00%
2
100.00%



static int perf_trace_event_open(struct perf_event *p_event)
{
	struct trace_event_call *tp_event = p_event->tp_event;
	return tp_event->class->reg(tp_event, TRACE_REG_PERF_OPEN, p_event);
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Jiri Olsa
33
97.06%
1
50.00%
Steven Rostedt
1
2.94%
1
50.00%
Total
34
100.00%
2
100.00%



static void perf_trace_event_close(struct perf_event *p_event)
{
	struct trace_event_call *tp_event = p_event->tp_event;
	tp_event->class->reg(tp_event, TRACE_REG_PERF_CLOSE, p_event);
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Jiri Olsa
32
96.97%
1
50.00%
Steven Rostedt
1
3.03%
1
50.00%
Total
33
100.00%
2
100.00%



static int perf_trace_event_init(struct trace_event_call *tp_event,
				 struct perf_event *p_event)
{
	int ret;

	ret = perf_trace_event_perm(tp_event, p_event);
	if (ret)
		return ret;

	ret = perf_trace_event_reg(tp_event, p_event);
	if (ret)
		return ret;

	ret = perf_trace_event_open(p_event);
	if (ret) {
		perf_trace_event_unreg(p_event);
		return ret;
	}

	return 0;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Jiri Olsa
74
98.67%
1
50.00%
Steven Rostedt
1
1.33%
1
50.00%
Total
75
100.00%
2
100.00%



int perf_trace_init(struct perf_event *p_event)
{
	struct trace_event_call *tp_event;
	u64 event_id = p_event->attr.config;
	int ret = -EINVAL;

	mutex_lock(&event_mutex);
	list_for_each_entry(tp_event, &ftrace_events, list) {
		if (tp_event->event.type == event_id &&
		    tp_event->class && tp_event->class->reg &&
		    try_module_get(tp_event->mod)) {
			ret = perf_trace_event_init(tp_event, p_event);
			if (ret)
				module_put(tp_event->mod);
			break;
		}
	}
	mutex_unlock(&event_mutex);

	return ret;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Peter Zijlstra
46
43.81%
4
30.77%
Li Zefan
42
40.00%
3
23.08%
Steven Rostedt
16
15.24%
5
38.46%
Vince Weaver
1
0.95%
1
7.69%
Total
105
100.00%
13
100.00%



void perf_trace_destroy(struct perf_event *p_event)
{
	mutex_lock(&event_mutex);
	perf_trace_event_close(p_event);
	perf_trace_event_unreg(p_event);
	mutex_unlock(&event_mutex);
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Jiri Olsa
32
100.00%
1
100.00%
Total
32
100.00%
1
100.00%



int perf_trace_add(struct perf_event *p_event, int flags)
{
	struct trace_event_call *tp_event = p_event->tp_event;

	if (!(flags & PERF_EF_START))
		p_event->hw.state = PERF_HES_STOPPED;

	/*
         * If TRACE_REG_PERF_ADD returns false; no custom action was performed
         * and we need to take the default action of enqueueing our event on
         * the right per-cpu hlist.
         */
	if (!tp_event->class->reg(tp_event, TRACE_REG_PERF_ADD, p_event)) {
		struct hlist_head __percpu *pcpu_list;
		struct hlist_head *list;

		pcpu_list = tp_event->perf_events;
		if (WARN_ON_ONCE(!pcpu_list))
			return -EINVAL;

		list = this_cpu_ptr(pcpu_list);
		hlist_add_head_rcu(&p_event->hlist_entry, list);
	}

	return 0;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Peter Zijlstra
94
87.85%
4
57.14%
Namhyung Kim
9
8.41%
1
14.29%
Frédéric Weisbecker
3
2.80%
1
14.29%
Steven Rostedt
1
0.93%
1
14.29%
Total
107
100.00%
7
100.00%



void perf_trace_del(struct perf_event *p_event, int flags)
{
	struct trace_event_call *tp_event = p_event->tp_event;

	/*
         * If TRACE_REG_PERF_DEL returns false; no custom action was performed
         * and we need to take the default action of dequeueing our event from
         * the right per-cpu hlist.
         */
	if (!tp_event->class->reg(tp_event, TRACE_REG_PERF_DEL, p_event))
		hlist_del_rcu(&p_event->hlist_entry);
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Peter Zijlstra
23
48.94%
3
50.00%
Jiri Olsa
21
44.68%
1
16.67%
Frédéric Weisbecker
2
4.26%
1
16.67%
Steven Rostedt
1
2.13%
1
16.67%
Total
47
100.00%
6
100.00%



void *perf_trace_buf_alloc(int size, struct pt_regs **regs, int *rctxp)
{
	char *raw_data;
	int rctx;

	BUILD_BUG_ON(PERF_MAX_TRACE_SIZE % sizeof(unsigned long));

	if (WARN_ONCE(size > PERF_MAX_TRACE_SIZE,
		      "perf buffer not large enough"))
		return NULL;

	*rctxp = rctx = perf_swevent_get_recursion_context();
	if (rctx < 0)
		return NULL;

	if (regs)
		*regs = this_cpu_ptr(&__perf_regs[rctx]);
	raw_data = this_cpu_ptr(perf_trace_buf[rctx]);

	/* zero the dead bytes from align to not leak stack to user */
	memset(&raw_data[size - sizeof(u64)], 0, sizeof(u64));
	return raw_data;
}
Contributors
Person
Tokens
Prop
Commits
CommitProp
Xiao Guangrong
47
39.50%
1
12.50%
Peter Zijlstra
27
22.69%
4
50.00%
Frédéric Weisbecker
21
17.65%
1
12.50%
Oleg Nesterov
14
11.76%
1
12.50%
Alexei Starovoitov
10
8.40%
1
12.50%
Total
119
100.00%
8
100.00%


EXPORT_SYMBOL_GPL(perf_trace_buf_alloc);

NOKPROBE_SYMBOL(perf_trace_buf_alloc);



void perf_trace_buf_update(void *record, u16 type)
{
	struct trace_entry *entry = record;
	int pc = preempt_count();
	unsigned long flags;

	local_save_flags(flags);
	tracing_generic_entry_update(entry, flags, pc);
	entry->type = type;
}
Contributors
Person
Tokens
Prop
Commits
CommitProp
Alexei Starovoitov
28
57.14%
1
25.00%
Xiao Guangrong
15
30.61%
1
25.00%
Peter Zijlstra
6
12.24%
2
50.00%
Total
49
100.00%
4
100.00%


NOKPROBE_SYMBOL(perf_trace_buf_update);

#ifdef CONFIG_FUNCTION_TRACER

static void

perf_ftrace_function_call(unsigned long ip, unsigned long parent_ip,
			  struct ftrace_ops *ops, struct pt_regs *pt_regs)
{
	struct ftrace_entry *entry;
	struct perf_event *event;
	struct hlist_head head;
	struct pt_regs regs;
	int rctx;

	if ((unsigned long)ops->private != smp_processor_id())
		return;

	event = container_of(ops, struct perf_event, ftrace_ops);

	/*
         * @event->hlist entry is NULL (per INIT_HLIST_NODE), and all
         * the perf code does is hlist_for_each_entry_rcu(), so we can
         * get away with simply setting the @head.first pointer in order
         * to create a singular list.
         */
	head.first = &event->hlist_entry;


#define ENTRY_SIZE (ALIGN(sizeof(struct ftrace_entry) + sizeof(u32), \
                    sizeof(u64)) - sizeof(u32))

	BUILD_BUG_ON(ENTRY_SIZE > PERF_MAX_TRACE_SIZE);

	memset(&regs, 0, sizeof(regs));
	perf_fetch_caller_regs(&regs);

	entry = perf_trace_buf_alloc(ENTRY_SIZE, NULL, &rctx);
	if (!entry)
		return;

	entry->ip = ip;
	entry->parent_ip = parent_ip;
	perf_trace_buf_submit(entry, ENTRY_SIZE, rctx, TRACE_FN,
			      1, &regs, &head, NULL);


#undef ENTRY_SIZE
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Jiri Olsa
96
58.18%
1
12.50%
Peter Zijlstra
37
22.42%
1
12.50%
Alexei Starovoitov
15
9.09%
2
25.00%
Steven Rostedt
10
6.06%
2
25.00%
Oleg Nesterov
5
3.03%
1
12.50%
Andrey Vagin
2
1.21%
1
12.50%
Total
165
100.00%
8
100.00%



static int perf_ftrace_function_register(struct perf_event *event)
{
	struct ftrace_ops *ops = &event->ftrace_ops;

	ops->flags   = FTRACE_OPS_FL_RCU;
	ops->func    = perf_ftrace_function_call;
	ops->private = (void *)(unsigned long)nr_cpu_ids;

	return register_ftrace_function(ops);
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Jiri Olsa
37
69.81%
1
25.00%
Peter Zijlstra
15
28.30%
2
50.00%
Steven Rostedt
1
1.89%
1
25.00%
Total
53
100.00%
4
100.00%



static int perf_ftrace_function_unregister(struct perf_event *event)
{
	struct ftrace_ops *ops = &event->ftrace_ops;
	int ret = unregister_ftrace_function(ops);
	ftrace_free_filter(ops);
	return ret;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Jiri Olsa
37
100.00%
2
100.00%
Total
37
100.00%
2
100.00%



int perf_ftrace_event_register(struct trace_event_call *call,
			       enum trace_reg type, void *data)
{
	struct perf_event *event = data;

	switch (type) {
	case TRACE_REG_REGISTER:
	case TRACE_REG_UNREGISTER:
		break;
	case TRACE_REG_PERF_REGISTER:
	case TRACE_REG_PERF_UNREGISTER:
		return 0;
	case TRACE_REG_PERF_OPEN:
		return perf_ftrace_function_register(data);
	case TRACE_REG_PERF_CLOSE:
		return perf_ftrace_function_unregister(data);
	case TRACE_REG_PERF_ADD:
		event->ftrace_ops.private = (void *)(unsigned long)smp_processor_id();
		return 1;
	case TRACE_REG_PERF_DEL:
		event->ftrace_ops.private = (void *)(unsigned long)nr_cpu_ids;
		return 1;
	}

	return -EINVAL;
}
Contributors
Person
Tokens
Prop
Commits
CommitProp
Jiri Olsa
73
64.04%
1
33.33%
Peter Zijlstra
40
35.09%
1
33.33%
Steven Rostedt
1
0.88%
1
33.33%
Total
114
100.00%
3
100.00%

#endif /* CONFIG_FUNCTION_TRACER */
Overall Contributors
Person
Tokens
Prop
Commits
CommitProp
Jiri Olsa
633
40.84%
8
15.09%
Peter Zijlstra
435
28.06%
13
24.53%
Frédéric Weisbecker
220
14.19%
10
18.87%
Xiao Guangrong
68
4.39%
1
1.89%
Alexei Starovoitov
62
4.00%
2
3.77%
Li Zefan
48
3.10%
4
7.55%
Steven Rostedt
45
2.90%
9
16.98%
Oleg Nesterov
19
1.23%
2
3.77%
Namhyung Kim
13
0.84%
1
1.89%
Masami Hiramatsu
4
0.26%
1
1.89%
Andrey Vagin
2
0.13%
1
1.89%
Vince Weaver
1
0.06%
1
1.89%
Total
1550
100.00%
53
100.00%
Directory: kernel/trace

Information contained on this website is for historical information purposes only and does not indicate or represent copyright ownership.