cregit-Linux how code gets into the kernel

Release 4.15 kernel/trace/trace_kprobe.c

Directory: kernel/trace
/*
 * Kprobes-based tracing events
 *
 * Created by Masami Hiramatsu <mhiramat@redhat.com>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 */

#define pr_fmt(fmt)	"trace_kprobe: " fmt

#include <linux/module.h>
#include <linux/uaccess.h>
#include <linux/rculist.h>

#include "trace_probe.h"


#define KPROBE_EVENT_SYSTEM "kprobes"

#define KRETPROBE_MAXACTIVE_MAX 4096

/**
 * Kprobe event core functions
 */

struct trace_kprobe {
	
struct list_head	list;
	
struct kretprobe	rp;	/* Use rp.kp for kprobe use */
	
unsigned long __percpu *nhit;
	
const char		*symbol;	/* symbol name */
	
struct trace_probe	tp;
};


#define SIZEOF_TRACE_KPROBE(n)				\
	(offsetof(struct trace_kprobe, tp.args) +       \
        (sizeof(struct probe_arg) * (n)))



static nokprobe_inline bool trace_kprobe_is_return(struct trace_kprobe *tk) { return tk->rp.handler != NULL; }

Contributors

PersonTokensPropCommitsCommitProp
Srikar Dronamraju1047.62%120.00%
Masami Hiramatsu733.33%360.00%
Namhyung Kim419.05%120.00%
Total21100.00%5100.00%


static nokprobe_inline const char *trace_kprobe_symbol(struct trace_kprobe *tk) { return tk->symbol ? tk->symbol : "unknown"; }

Contributors

PersonTokensPropCommitsCommitProp
Srikar Dronamraju1248.00%125.00%
Masami Hiramatsu832.00%250.00%
Namhyung Kim520.00%125.00%
Total25100.00%4100.00%


static nokprobe_inline unsigned long trace_kprobe_offset(struct trace_kprobe *tk) { return tk->rp.kp.offset; }

Contributors

PersonTokensPropCommitsCommitProp
Srikar Dronamraju1150.00%125.00%
Masami Hiramatsu731.82%250.00%
Namhyung Kim418.18%125.00%
Total22100.00%4100.00%


static nokprobe_inline bool trace_kprobe_has_gone(struct trace_kprobe *tk) { return !!(kprobe_gone(&tk->rp.kp)); }

Contributors

PersonTokensPropCommitsCommitProp
Masami Hiramatsu1244.44%250.00%
Srikar Dronamraju1140.74%125.00%
Namhyung Kim414.81%125.00%
Total27100.00%4100.00%


static nokprobe_inline bool trace_kprobe_within_module(struct trace_kprobe *tk, struct module *mod) { int len = strlen(mod->name); const char *name = trace_kprobe_symbol(tk); return strncmp(mod->name, name, len) == 0 && name[len] == ':'; }

Contributors

PersonTokensPropCommitsCommitProp
Srikar Dronamraju4374.14%125.00%
Masami Hiramatsu1017.24%250.00%
Namhyung Kim58.62%125.00%
Total58100.00%4100.00%


static nokprobe_inline bool trace_kprobe_is_on_module(struct trace_kprobe *tk) { return !!strchr(trace_kprobe_symbol(tk), ':'); }

Contributors

PersonTokensPropCommitsCommitProp
Srikar Dronamraju1352.00%125.00%
Masami Hiramatsu728.00%250.00%
Namhyung Kim520.00%125.00%
Total25100.00%4100.00%


static nokprobe_inline unsigned long trace_kprobe_nhit(struct trace_kprobe *tk) { unsigned long nhit = 0; int cpu; for_each_possible_cpu(cpu) nhit += *per_cpu_ptr(tk->nhit, cpu); return nhit; }

Contributors

PersonTokensPropCommitsCommitProp
Marcin Nowakowski41100.00%1100.00%
Total41100.00%1100.00%

static int register_kprobe_event(struct trace_kprobe *tk); static int unregister_kprobe_event(struct trace_kprobe *tk); static DEFINE_MUTEX(probe_lock); static LIST_HEAD(probe_list); static int kprobe_dispatcher(struct kprobe *kp, struct pt_regs *regs); static int kretprobe_dispatcher(struct kretprobe_instance *ri, struct pt_regs *regs); /* Memory fetching by symbol */ struct symbol_cache { char *symbol; long offset; unsigned long addr; };
unsigned long update_symbol_cache(struct symbol_cache *sc) { sc->addr = (unsigned long)kallsyms_lookup_name(sc->symbol); if (sc->addr) sc->addr += sc->offset; return sc->addr; }

Contributors

PersonTokensPropCommitsCommitProp
Namhyung Kim45100.00%1100.00%
Total45100.00%1100.00%


void free_symbol_cache(struct symbol_cache *sc) { kfree(sc->symbol); kfree(sc); }

Contributors

PersonTokensPropCommitsCommitProp
Namhyung Kim22100.00%1100.00%
Total22100.00%1100.00%


struct symbol_cache *alloc_symbol_cache(const char *sym, long offset) { struct symbol_cache *sc; if (!sym || strlen(sym) == 0) return NULL; sc = kzalloc(sizeof(struct symbol_cache), GFP_KERNEL); if (!sc) return NULL; sc->symbol = kstrdup(sym, GFP_KERNEL); if (!sc->symbol) { kfree(sc); return NULL; } sc->offset = offset; update_symbol_cache(sc); return sc; }

Contributors

PersonTokensPropCommitsCommitProp
Namhyung Kim98100.00%1100.00%
Total98100.00%1100.00%

/* * Kprobes-specific fetch functions */ #define DEFINE_FETCH_stack(type) \ static void FETCH_FUNC_NAME(stack, type)(struct pt_regs *regs, \ void *offset, void *dest) \ { \ *(type *)dest = (type)regs_get_kernel_stack_nth(regs, \ (unsigned int)((unsigned long)offset)); \ } \ NOKPROBE_SYMBOL(FETCH_FUNC_NAME(stack, type)); DEFINE_BASIC_FETCH_FUNCS(stack) /* No string on the stack entry */ #define fetch_stack_string NULL #define fetch_stack_string_size NULL #define DEFINE_FETCH_memory(type) \ static void FETCH_FUNC_NAME(memory, type)(struct pt_regs *regs, \ void *addr, void *dest) \ { \ type retval; \ if (probe_kernel_address(addr, retval)) \ *(type *)dest = 0; \ else \ *(type *)dest = retval; \ } \ NOKPROBE_SYMBOL(FETCH_FUNC_NAME(memory, type)); DEFINE_BASIC_FETCH_FUNCS(memory) /* * Fetch a null-terminated string. Caller MUST set *(u32 *)dest with max * length and relative data location. */ static void FETCH_FUNC_NAME(memory, string)(struct pt_regs *regs, void *addr, void *dest) { int maxlen = get_rloc_len(*(u32 *)dest); u8 *dst = get_rloc_data(dest); long ret; if (!maxlen) return; /* * Try to get string again, since the string can be changed while * probing. */ ret = strncpy_from_unsafe(dst, addr, maxlen); if (ret < 0) { /* Failed to fetch string */ dst[0] = '\0'; *(u32 *)dest = make_data_rloc(0, get_rloc_offs(*(u32 *)dest)); } else { *(u32 *)dest = make_data_rloc(ret, get_rloc_offs(*(u32 *)dest)); } } NOKPROBE_SYMBOL(FETCH_FUNC_NAME(memory, string)); /* Return the length of string -- including null terminal byte */ static void FETCH_FUNC_NAME(memory, string_size)(struct pt_regs *regs, void *addr, void *dest) { mm_segment_t old_fs; int ret, len = 0; u8 c; old_fs = get_fs(); set_fs(KERNEL_DS); pagefault_disable(); do { ret = __copy_from_user_inatomic(&c, (u8 *)addr + len, 1); len++; } while (c && ret == 0 && len < MAX_STRING_SIZE); pagefault_enable(); set_fs(old_fs); if (ret < 0) /* Failed to check the length */ *(u32 *)dest = 0; else *(u32 *)dest = len; } NOKPROBE_SYMBOL(FETCH_FUNC_NAME(memory, string_size)); #define DEFINE_FETCH_symbol(type) \ void FETCH_FUNC_NAME(symbol, type)(struct pt_regs *regs, void *data, void *dest)\ { \ struct symbol_cache *sc = data; \ if (sc->addr) \ fetch_memory_##type(regs, (void *)sc->addr, dest); \ else \ *(type *)dest = 0; \ } \ NOKPROBE_SYMBOL(FETCH_FUNC_NAME(symbol, type)); DEFINE_BASIC_FETCH_FUNCS(symbol) DEFINE_FETCH_symbol(string) DEFINE_FETCH_symbol(string_size) /* kprobes don't support file_offset fetch methods */ #define fetch_file_offset_u8 NULL #define fetch_file_offset_u16 NULL #define fetch_file_offset_u32 NULL #define fetch_file_offset_u64 NULL #define fetch_file_offset_string NULL #define fetch_file_offset_string_size NULL /* Fetch type information table */ static const struct fetch_type kprobes_fetch_type_table[] = { /* Special types */ [FETCH_TYPE_STRING] = __ASSIGN_FETCH_TYPE("string", string, string, sizeof(u32), 1, "__data_loc char[]"), [FETCH_TYPE_STRSIZE] = __ASSIGN_FETCH_TYPE("string_size", u32, string_size, sizeof(u32), 0, "u32"), /* Basic types */ ASSIGN_FETCH_TYPE(u8, u8, 0), ASSIGN_FETCH_TYPE(u16, u16, 0), ASSIGN_FETCH_TYPE(u32, u32, 0), ASSIGN_FETCH_TYPE(u64, u64, 0), ASSIGN_FETCH_TYPE(s8, u8, 1), ASSIGN_FETCH_TYPE(s16, u16, 1), ASSIGN_FETCH_TYPE(s32, u32, 1), ASSIGN_FETCH_TYPE(s64, u64, 1), ASSIGN_FETCH_TYPE_ALIAS(x8, u8, u8, 0), ASSIGN_FETCH_TYPE_ALIAS(x16, u16, u16, 0), ASSIGN_FETCH_TYPE_ALIAS(x32, u32, u32, 0), ASSIGN_FETCH_TYPE_ALIAS(x64, u64, u64, 0), ASSIGN_FETCH_TYPE_END }; /* * Allocate new trace_probe and initialize it (including kprobes). */
static struct trace_kprobe *alloc_trace_kprobe(const char *group, const char *event, void *addr, const char *symbol, unsigned long offs, int maxactive, int nargs, bool is_return) { struct trace_kprobe *tk; int ret = -ENOMEM; tk = kzalloc(SIZEOF_TRACE_KPROBE(nargs), GFP_KERNEL); if (!tk) return ERR_PTR(ret); tk->nhit = alloc_percpu(unsigned long); if (!tk->nhit) goto error; if (symbol) { tk->symbol = kstrdup(symbol, GFP_KERNEL); if (!tk->symbol) goto error; tk->rp.kp.symbol_name = tk->symbol; tk->rp.kp.offset = offs; } else tk->rp.kp.addr = addr; if (is_return) tk->rp.handler = kretprobe_dispatcher; else tk->rp.kp.pre_handler = kprobe_dispatcher; tk->rp.maxactive = maxactive; if (!event || !is_good_name(event)) { ret = -EINVAL; goto error; } tk->tp.call.class = &tk->tp.class; tk->tp.call.name = kstrdup(event, GFP_KERNEL); if (!tk->tp.call.name) goto error; if (!group || !is_good_name(group)) { ret = -EINVAL; goto error; } tk->tp.class.system = kstrdup(group, GFP_KERNEL); if (!tk->tp.class.system) goto error; INIT_LIST_HEAD(&tk->list); INIT_LIST_HEAD(&tk->tp.files); return tk; error: kfree(tk->tp.call.name); kfree(tk->symbol); free_percpu(tk->nhit); kfree(tk); return ERR_PTR(ret); }

Contributors

PersonTokensPropCommitsCommitProp
Srikar Dronamraju18551.68%17.69%
Masami Hiramatsu8523.74%753.85%
Namhyung Kim4312.01%17.69%
Martin KaFai Lau267.26%17.69%
Alban Crequy113.07%17.69%
Oleg Nesterov71.96%17.69%
Steven Rostedt10.28%17.69%
Total358100.00%13100.00%


static void free_trace_kprobe(struct trace_kprobe *tk) { int i; for (i = 0; i < tk->tp.nr_args; i++) traceprobe_free_probe_arg(&tk->tp.args[i]); kfree(tk->tp.call.class->system); kfree(tk->tp.call.name); kfree(tk->symbol); free_percpu(tk->nhit); kfree(tk); }

Contributors

PersonTokensPropCommitsCommitProp
Srikar Dronamraju3944.83%120.00%
Masami Hiramatsu2427.59%240.00%
Namhyung Kim1719.54%120.00%
Martin KaFai Lau78.05%120.00%
Total87100.00%5100.00%


static struct trace_kprobe *find_trace_kprobe(const char *event, const char *group) { struct trace_kprobe *tk; list_for_each_entry(tk, &probe_list, list) if (strcmp(trace_event_name(&tk->tp.call), event) == 0 && strcmp(tk->tp.call.class->system, group) == 0) return tk; return NULL; }

Contributors

PersonTokensPropCommitsCommitProp
Srikar Dronamraju3142.47%120.00%
Masami Hiramatsu2635.62%120.00%
Namhyung Kim1216.44%120.00%
Mathieu Desnoyers34.11%120.00%
Steven Rostedt11.37%120.00%
Total73100.00%5100.00%

/* * Enable trace_probe * if the file is NULL, enable "perf" handler, or enable "trace" handler. */
static int enable_trace_kprobe(struct trace_kprobe *tk, struct trace_event_file *file) { int ret = 0; if (file) { struct event_file_link *link; link = kmalloc(sizeof(*link), GFP_KERNEL); if (!link) { ret = -ENOMEM; goto out; } link->file = file; list_add_tail_rcu(&link->list, &tk->tp.files); tk->tp.flags |= TP_FLAG_TRACE; } else tk->tp.flags |= TP_FLAG_PROFILE; if (trace_probe_is_registered(&tk->tp) && !trace_kprobe_has_gone(tk)) { if (trace_kprobe_is_return(tk)) ret = enable_kretprobe(&tk->rp); else ret = enable_kprobe(&tk->rp.kp); } out: return ret; }

Contributors

PersonTokensPropCommitsCommitProp
Masami Hiramatsu9059.60%654.55%
Srikar Dronamraju2113.91%19.09%
Namhyung Kim2113.91%19.09%
Oleg Nesterov1811.92%218.18%
Steven Rostedt10.66%19.09%
Total151100.00%11100.00%

/* * Disable trace_probe * if the file is NULL, disable "perf" handler, or disable "trace" handler. */
static int disable_trace_kprobe(struct trace_kprobe *tk, struct trace_event_file *file) { struct event_file_link *link = NULL; int wait = 0; int ret = 0; if (file) { link = find_event_file_link(&tk->tp, file); if (!link) { ret = -EINVAL; goto out; } list_del_rcu(&link->list); wait = 1; if (!list_empty(&tk->tp.files)) goto out; tk->tp.flags &= ~TP_FLAG_TRACE; } else tk->tp.flags &= ~TP_FLAG_PROFILE; if (!trace_probe_is_enabled(&tk->tp) && trace_probe_is_registered(&tk->tp)) { if (trace_kprobe_is_return(tk)) disable_kretprobe(&tk->rp); else disable_kprobe(&tk->rp.kp); wait = 1; } out: if (wait) { /* * Synchronize with kprobe_trace_func/kretprobe_trace_func * to ensure disabled (all running handlers are finished). * This is not only for kfree(), but also the caller, * trace_remove_event_call() supposes it for releasing * event_call related objects, which will be accessed in * the kprobe_trace_func/kretprobe_trace_func. */ synchronize_sched(); kfree(link); /* Ignored if link == NULL */ } return ret; }

Contributors

PersonTokensPropCommitsCommitProp
Masami Hiramatsu11361.08%550.00%
Oleg Nesterov2513.51%220.00%
Namhyung Kim2513.51%110.00%
Srikar Dronamraju2111.35%110.00%
Steven Rostedt10.54%110.00%
Total185100.00%10100.00%

/* Internal register function - just handle k*probes and flags */
static int __register_trace_kprobe(struct trace_kprobe *tk) { int i, ret; if (trace_probe_is_registered(&tk->tp)) return -EINVAL; for (i = 0; i < tk->tp.nr_args; i++) traceprobe_update_arg(&tk->tp.args[i]); /* Set/clear disabled flag according to tp->flag */ if (trace_probe_is_enabled(&tk->tp)) tk->rp.kp.flags &= ~KPROBE_FLAG_DISABLED; else tk->rp.kp.flags |= KPROBE_FLAG_DISABLED; if (trace_kprobe_is_return(tk)) ret = register_kretprobe(&tk->rp); else ret = register_kprobe(&tk->rp.kp); if (ret == 0) tk->tp.flags |= TP_FLAG_REGISTERED; else { pr_warn("Could not insert probe at %s+%lu: %d\n", trace_kprobe_symbol(tk), trace_kprobe_offset(tk), ret); if (ret == -ENOENT && trace_kprobe_is_on_module(tk)) { pr_warn("This probe might be able to register after target module is loaded. Continue.\n"); ret = 0; } else if (ret == -EILSEQ) { pr_warn("Probing address(0x%p) is not an instruction boundary.\n", tk->rp.kp.addr); ret = -EINVAL; } } return ret; }

Contributors

PersonTokensPropCommitsCommitProp
Srikar Dronamraju10951.66%116.67%
Masami Hiramatsu6631.28%350.00%
Namhyung Kim3114.69%116.67%
Joe Perches52.37%116.67%
Total211100.00%6100.00%

/* Internal unregister function - just handle k*probes and flags */
static void __unregister_trace_kprobe(struct trace_kprobe *tk) { if (trace_probe_is_registered(&tk->tp)) { if (trace_kprobe_is_return(tk)) unregister_kretprobe(&tk->rp); else unregister_kprobe(&tk->rp.kp); tk->tp.flags &= ~TP_FLAG_REGISTERED; /* Cleanup kprobe for reuse */ if (tk->rp.kp.symbol_name) tk->rp.kp.addr = NULL; } }

Contributors

PersonTokensPropCommitsCommitProp
Srikar Dronamraju4050.63%133.33%
Masami Hiramatsu2430.38%133.33%
Namhyung Kim1518.99%133.33%
Total79100.00%3100.00%

/* Unregister a trace_probe and probe_event: call with locking probe_lock */
static int unregister_trace_kprobe(struct trace_kprobe *tk) { /* Enabled event can not be unregistered */ if (trace_probe_is_enabled(&tk->tp)) return -EBUSY; /* Will fail if probe is being used by ftrace or perf */ if (unregister_kprobe_event(tk)) return -EBUSY; __unregister_trace_kprobe(tk); list_del(&tk->list); return 0; }

Contributors

PersonTokensPropCommitsCommitProp
Srikar Dronamraju1935.19%125.00%
Masami Hiramatsu1425.93%125.00%
Namhyung Kim1120.37%125.00%
Steven Rostedt1018.52%125.00%
Total54100.00%4100.00%

/* Register a trace_probe and probe_event */
static int register_trace_kprobe(struct trace_kprobe *tk) { struct trace_kprobe *old_tk; int ret; mutex_lock(&probe_lock); /* Delete old (same name) event if exist */ old_tk = find_trace_kprobe(trace_event_name(&tk->tp.call), tk->tp.call.class->system); if (old_tk) { ret = unregister_trace_kprobe(old_tk); if (ret < 0) goto end; free_trace_kprobe(old_tk); } /* Register new event */ ret = register_kprobe_event(tk); if (ret) { pr_warn("Failed to register probe event(%d)\n", ret); goto end; } /* Register k*probe */ ret = __register_trace_kprobe(tk); if (ret < 0) unregister_kprobe_event(tk); else list_add_tail(&tk->list, &probe_list); end: mutex_unlock(&probe_lock); return ret; }

Contributors

PersonTokensPropCommitsCommitProp
Masami Hiramatsu5940.97%444.44%
Srikar Dronamraju5538.19%111.11%
Namhyung Kim2517.36%111.11%
Mathieu Desnoyers32.08%111.11%
Steven Rostedt10.69%111.11%
Joe Perches10.69%111.11%
Total144100.00%9100.00%

/* Module notifier call back, checking event on the module */
static int trace_kprobe_module_callback(struct notifier_block *nb, unsigned long val, void *data) { struct module *mod = data; struct trace_kprobe *tk; int ret; if (val != MODULE_STATE_COMING) return NOTIFY_DONE; /* Update probes on coming module */ mutex_lock(&probe_lock); list_for_each_entry(tk, &probe_list, list) { if (trace_kprobe_within_module(tk, mod)) { /* Don't need to check busy - this should have gone. */ __unregister_trace_kprobe(tk); ret = __register_trace_kprobe(tk); if (ret) pr_warn("Failed to re-register probe %s on %s: %d\n", trace_event_name(&tk->tp.call), mod->name, ret); } } mutex_unlock(&probe_lock); return NOTIFY_DONE; }

Contributors

PersonTokensPropCommitsCommitProp
Srikar Dronamraju7160.17%116.67%
Masami Hiramatsu2823.73%116.67%
Namhyung Kim1311.02%116.67%
Mathieu Desnoyers32.54%116.67%
Joe Perches21.69%116.67%
Steven Rostedt10.85%116.67%
Total118100.00%6100.00%

static struct notifier_block trace_kprobe_module_nb = { .notifier_call = trace_kprobe_module_callback, .priority = 1 /* Invoked after kprobe module callback */ }; /* Convert certain expected symbols into '_' when generating event names */
static inline void sanitize_event_name(char *name) { while (*name++ != '\0') if (*name == ':' || *name == '.') *name = '_'; }

Contributors

PersonTokensPropCommitsCommitProp
Naveen N. Rao36100.00%1100.00%
Total36100.00%1100.00%


static int create_trace_kprobe(int argc, char **argv) { /* * Argument syntax: * - Add kprobe: * p[:[GRP/]EVENT] [MOD:]KSYM[+OFFS]|KADDR [FETCHARGS] * - Add kretprobe: * r[MAXACTIVE][:[GRP/]EVENT] [MOD:]KSYM[+0] [FETCHARGS] * Fetch args: * $retval : fetch return value * $stack : fetch stack address * $stackN : fetch Nth of stack (N:0-) * $comm : fetch current task comm * @ADDR : fetch memory at ADDR (ADDR should be in kernel) * @SYM[+|-offs] : fetch memory at SYM +|- offs (SYM is a data symbol) * %REG : fetch register REG * Dereferencing memory fetch: * +|-offs(ARG) : fetch memory at ARG +|- offs address. * Alias name of args: * NAME=FETCHARG : set NAME as alias of FETCHARG. * Type of args: * FETCHARG:TYPE : use TYPE instead of unsigned long. */ struct trace_kprobe *tk; int i, ret = 0; bool is_return = false, is_delete = false; char *symbol = NULL, *event = NULL, *group = NULL; int maxactive = 0; char *arg; unsigned long offset = 0; void *addr = NULL; char buf[MAX_EVENT_NAME_LEN]; /* argc must be >= 1 */ if (argv[0][0] == 'p') is_return = false; else if (argv[0][0] == 'r') is_return = true