Contributors: 17
| Author |
Tokens |
Token Proportion |
Commits |
Commit Proportion |
| Alan Kao |
371 |
35.37% |
5 |
16.67% |
| Puranjay Mohan |
263 |
25.07% |
2 |
6.67% |
| Andy Chiu |
252 |
24.02% |
3 |
10.00% |
| Zong Li |
33 |
3.15% |
4 |
13.33% |
| Song Shuai |
30 |
2.86% |
1 |
3.33% |
| Alexandre Ghiti |
27 |
2.57% |
2 |
6.67% |
| Guo Ren |
24 |
2.29% |
2 |
6.67% |
| Palmer Dabbelt |
17 |
1.62% |
1 |
3.33% |
| Masami Hiramatsu |
14 |
1.33% |
1 |
3.33% |
| Qing Zhang |
6 |
0.57% |
1 |
3.33% |
| Steven Rostedt |
5 |
0.48% |
2 |
6.67% |
| Conor Dooley |
2 |
0.19% |
1 |
3.33% |
| Ryan Kosta |
1 |
0.10% |
1 |
3.33% |
| Li kunyu |
1 |
0.10% |
1 |
3.33% |
| Charlie Jenkins |
1 |
0.10% |
1 |
3.33% |
| Christoph Hellwig |
1 |
0.10% |
1 |
3.33% |
| Mike Rapoport |
1 |
0.10% |
1 |
3.33% |
| Total |
1049 |
|
30 |
|
// SPDX-License-Identifier: GPL-2.0
/*
* Copyright (C) 2013 Linaro Limited
* Author: AKASHI Takahiro <takahiro.akashi@linaro.org>
* Copyright (C) 2017 Andes Technology Corporation
*/
#include <linux/ftrace.h>
#include <linux/uaccess.h>
#include <linux/memory.h>
#include <linux/irqflags.h>
#include <linux/stop_machine.h>
#include <asm/cacheflush.h>
#include <asm/text-patching.h>
#ifdef CONFIG_DYNAMIC_FTRACE
void ftrace_arch_code_modify_prepare(void)
__acquires(&text_mutex)
{
mutex_lock(&text_mutex);
}
void ftrace_arch_code_modify_post_process(void)
__releases(&text_mutex)
{
mutex_unlock(&text_mutex);
}
unsigned long ftrace_call_adjust(unsigned long addr)
{
if (IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_CALL_OPS))
return addr + 8 + MCOUNT_AUIPC_SIZE;
return addr + MCOUNT_AUIPC_SIZE;
}
unsigned long arch_ftrace_get_symaddr(unsigned long fentry_ip)
{
return fentry_ip - MCOUNT_AUIPC_SIZE;
}
void arch_ftrace_update_code(int command)
{
command |= FTRACE_MAY_SLEEP;
ftrace_modify_all_code(command);
flush_icache_all();
}
static int __ftrace_modify_call(unsigned long source, unsigned long target, bool validate)
{
unsigned int call[2], offset;
unsigned int replaced[2];
offset = target - source;
call[1] = to_jalr_t0(offset);
if (validate) {
call[0] = to_auipc_t0(offset);
/*
* Read the text we want to modify;
* return must be -EFAULT on read error
*/
if (copy_from_kernel_nofault(replaced, (void *)source, 2 * MCOUNT_INSN_SIZE))
return -EFAULT;
if (replaced[0] != call[0]) {
pr_err("%p: expected (%08x) but got (%08x)\n",
(void *)source, call[0], replaced[0]);
return -EINVAL;
}
}
/* Replace the jalr at once. Return -EPERM on write error. */
if (patch_insn_write((void *)(source + MCOUNT_AUIPC_SIZE), call + 1, MCOUNT_JALR_SIZE))
return -EPERM;
return 0;
}
#ifdef CONFIG_DYNAMIC_FTRACE_WITH_CALL_OPS
static const struct ftrace_ops *riscv64_rec_get_ops(struct dyn_ftrace *rec)
{
const struct ftrace_ops *ops = NULL;
if (rec->flags & FTRACE_FL_CALL_OPS_EN) {
ops = ftrace_find_unique_ops(rec);
WARN_ON_ONCE(!ops);
}
if (!ops)
ops = &ftrace_list_ops;
return ops;
}
static int ftrace_rec_set_ops(const struct dyn_ftrace *rec, const struct ftrace_ops *ops)
{
unsigned long literal = ALIGN_DOWN(rec->ip - 12, 8);
return patch_text_nosync((void *)literal, &ops, sizeof(ops));
}
static int ftrace_rec_set_nop_ops(struct dyn_ftrace *rec)
{
return ftrace_rec_set_ops(rec, &ftrace_nop_ops);
}
static int ftrace_rec_update_ops(struct dyn_ftrace *rec)
{
return ftrace_rec_set_ops(rec, riscv64_rec_get_ops(rec));
}
#else
static int ftrace_rec_set_nop_ops(struct dyn_ftrace *rec) { return 0; }
static int ftrace_rec_update_ops(struct dyn_ftrace *rec) { return 0; }
#endif
int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
{
unsigned long distance, orig_addr, pc = rec->ip - MCOUNT_AUIPC_SIZE;
int ret;
ret = ftrace_rec_update_ops(rec);
if (ret)
return ret;
orig_addr = (unsigned long)&ftrace_caller;
distance = addr > orig_addr ? addr - orig_addr : orig_addr - addr;
if (distance > JALR_RANGE)
addr = FTRACE_ADDR;
return __ftrace_modify_call(pc, addr, false);
}
int ftrace_make_nop(struct module *mod, struct dyn_ftrace *rec, unsigned long addr)
{
u32 nop4 = RISCV_INSN_NOP4;
int ret;
ret = ftrace_rec_set_nop_ops(rec);
if (ret)
return ret;
if (patch_insn_write((void *)rec->ip, &nop4, MCOUNT_NOP4_SIZE))
return -EPERM;
return 0;
}
/*
* This is called early on, and isn't wrapped by
* ftrace_arch_code_modify_{prepare,post_process}() and therefor doesn't hold
* text_mutex, which triggers a lockdep failure. SMP isn't running so we could
* just directly poke the text, but it's simpler to just take the lock
* ourselves.
*/
int ftrace_init_nop(struct module *mod, struct dyn_ftrace *rec)
{
unsigned long pc = rec->ip - MCOUNT_AUIPC_SIZE;
unsigned int nops[2], offset;
int ret;
guard(mutex)(&text_mutex);
ret = ftrace_rec_set_nop_ops(rec);
if (ret)
return ret;
offset = (unsigned long) &ftrace_caller - pc;
nops[0] = to_auipc_t0(offset);
nops[1] = RISCV_INSN_NOP4;
ret = patch_insn_write((void *)pc, nops, 2 * MCOUNT_INSN_SIZE);
return ret;
}
ftrace_func_t ftrace_call_dest = ftrace_stub;
int ftrace_update_ftrace_func(ftrace_func_t func)
{
/*
* When using CALL_OPS, the function to call is associated with the
* call site, and we don't have a global function pointer to update.
*/
if (IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_CALL_OPS))
return 0;
WRITE_ONCE(ftrace_call_dest, func);
/*
* The data fence ensure that the update to ftrace_call_dest happens
* before the write to function_trace_op later in the generic ftrace.
* If the sequence is not enforced, then an old ftrace_call_dest may
* race loading a new function_trace_op set in ftrace_modify_all_code
*/
smp_wmb();
/*
* Updating ftrace dpes not take stop_machine path, so irqs should not
* be disabled.
*/
WARN_ON(irqs_disabled());
smp_call_function(ftrace_sync_ipi, NULL, 1);
return 0;
}
#else /* CONFIG_DYNAMIC_FTRACE */
unsigned long ftrace_call_adjust(unsigned long addr)
{
return addr;
}
#endif /* CONFIG_DYNAMIC_FTRACE */
#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr,
unsigned long addr)
{
unsigned long caller = rec->ip - MCOUNT_AUIPC_SIZE;
int ret;
ret = ftrace_rec_update_ops(rec);
if (ret)
return ret;
return __ftrace_modify_call(caller, FTRACE_ADDR, true);
}
#endif
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
/*
* Most of this function is copied from arm64.
*/
void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr,
unsigned long frame_pointer)
{
unsigned long return_hooker = (unsigned long)&return_to_handler;
unsigned long old;
if (unlikely(atomic_read(¤t->tracing_graph_pause)))
return;
/*
* We don't suffer access faults, so no extra fault-recovery assembly
* is needed here.
*/
old = *parent;
if (!function_graph_enter(old, self_addr, frame_pointer, parent))
*parent = return_hooker;
}
#ifdef CONFIG_DYNAMIC_FTRACE
void ftrace_graph_func(unsigned long ip, unsigned long parent_ip,
struct ftrace_ops *op, struct ftrace_regs *fregs)
{
unsigned long return_hooker = (unsigned long)&return_to_handler;
unsigned long frame_pointer = arch_ftrace_regs(fregs)->s0;
unsigned long *parent = &arch_ftrace_regs(fregs)->ra;
unsigned long old;
if (unlikely(atomic_read(¤t->tracing_graph_pause)))
return;
/*
* We don't suffer access faults, so no extra fault-recovery assembly
* is needed here.
*/
old = *parent;
if (!function_graph_enter_regs(old, ip, frame_pointer, parent, fregs))
*parent = return_hooker;
}
#endif /* CONFIG_DYNAMIC_FTRACE */
#endif /* CONFIG_FUNCTION_GRAPH_TRACER */