Contributors: 32
Author Tokens Token Proportion Commits Commit Proportion
Palmer Dabbelt 628 31.46% 4 7.55%
Guo Ren 364 18.24% 5 9.43%
Björn Töpel 189 9.47% 3 5.66%
Deepak Gupta 149 7.46% 1 1.89%
Clément Leger 116 5.81% 2 3.77%
tongtiangen 102 5.11% 1 1.89%
Yunhui Cui 92 4.61% 1 1.89%
Damien Le Moal 68 3.41% 1 1.89%
Vincent Chen 67 3.36% 4 7.55%
Andy Chiu 43 2.15% 2 3.77%
Vitaly Wool 23 1.15% 1 1.89%
Eric W. Biedermann 18 0.90% 3 5.66%
Sami Tolvanen 18 0.90% 2 3.77%
Christoph Hellwig 16 0.80% 4 7.55%
Nam Cao 16 0.80% 2 3.77%
Xianting Tian 15 0.75% 1 1.89%
Song Shuai 14 0.70% 1 1.89%
Lukas Gerlach 11 0.55% 1 1.89%
Mattias Nissler 8 0.40% 1 1.89%
Pekka J Enberg 6 0.30% 1 1.89%
JiSheng Zhang 5 0.25% 1 1.89%
Kefeng Wang 5 0.25% 1 1.89%
Celeste Liu 5 0.25% 1 1.89%
Zong Li 3 0.15% 1 1.89%
Nanyong Sun 3 0.15% 1 1.89%
Jim Wilson 3 0.15% 1 1.89%
Greentime Hu 3 0.15% 1 1.89%
Thomas Gleixner 2 0.10% 1 1.89%
Andreas Schwab 1 0.05% 1 1.89%
Conor Dooley 1 0.05% 1 1.89%
Heiko Stübner 1 0.05% 1 1.89%
Paul Walmsley 1 0.05% 1 1.89%
Total 1996 53 


// SPDX-License-Identifier: GPL-2.0-only
/*
 * Copyright (C) 2012 Regents of the University of California
 */

#include <linux/cpu.h>
#include <linux/kernel.h>
#include <linux/init.h>
#include <linux/irqflags.h>
#include <linux/randomize_kstack.h>
#include <linux/sched.h>
#include <linux/sched/debug.h>
#include <linux/sched/signal.h>
#include <linux/signal.h>
#include <linux/kdebug.h>
#include <linux/uaccess.h>
#include <linux/kprobes.h>
#include <linux/uprobes.h>
#include <asm/uprobes.h>
#include <linux/mm.h>
#include <linux/module.h>
#include <linux/irq.h>
#include <linux/kexec.h>
#include <linux/entry-common.h>

#include <asm/asm-prototypes.h>
#include <asm/bug.h>
#include <asm/cfi.h>
#include <asm/csr.h>
#include <asm/processor.h>
#include <asm/ptrace.h>
#include <asm/syscall.h>
#include <asm/thread_info.h>
#include <asm/vector.h>
#include <asm/irq_stack.h>

int show_unhandled_signals = 1;

static DEFINE_RAW_SPINLOCK(die_lock);

static int copy_code(struct pt_regs *regs, u16 *val, const u16 *insns)
{
	const void __user *uaddr = (__force const void __user *)insns;

	if (!user_mode(regs))
		return get_kernel_nofault(*val, insns);

	/* The user space code from other tasks cannot be accessed. */
	if (regs != task_pt_regs(current))
		return -EPERM;

	return copy_from_user_nofault(val, uaddr, sizeof(*val));
}

static void dump_instr(const char *loglvl, struct pt_regs *regs)
{
	char str[sizeof("0000 ") * 12 + 2 + 1], *p = str;
	const u16 *insns = (u16 *)instruction_pointer(regs);
	long bad;
	u16 val;
	int i;

	for (i = -10; i < 2; i++) {
		bad = copy_code(regs, &val, &insns[i]);
		if (!bad) {
			p += sprintf(p, i == 0 ? "(%04hx) " : "%04hx ", val);
		} else {
			printk("%sCode: Unable to access instruction at 0x%px.\n",
			       loglvl, &insns[i]);
			return;
		}
	}
	printk("%sCode: %s\n", loglvl, str);
}

void die(struct pt_regs *regs, const char *str)
{
	static int die_counter;
	int ret;
	long cause;
	unsigned long flags;

	oops_enter();

	raw_spin_lock_irqsave(&die_lock, flags);
	console_verbose();
	bust_spinlocks(1);

	pr_emerg("%s [#%d]\n", str, ++die_counter);
	print_modules();
	if (regs) {
		show_regs(regs);
		dump_instr(KERN_EMERG, regs);
	}

	cause = regs ? regs->cause : -1;
	ret = notify_die(DIE_OOPS, str, regs, 0, cause, SIGSEGV);

	if (kexec_should_crash(current))
		crash_kexec(regs);

	bust_spinlocks(0);
	add_taint(TAINT_DIE, LOCKDEP_NOW_UNRELIABLE);
	raw_spin_unlock_irqrestore(&die_lock, flags);
	oops_exit();

	if (in_interrupt())
		panic("Fatal exception in interrupt");
	if (panic_on_oops)
		panic("Fatal exception");
	if (ret != NOTIFY_STOP)
		make_task_dead(SIGSEGV);
}

void do_trap(struct pt_regs *regs, int signo, int code, unsigned long addr)
{
	struct task_struct *tsk = current;

	if (show_unhandled_signals && unhandled_signal(tsk, signo)
	    && printk_ratelimit()) {
		pr_info("%s[%d]: unhandled signal %d code 0x%x at 0x" REG_FMT,
			tsk->comm, task_pid_nr(tsk), signo, code, addr);
		print_vma_addr(KERN_CONT " in ", instruction_pointer(regs));
		pr_cont("\n");
		__show_regs(regs);
		dump_instr(KERN_INFO, regs);
	}

	force_sig_fault(signo, code, (void __user *)addr);
}

static void do_trap_error(struct pt_regs *regs, int signo, int code,
	unsigned long addr, const char *str)
{
	current->thread.bad_cause = regs->cause;

	if (user_mode(regs)) {
		do_trap(regs, signo, code, addr);
	} else {
		if (!fixup_exception(regs))
			die(regs, str);
	}
}

#if defined(CONFIG_XIP_KERNEL) && defined(CONFIG_RISCV_ALTERNATIVE)
#define __trap_section __noinstr_section(".xip.traps")
#else
#define __trap_section noinstr
#endif
#define DO_ERROR_INFO(name, signo, code, str)					\
asmlinkage __visible __trap_section void name(struct pt_regs *regs)		\
{										\
	if (user_mode(regs)) {							\
		irqentry_enter_from_user_mode(regs);				\
		local_irq_enable();						\
		do_trap_error(regs, signo, code, regs->epc, "Oops - " str);	\
		local_irq_disable();						\
		irqentry_exit_to_user_mode(regs);				\
	} else {								\
		irqentry_state_t state = irqentry_nmi_enter(regs);		\
		do_trap_error(regs, signo, code, regs->epc, "Oops - " str);	\
		irqentry_nmi_exit(regs, state);					\
	}									\
}

DO_ERROR_INFO(do_trap_unknown,
	SIGILL, ILL_ILLTRP, "unknown exception");
DO_ERROR_INFO(do_trap_insn_misaligned,
	SIGBUS, BUS_ADRALN, "instruction address misaligned");
DO_ERROR_INFO(do_trap_insn_fault,
	SIGSEGV, SEGV_ACCERR, "instruction access fault");

asmlinkage __visible __trap_section void do_trap_insn_illegal(struct pt_regs *regs)
{
	bool handled;

	if (user_mode(regs)) {
		irqentry_enter_from_user_mode(regs);
		local_irq_enable();

		handled = riscv_v_first_use_handler(regs);
		if (!handled)
			do_trap_error(regs, SIGILL, ILL_ILLOPC, regs->epc,
				      "Oops - illegal instruction");

		local_irq_disable();
		irqentry_exit_to_user_mode(regs);
	} else {
		irqentry_state_t state = irqentry_nmi_enter(regs);

		do_trap_error(regs, SIGILL, ILL_ILLOPC, regs->epc,
			      "Oops - illegal instruction");

		irqentry_nmi_exit(regs, state);
	}
}

DO_ERROR_INFO(do_trap_load_fault,
	SIGSEGV, SEGV_ACCERR, "load access fault");

enum misaligned_access_type {
	MISALIGNED_STORE,
	MISALIGNED_LOAD,
};
static const struct {
	const char *type_str;
	int (*handler)(struct pt_regs *regs);
} misaligned_handler[] = {
	[MISALIGNED_STORE] = {
		.type_str = "Oops - store (or AMO) address misaligned",
		.handler = handle_misaligned_store,
	},
	[MISALIGNED_LOAD] = {
		.type_str = "Oops - load address misaligned",
		.handler = handle_misaligned_load,
	},
};

static void do_trap_misaligned(struct pt_regs *regs, enum misaligned_access_type type)
{
	irqentry_state_t state;

	if (user_mode(regs)) {
		irqentry_enter_from_user_mode(regs);
		local_irq_enable();
	} else {
		state = irqentry_nmi_enter(regs);
	}

	if (misaligned_handler[type].handler(regs))
		do_trap_error(regs, SIGBUS, BUS_ADRALN, regs->epc,
			      misaligned_handler[type].type_str);

	if (user_mode(regs)) {
		local_irq_disable();
		irqentry_exit_to_user_mode(regs);
	} else {
		irqentry_nmi_exit(regs, state);
	}
}

asmlinkage __visible __trap_section void do_trap_load_misaligned(struct pt_regs *regs)
{
	do_trap_misaligned(regs, MISALIGNED_LOAD);
}

asmlinkage __visible __trap_section void do_trap_store_misaligned(struct pt_regs *regs)
{
	do_trap_misaligned(regs, MISALIGNED_STORE);
}

DO_ERROR_INFO(do_trap_store_fault,
	SIGSEGV, SEGV_ACCERR, "store (or AMO) access fault");
DO_ERROR_INFO(do_trap_ecall_s,
	SIGILL, ILL_ILLTRP, "environment call from S-mode");
DO_ERROR_INFO(do_trap_ecall_m,
	SIGILL, ILL_ILLTRP, "environment call from M-mode");

static inline unsigned long get_break_insn_length(unsigned long pc)
{
	bug_insn_t insn;

	if (get_kernel_nofault(insn, (bug_insn_t *)pc))
		return 0;

	return GET_INSN_LENGTH(insn);
}

static bool probe_single_step_handler(struct pt_regs *regs)
{
	bool user = user_mode(regs);

	return user ? uprobe_single_step_handler(regs) : kprobe_single_step_handler(regs);
}

static bool probe_breakpoint_handler(struct pt_regs *regs)
{
	bool user = user_mode(regs);

	return user ? uprobe_breakpoint_handler(regs) : kprobe_breakpoint_handler(regs);
}

void handle_break(struct pt_regs *regs)
{
	if (probe_single_step_handler(regs))
		return;

	if (probe_breakpoint_handler(regs))
		return;

	current->thread.bad_cause = regs->cause;

	if (user_mode(regs))
		force_sig_fault(SIGTRAP, TRAP_BRKPT, (void __user *)regs->epc);
#ifdef CONFIG_KGDB
	else if (notify_die(DIE_TRAP, "EBREAK", regs, 0, regs->cause, SIGTRAP)
								== NOTIFY_STOP)
		return;
#endif
	else if (report_bug(regs->epc, regs) == BUG_TRAP_TYPE_WARN ||
		 handle_cfi_failure(regs) == BUG_TRAP_TYPE_WARN)
		regs->epc += get_break_insn_length(regs->epc);
	else
		die(regs, "Kernel BUG");
}

asmlinkage __visible __trap_section void do_trap_break(struct pt_regs *regs)
{
	if (user_mode(regs)) {
		irqentry_enter_from_user_mode(regs);
		local_irq_enable();

		handle_break(regs);

		local_irq_disable();
		irqentry_exit_to_user_mode(regs);
	} else {
		irqentry_state_t state = irqentry_nmi_enter(regs);

		handle_break(regs);

		irqentry_nmi_exit(regs, state);
	}
}

asmlinkage __visible __trap_section  __no_stack_protector
void do_trap_ecall_u(struct pt_regs *regs)
{
	if (user_mode(regs)) {
		long syscall = regs->a7;

		regs->epc += 4;
		regs->orig_a0 = regs->a0;
		regs->a0 = -ENOSYS;

		riscv_v_vstate_discard(regs);

		syscall = syscall_enter_from_user_mode(regs, syscall);

		add_random_kstack_offset();

		if (syscall >= 0 && syscall < NR_syscalls) {
			syscall = array_index_nospec(syscall, NR_syscalls);
			syscall_handler(regs, syscall);
		}

		/*
		 * Ultimately, this value will get limited by KSTACK_OFFSET_MAX(),
		 * so the maximum stack offset is 1k bytes (10 bits).
		 *
		 * The actual entropy will be further reduced by the compiler when
		 * applying stack alignment constraints: 16-byte (i.e. 4-bit) aligned
		 * for RV32I or RV64I.
		 *
		 * The resulting 6 bits of entropy is seen in SP[9:4].
		 */
		choose_random_kstack_offset(get_random_u16());

		syscall_exit_to_user_mode(regs);
	} else {
		irqentry_state_t state = irqentry_nmi_enter(regs);

		do_trap_error(regs, SIGILL, ILL_ILLTRP, regs->epc,
			"Oops - environment call from U-mode");

		irqentry_nmi_exit(regs, state);
	}

}

#define CFI_TVAL_FCFI_CODE	2
#define CFI_TVAL_BCFI_CODE	3
/* handle cfi violations */
bool handle_user_cfi_violation(struct pt_regs *regs)
{
	unsigned long tval = csr_read(CSR_TVAL);
	bool is_fcfi = (tval == CFI_TVAL_FCFI_CODE && cpu_supports_indirect_br_lp_instr());
	bool is_bcfi = (tval == CFI_TVAL_BCFI_CODE && cpu_supports_shadow_stack());

	/*
	 * Handle uprobe event first. The probe point can be a valid target
	 * of indirect jumps or calls, in this case, forward cfi violation
	 * will be triggered instead of breakpoint exception. Clear ELP flag
	 * on sstatus image as well to avoid recurring fault.
	 */
	if (is_fcfi && probe_breakpoint_handler(regs)) {
		regs->status &= ~SR_ELP;
		return true;
	}

	if (is_fcfi || is_bcfi) {
		do_trap_error(regs, SIGSEGV, SEGV_CPERR, regs->epc,
			      "Oops - control flow violation");
		return true;
	}

	return false;
}

/*
 * software check exception is defined with risc-v cfi spec. Software check
 * exception is raised when:
 * a) An indirect branch doesn't land on 4 byte aligned PC or `lpad`
 *    instruction or `label` value programmed in `lpad` instr doesn't
 *    match with value setup in `x7`. reported code in `xtval` is 2.
 * b) `sspopchk` instruction finds a mismatch between top of shadow stack (ssp)
 *    and x1/x5. reported code in `xtval` is 3.
 */
asmlinkage __visible __trap_section void do_trap_software_check(struct pt_regs *regs)
{
	if (user_mode(regs)) {
		irqentry_enter_from_user_mode(regs);

		/* not a cfi violation, then merge into flow of unknown trap handler */
		if (!handle_user_cfi_violation(regs))
			do_trap_unknown(regs);

		irqentry_exit_to_user_mode(regs);
	} else {
		/* sw check exception coming from kernel is a bug in kernel */
		die(regs, "Kernel BUG");
	}
}

#ifdef CONFIG_MMU
asmlinkage __visible noinstr void do_page_fault(struct pt_regs *regs)
{
	irqentry_state_t state = irqentry_enter(regs);

	handle_page_fault(regs);

	local_irq_disable();

	irqentry_exit(regs, state);
}
#endif

static void noinstr handle_riscv_irq(struct pt_regs *regs)
{
	struct pt_regs *old_regs;

	irq_enter_rcu();
	old_regs = set_irq_regs(regs);
	handle_arch_irq(regs);
	set_irq_regs(old_regs);
	irq_exit_rcu();
}

asmlinkage void noinstr do_irq(struct pt_regs *regs)
{
	irqentry_state_t state = irqentry_enter(regs);

	if (IS_ENABLED(CONFIG_IRQ_STACKS) && on_thread_stack())
		call_on_irq_stack(regs, handle_riscv_irq);
	else
		handle_riscv_irq(regs);

	irqentry_exit(regs, state);
}

#ifdef CONFIG_GENERIC_BUG
int is_valid_bugaddr(unsigned long pc)
{
	bug_insn_t insn;

	if (pc < VMALLOC_START)
		return 0;
	if (get_kernel_nofault(insn, (bug_insn_t *)pc))
		return 0;
	if ((insn & __INSN_LENGTH_MASK) == __INSN_LENGTH_32)
		return (insn == __BUG_INSN_32);
	else
		return ((insn & __COMPRESSED_INSN_MASK) == __BUG_INSN_16);
}
#endif /* CONFIG_GENERIC_BUG */

#ifdef CONFIG_VMAP_STACK
DEFINE_PER_CPU(unsigned long [OVERFLOW_STACK_SIZE/sizeof(long)],
		overflow_stack)__aligned(16);

asmlinkage void handle_bad_stack(struct pt_regs *regs)
{
	unsigned long tsk_stk = (unsigned long)current->stack;
	unsigned long ovf_stk = (unsigned long)this_cpu_ptr(overflow_stack);

	console_verbose();

	pr_emerg("Insufficient stack space to handle exception!\n");
	pr_emerg("Task stack:     [0x%016lx..0x%016lx]\n",
			tsk_stk, tsk_stk + THREAD_SIZE);
	pr_emerg("Overflow stack: [0x%016lx..0x%016lx]\n",
			ovf_stk, ovf_stk + OVERFLOW_STACK_SIZE);

	__show_regs(regs);
	panic("Kernel stack overflow");

	for (;;)
		wait_for_interrupt();
}
#endif