Contributors: 11
Author Tokens Token Proportion Commits Commit Proportion
Nicholas Piggin 330 50.77% 12 34.29%
Aneesh Kumar K.V 93 14.31% 2 5.71%
Rohan McLure 90 13.85% 2 5.71%
Xiu Jianfeng 69 10.62% 2 5.71%
Christophe Leroy 29 4.46% 10 28.57%
Eric Paris 26 4.00% 1 2.86%
Mark Rutland 7 1.08% 2 5.71%
Anton Blanchard 2 0.31% 1 2.86%
Michal Suchanek 2 0.31% 1 2.86%
Valentin Schneider 1 0.15% 1 2.86%
Paul Mackerras 1 0.15% 1 2.86%
Total 650 35 


// SPDX-License-Identifier: GPL-2.0-or-later

#include <linux/compat.h>
#include <linux/context_tracking.h>
#include <linux/randomize_kstack.h>

#include <asm/interrupt.h>
#include <asm/kup.h>
#include <asm/syscall.h>
#include <asm/time.h>
#include <asm/tm.h>
#include <asm/unistd.h>


/* Has to run notrace because it is entered not completely "reconciled" */
notrace long system_call_exception(struct pt_regs *regs, unsigned long r0)
{
	long ret;
	syscall_fn f;

	kuap_lock();

	add_random_kstack_offset();

	if (IS_ENABLED(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG))
		BUG_ON(irq_soft_mask_return() != IRQS_ALL_DISABLED);

	trace_hardirqs_off(); /* finish reconciling */

	CT_WARN_ON(ct_state() == CT_STATE_KERNEL);
	user_exit_irqoff();

	BUG_ON(regs_is_unrecoverable(regs));
	BUG_ON(!user_mode(regs));
	BUG_ON(arch_irq_disabled_regs(regs));

#ifdef CONFIG_PPC_PKEY
	if (mmu_has_feature(MMU_FTR_PKEY)) {
		unsigned long amr, iamr;
		bool flush_needed = false;
		/*
		 * When entering from userspace we mostly have the AMR/IAMR
		 * different from kernel default values. Hence don't compare.
		 */
		amr = mfspr(SPRN_AMR);
		iamr = mfspr(SPRN_IAMR);
		regs->amr  = amr;
		regs->iamr = iamr;
		if (mmu_has_feature(MMU_FTR_KUAP)) {
			mtspr(SPRN_AMR, AMR_KUAP_BLOCKED);
			flush_needed = true;
		}
		if (mmu_has_feature(MMU_FTR_BOOK3S_KUEP)) {
			mtspr(SPRN_IAMR, AMR_KUEP_BLOCKED);
			flush_needed = true;
		}
		if (flush_needed)
			isync();
	} else
#endif
		kuap_assert_locked();

	booke_restore_dbcr0();

	account_cpu_user_entry();

	account_stolen_time();

	/*
	 * This is not required for the syscall exit path, but makes the
	 * stack frame look nicer. If this was initialised in the first stack
	 * frame, or if the unwinder was taught the first stack frame always
	 * returns to user with IRQS_ENABLED, this store could be avoided!
	 */
	irq_soft_mask_regs_set_state(regs, IRQS_ENABLED);

	/*
	 * If system call is called with TM active, set _TIF_RESTOREALL to
	 * prevent RFSCV being used to return to userspace, because POWER9
	 * TM implementation has problems with this instruction returning to
	 * transactional state. Final register values are not relevant because
	 * the transaction will be aborted upon return anyway. Or in the case
	 * of unsupported_scv SIGILL fault, the return state does not much
	 * matter because it's an edge case.
	 */
	if (IS_ENABLED(CONFIG_PPC_TRANSACTIONAL_MEM) &&
			unlikely(MSR_TM_TRANSACTIONAL(regs->msr)))
		set_bits(_TIF_RESTOREALL, &current_thread_info()->flags);

	/*
	 * If the system call was made with a transaction active, doom it and
	 * return without performing the system call. Unless it was an
	 * unsupported scv vector, in which case it's treated like an illegal
	 * instruction.
	 */
#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
	if (unlikely(MSR_TM_TRANSACTIONAL(regs->msr)) &&
	    !trap_is_unsupported_scv(regs)) {
		/* Enable TM in the kernel, and disable EE (for scv) */
		hard_irq_disable();
		mtmsr(mfmsr() | MSR_TM);

		/* tabort, this dooms the transaction, nothing else */
		asm volatile(".long 0x7c00071d | ((%0) << 16)"
				:: "r"(TM_CAUSE_SYSCALL|TM_CAUSE_PERSISTENT));

		/*
		 * Userspace will never see the return value. Execution will
		 * resume after the tbegin. of the aborted transaction with the
		 * checkpointed register state. A context switch could occur
		 * or signal delivered to the process before resuming the
		 * doomed transaction context, but that should all be handled
		 * as expected.
		 */
		return -ENOSYS;
	}
#endif // CONFIG_PPC_TRANSACTIONAL_MEM

	local_irq_enable();

	if (unlikely(read_thread_flags() & _TIF_SYSCALL_DOTRACE)) {
		if (unlikely(trap_is_unsupported_scv(regs))) {
			/* Unsupported scv vector */
			_exception(SIGILL, regs, ILL_ILLOPC, regs->nip);
			return regs->gpr[3];
		}
		/*
		 * We use the return value of do_syscall_trace_enter() as the
		 * syscall number. If the syscall was rejected for any reason
		 * do_syscall_trace_enter() returns an invalid syscall number
		 * and the test against NR_syscalls will fail and the return
		 * value to be used is in regs->gpr[3].
		 */
		r0 = do_syscall_trace_enter(regs);
		if (unlikely(r0 >= NR_syscalls))
			return regs->gpr[3];

	} else if (unlikely(r0 >= NR_syscalls)) {
		if (unlikely(trap_is_unsupported_scv(regs))) {
			/* Unsupported scv vector */
			_exception(SIGILL, regs, ILL_ILLOPC, regs->nip);
			return regs->gpr[3];
		}
		return -ENOSYS;
	}

	/* May be faster to do array_index_nospec? */
	barrier_nospec();

#ifdef CONFIG_ARCH_HAS_SYSCALL_WRAPPER
	// No COMPAT if we have SYSCALL_WRAPPER, see Kconfig
	f = (void *)sys_call_table[r0];
	ret = f(regs);
#else
	if (unlikely(is_compat_task())) {
		unsigned long r3, r4, r5, r6, r7, r8;

		f = (void *)compat_sys_call_table[r0];

		r3 = regs->gpr[3] & 0x00000000ffffffffULL;
		r4 = regs->gpr[4] & 0x00000000ffffffffULL;
		r5 = regs->gpr[5] & 0x00000000ffffffffULL;
		r6 = regs->gpr[6] & 0x00000000ffffffffULL;
		r7 = regs->gpr[7] & 0x00000000ffffffffULL;
		r8 = regs->gpr[8] & 0x00000000ffffffffULL;

		ret = f(r3, r4, r5, r6, r7, r8);
	} else {
		f = (void *)sys_call_table[r0];

		ret = f(regs->gpr[3], regs->gpr[4], regs->gpr[5],
			regs->gpr[6], regs->gpr[7], regs->gpr[8]);
	}
#endif

	/*
	 * Ultimately, this value will get limited by KSTACK_OFFSET_MAX(),
	 * so the maximum stack offset is 1k bytes (10 bits).
	 *
	 * The actual entropy will be further reduced by the compiler when
	 * applying stack alignment constraints: the powerpc architecture
	 * may have two kinds of stack alignment (16-bytes and 8-bytes).
	 *
	 * So the resulting 6 or 7 bits of entropy is seen in SP[9:4] or SP[9:3].
	 */
	choose_random_kstack_offset(mftb());

	return ret;
}