Contributors: 6
Author Tokens Token Proportion Commits Commit Proportion
Marc Zyngier 873 91.61% 6 42.86%
Vladimir Murzin 33 3.46% 1 7.14%
Christoffer Dall 21 2.20% 4 28.57%
Zhichao Huang 15 1.57% 1 7.14%
James Morse 9 0.94% 1 7.14%
Thomas Gleixner 2 0.21% 1 7.14%
Total 953 14 


// SPDX-License-Identifier: GPL-2.0-only
/*
 * Copyright (C) 2015 - ARM Ltd
 * Author: Marc Zyngier <marc.zyngier@arm.com>
 */
#include <linux/jump_label.h>

#include <asm/kvm_asm.h>
#include <asm/kvm_hyp.h>
#include <asm/kvm_mmu.h>

__asm__(".arch_extension     virt");

/*
 * Activate the traps, saving the host's fpexc register before
 * overwriting it. We'll restore it on VM exit.
 */
static void __hyp_text __activate_traps(struct kvm_vcpu *vcpu, u32 *fpexc_host)
{
	u32 val;

	/*
	 * We are about to set HCPTR.TCP10/11 to trap all floating point
	 * register accesses to HYP, however, the ARM ARM clearly states that
	 * traps are only taken to HYP if the operation would not otherwise
	 * trap to SVC.  Therefore, always make sure that for 32-bit guests,
	 * we set FPEXC.EN to prevent traps to SVC, when setting the TCP bits.
	 */
	val = read_sysreg(VFP_FPEXC);
	*fpexc_host = val;
	if (!(val & FPEXC_EN)) {
		write_sysreg(val | FPEXC_EN, VFP_FPEXC);
		isb();
	}

	write_sysreg(vcpu->arch.hcr, HCR);
	/* Trap on AArch32 cp15 c15 accesses (EL1 or EL0) */
	write_sysreg(HSTR_T(15), HSTR);
	write_sysreg(HCPTR_TTA | HCPTR_TCP(10) | HCPTR_TCP(11), HCPTR);
	val = read_sysreg(HDCR);
	val |= HDCR_TPM | HDCR_TPMCR; /* trap performance monitors */
	val |= HDCR_TDRA | HDCR_TDOSA | HDCR_TDA; /* trap debug regs */
	write_sysreg(val, HDCR);
}

static void __hyp_text __deactivate_traps(struct kvm_vcpu *vcpu)
{
	u32 val;

	/*
	 * If we pended a virtual abort, preserve it until it gets
	 * cleared. See B1.9.9 (Virtual Abort exception) for details,
	 * but the crucial bit is the zeroing of HCR.VA in the
	 * pseudocode.
	 */
	if (vcpu->arch.hcr & HCR_VA)
		vcpu->arch.hcr = read_sysreg(HCR);

	write_sysreg(0, HCR);
	write_sysreg(0, HSTR);
	val = read_sysreg(HDCR);
	write_sysreg(val & ~(HDCR_TPM | HDCR_TPMCR), HDCR);
	write_sysreg(0, HCPTR);
}

static void __hyp_text __activate_vm(struct kvm_vcpu *vcpu)
{
	struct kvm *kvm = kern_hyp_va(vcpu->kvm);
	write_sysreg(kvm_get_vttbr(kvm), VTTBR);
	write_sysreg(vcpu->arch.midr, VPIDR);
}

static void __hyp_text __deactivate_vm(struct kvm_vcpu *vcpu)
{
	write_sysreg(0, VTTBR);
	write_sysreg(read_sysreg(MIDR), VPIDR);
}


static void __hyp_text __vgic_save_state(struct kvm_vcpu *vcpu)
{
	if (static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif)) {
		__vgic_v3_save_state(vcpu);
		__vgic_v3_deactivate_traps(vcpu);
	}
}

static void __hyp_text __vgic_restore_state(struct kvm_vcpu *vcpu)
{
	if (static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif)) {
		__vgic_v3_activate_traps(vcpu);
		__vgic_v3_restore_state(vcpu);
	}
}

static bool __hyp_text __populate_fault_info(struct kvm_vcpu *vcpu)
{
	u32 hsr = read_sysreg(HSR);
	u8 ec = hsr >> HSR_EC_SHIFT;
	u32 hpfar, far;

	vcpu->arch.fault.hsr = hsr;

	if (ec == HSR_EC_IABT)
		far = read_sysreg(HIFAR);
	else if (ec == HSR_EC_DABT)
		far = read_sysreg(HDFAR);
	else
		return true;

	/*
	 * B3.13.5 Reporting exceptions taken to the Non-secure PL2 mode:
	 *
	 * Abort on the stage 2 translation for a memory access from a
	 * Non-secure PL1 or PL0 mode:
	 *
	 * For any Access flag fault or Translation fault, and also for any
	 * Permission fault on the stage 2 translation of a memory access
	 * made as part of a translation table walk for a stage 1 translation,
	 * the HPFAR holds the IPA that caused the fault. Otherwise, the HPFAR
	 * is UNKNOWN.
	 */
	if (!(hsr & HSR_DABT_S1PTW) && (hsr & HSR_FSC_TYPE) == FSC_PERM) {
		u64 par, tmp;

		par = read_sysreg(PAR);
		write_sysreg(far, ATS1CPR);
		isb();

		tmp = read_sysreg(PAR);
		write_sysreg(par, PAR);

		if (unlikely(tmp & 1))
			return false; /* Translation failed, back to guest */

		hpfar = ((tmp >> 12) & ((1UL << 28) - 1)) << 4;
	} else {
		hpfar = read_sysreg(HPFAR);
	}

	vcpu->arch.fault.hxfar = far;
	vcpu->arch.fault.hpfar = hpfar;
	return true;
}

int __hyp_text __kvm_vcpu_run_nvhe(struct kvm_vcpu *vcpu)
{
	struct kvm_cpu_context *host_ctxt;
	struct kvm_cpu_context *guest_ctxt;
	bool fp_enabled;
	u64 exit_code;
	u32 fpexc;

	vcpu = kern_hyp_va(vcpu);
	write_sysreg(vcpu, HTPIDR);

	host_ctxt = kern_hyp_va(vcpu->arch.host_cpu_context);
	guest_ctxt = &vcpu->arch.ctxt;

	__sysreg_save_state(host_ctxt);
	__banked_save_state(host_ctxt);

	__activate_traps(vcpu, &fpexc);
	__activate_vm(vcpu);

	__vgic_restore_state(vcpu);
	__timer_enable_traps(vcpu);

	__sysreg_restore_state(guest_ctxt);
	__banked_restore_state(guest_ctxt);

	/* Jump in the fire! */
again:
	exit_code = __guest_enter(vcpu, host_ctxt);
	/* And we're baaack! */

	if (exit_code == ARM_EXCEPTION_HVC && !__populate_fault_info(vcpu))
		goto again;

	fp_enabled = __vfp_enabled();

	__banked_save_state(guest_ctxt);
	__sysreg_save_state(guest_ctxt);
	__timer_disable_traps(vcpu);

	__vgic_save_state(vcpu);

	__deactivate_traps(vcpu);
	__deactivate_vm(vcpu);

	__banked_restore_state(host_ctxt);
	__sysreg_restore_state(host_ctxt);

	if (fp_enabled) {
		__vfp_save_state(&guest_ctxt->vfp);
		__vfp_restore_state(&host_ctxt->vfp);
	}

	write_sysreg(fpexc, VFP_FPEXC);

	return exit_code;
}

static const char * const __hyp_panic_string[] = {
	[ARM_EXCEPTION_RESET]      = "\nHYP panic: RST   PC:%08x CPSR:%08x",
	[ARM_EXCEPTION_UNDEFINED]  = "\nHYP panic: UNDEF PC:%08x CPSR:%08x",
	[ARM_EXCEPTION_SOFTWARE]   = "\nHYP panic: SVC   PC:%08x CPSR:%08x",
	[ARM_EXCEPTION_PREF_ABORT] = "\nHYP panic: PABRT PC:%08x CPSR:%08x",
	[ARM_EXCEPTION_DATA_ABORT] = "\nHYP panic: DABRT PC:%08x ADDR:%08x",
	[ARM_EXCEPTION_IRQ]        = "\nHYP panic: IRQ   PC:%08x CPSR:%08x",
	[ARM_EXCEPTION_FIQ]        = "\nHYP panic: FIQ   PC:%08x CPSR:%08x",
	[ARM_EXCEPTION_HVC]        = "\nHYP panic: HVC   PC:%08x CPSR:%08x",
};

void __hyp_text __noreturn __hyp_panic(int cause)
{
	u32 elr = read_special(ELR_hyp);
	u32 val;

	if (cause == ARM_EXCEPTION_DATA_ABORT)
		val = read_sysreg(HDFAR);
	else
		val = read_special(SPSR);

	if (read_sysreg(VTTBR)) {
		struct kvm_vcpu *vcpu;
		struct kvm_cpu_context *host_ctxt;

		vcpu = (struct kvm_vcpu *)read_sysreg(HTPIDR);
		host_ctxt = kern_hyp_va(vcpu->arch.host_cpu_context);
		__timer_disable_traps(vcpu);
		__deactivate_traps(vcpu);
		__deactivate_vm(vcpu);
		__banked_restore_state(host_ctxt);
		__sysreg_restore_state(host_ctxt);
	}

	/* Call panic for real */
	__hyp_do_panic(__hyp_panic_string[cause], elr, val);

	unreachable();
}