254 lines
6.7 KiB
C
254 lines
6.7 KiB
C
|
/*
|
||
|
|
* Copyright (C) 2015 - ARM Ltd
|
||
|
|
* Author: Marc Zyngier <marc.zyngier@arm.com>
|
||
|
|
*
|
||
|
|
* This program is free software; you can redistribute it and/or modify
|
||
|
|
* it under the terms of the GNU General Public License version 2 as
|
||
|
|
* published by the Free Software Foundation.
|
||
|
|
*
|
||
|
|
* This program is distributed in the hope that it will be useful,
|
||
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||
|
|
* GNU General Public License for more details.
|
||
|
|
*
|
||
|
|
* You should have received a copy of the GNU General Public License
|
||
|
|
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||
|
|
*/
|
||
|
|
#include <linux/jump_label.h>
|
||
|
|
|
||
|
|
#include <asm/kvm_asm.h>
|
||
|
|
#include <asm/kvm_hyp.h>
|
||
|
|
#include <asm/kvm_mmu.h>
|
||
|
|
|
||
|
|
__asm__(".arch_extension virt");
|
||
|
|
|
||
|
|
/*
|
||
|
|
* Activate the traps, saving the host's fpexc register before
|
||
|
|
* overwriting it. We'll restore it on VM exit.
|
||
|
|
*/
|
||
|
|
static void __hyp_text __activate_traps(struct kvm_vcpu *vcpu, u32 *fpexc_host)
|
||
|
|
{
|
||
|
|
u32 val;
|
||
|
|
|
||
|
|
/*
|
||
|
|
* We are about to set HCPTR.TCP10/11 to trap all floating point
|
||
|
|
* register accesses to HYP, however, the ARM ARM clearly states that
|
||
|
|
* traps are only taken to HYP if the operation would not otherwise
|
||
|
|
* trap to SVC. Therefore, always make sure that for 32-bit guests,
|
||
|
|
* we set FPEXC.EN to prevent traps to SVC, when setting the TCP bits.
|
||
|
|
*/
|
||
|
|
val = read_sysreg(VFP_FPEXC);
|
||
|
|
*fpexc_host = val;
|
||
|
|
if (!(val & FPEXC_EN)) {
|
||
|
|
write_sysreg(val | FPEXC_EN, VFP_FPEXC);
|
||
|
|
isb();
|
||
|
|
}
|
||
|
|
|
||
|
|
write_sysreg(vcpu->arch.hcr, HCR);
|
||
|
|
/* Trap on AArch32 cp15 c15 accesses (EL1 or EL0) */
|
||
|
|
write_sysreg(HSTR_T(15), HSTR);
|
||
|
|
write_sysreg(HCPTR_TTA | HCPTR_TCP(10) | HCPTR_TCP(11), HCPTR);
|
||
|
|
val = read_sysreg(HDCR);
|
||
|
|
val |= HDCR_TPM | HDCR_TPMCR; /* trap performance monitors */
|
||
|
|
val |= HDCR_TDRA | HDCR_TDOSA | HDCR_TDA; /* trap debug regs */
|
||
|
|
write_sysreg(val, HDCR);
|
||
|
|
}
|
||
|
|
|
||
|
|
static void __hyp_text __deactivate_traps(struct kvm_vcpu *vcpu)
|
||
|
|
{
|
||
|
|
u32 val;
|
||
|
|
|
||
|
|
/*
|
||
|
|
* If we pended a virtual abort, preserve it until it gets
|
||
|
|
* cleared. See B1.9.9 (Virtual Abort exception) for details,
|
||
|
|
* but the crucial bit is the zeroing of HCR.VA in the
|
||
|
|
* pseudocode.
|
||
|
|
*/
|
||
|
|
if (vcpu->arch.hcr & HCR_VA)
|
||
|
|
vcpu->arch.hcr = read_sysreg(HCR);
|
||
|
|
|
||
|
|
write_sysreg(0, HCR);
|
||
|
|
write_sysreg(0, HSTR);
|
||
|
|
val = read_sysreg(HDCR);
|
||
|
|
write_sysreg(val & ~(HDCR_TPM | HDCR_TPMCR), HDCR);
|
||
|
|
write_sysreg(0, HCPTR);
|
||
|
|
}
|
||
|
|
|
||
|
|
static void __hyp_text __activate_vm(struct kvm_vcpu *vcpu)
|
||
|
|
{
|
||
|
|
struct kvm *kvm = kern_hyp_va(vcpu->kvm);
|
||
|
|
write_sysreg(kvm->arch.vttbr, VTTBR);
|
||
|
|
write_sysreg(vcpu->arch.midr, VPIDR);
|
||
|
|
}
|
||
|
|
|
||
|
|
static void __hyp_text __deactivate_vm(struct kvm_vcpu *vcpu)
|
||
|
|
{
|
||
|
|
write_sysreg(0, VTTBR);
|
||
|
|
write_sysreg(read_sysreg(MIDR), VPIDR);
|
||
|
|
}
|
||
|
|
|
||
|
|
|
||
|
|
static void __hyp_text __vgic_save_state(struct kvm_vcpu *vcpu)
|
||
|
|
{
|
||
|
|
if (static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif)) {
|
||
|
|
__vgic_v3_save_state(vcpu);
|
||
|
|
__vgic_v3_deactivate_traps(vcpu);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
static void __hyp_text __vgic_restore_state(struct kvm_vcpu *vcpu)
|
||
|
|
{
|
||
|
|
if (static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif)) {
|
||
|
|
__vgic_v3_activate_traps(vcpu);
|
||
|
|
__vgic_v3_restore_state(vcpu);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
static bool __hyp_text __populate_fault_info(struct kvm_vcpu *vcpu)
|
||
|
|
{
|
||
|
|
u32 hsr = read_sysreg(HSR);
|
||
|
|
u8 ec = hsr >> HSR_EC_SHIFT;
|
||
|
|
u32 hpfar, far;
|
||
|
|
|
||
|
|
vcpu->arch.fault.hsr = hsr;
|
||
|
|
|
||
|
|
if (ec == HSR_EC_IABT)
|
||
|
|
far = read_sysreg(HIFAR);
|
||
|
|
else if (ec == HSR_EC_DABT)
|
||
|
|
far = read_sysreg(HDFAR);
|
||
|
|
else
|
||
|
|
return true;
|
||
|
|
|
||
|
|
/*
|
||
|
|
* B3.13.5 Reporting exceptions taken to the Non-secure PL2 mode:
|
||
|
|
*
|
||
|
|
* Abort on the stage 2 translation for a memory access from a
|
||
|
|
* Non-secure PL1 or PL0 mode:
|
||
|
|
*
|
||
|
|
* For any Access flag fault or Translation fault, and also for any
|
||
|
|
* Permission fault on the stage 2 translation of a memory access
|
||
|
|
* made as part of a translation table walk for a stage 1 translation,
|
||
|
|
* the HPFAR holds the IPA that caused the fault. Otherwise, the HPFAR
|
||
|
|
* is UNKNOWN.
|
||
|
|
*/
|
||
|
|
if (!(hsr & HSR_DABT_S1PTW) && (hsr & HSR_FSC_TYPE) == FSC_PERM) {
|
||
|
|
u64 par, tmp;
|
||
|
|
|
||
|
|
par = read_sysreg(PAR);
|
||
|
|
write_sysreg(far, ATS1CPR);
|
||
|
|
isb();
|
||
|
|
|
||
|
|
tmp = read_sysreg(PAR);
|
||
|
|
write_sysreg(par, PAR);
|
||
|
|
|
||
|
|
if (unlikely(tmp & 1))
|
||
|
|
return false; /* Translation failed, back to guest */
|
||
|
|
|
||
|
|
hpfar = ((tmp >> 12) & ((1UL << 28) - 1)) << 4;
|
||
|
|
} else {
|
||
|
|
hpfar = read_sysreg(HPFAR);
|
||
|
|
}
|
||
|
|
|
||
|
|
vcpu->arch.fault.hxfar = far;
|
||
|
|
vcpu->arch.fault.hpfar = hpfar;
|
||
|
|
return true;
|
||
|
|
}
|
||
|
|
|
||
|
|
int __hyp_text __kvm_vcpu_run_nvhe(struct kvm_vcpu *vcpu)
|
||
|
|
{
|
||
|
|
struct kvm_cpu_context *host_ctxt;
|
||
|
|
struct kvm_cpu_context *guest_ctxt;
|
||
|
|
bool fp_enabled;
|
||
|
|
u64 exit_code;
|
||
|
|
u32 fpexc;
|
||
|
|
|
||
|
|
vcpu = kern_hyp_va(vcpu);
|
||
|
|
write_sysreg(vcpu, HTPIDR);
|
||
|
|
|
||
|
|
host_ctxt = kern_hyp_va(vcpu->arch.host_cpu_context);
|
||
|
|
guest_ctxt = &vcpu->arch.ctxt;
|
||
|
|
|
||
|
|
__sysreg_save_state(host_ctxt);
|
||
|
|
__banked_save_state(host_ctxt);
|
||
|
|
|
||
|
|
__activate_traps(vcpu, &fpexc);
|
||
|
|
__activate_vm(vcpu);
|
||
|
|
|
||
|
|
__vgic_restore_state(vcpu);
|
||
|
|
__timer_enable_traps(vcpu);
|
||
|
|
|
||
|
|
__sysreg_restore_state(guest_ctxt);
|
||
|
|
__banked_restore_state(guest_ctxt);
|
||
|
|
|
||
|
|
/* Jump in the fire! */
|
||
|
|
again:
|
||
|
|
exit_code = __guest_enter(vcpu, host_ctxt);
|
||
|
|
/* And we're baaack! */
|
||
|
|
|
||
|
|
if (exit_code == ARM_EXCEPTION_HVC && !__populate_fault_info(vcpu))
|
||
|
|
goto again;
|
||
|
|
|
||
|
|
fp_enabled = __vfp_enabled();
|
||
|
|
|
||
|
|
__banked_save_state(guest_ctxt);
|
||
|
|
__sysreg_save_state(guest_ctxt);
|
||
|
|
__timer_disable_traps(vcpu);
|
||
|
|
|
||
|
|
__vgic_save_state(vcpu);
|
||
|
|
|
||
|
|
__deactivate_traps(vcpu);
|
||
|
|
__deactivate_vm(vcpu);
|
||
|
|
|
||
|
|
__banked_restore_state(host_ctxt);
|
||
|
|
__sysreg_restore_state(host_ctxt);
|
||
|
|
|
||
|
|
if (fp_enabled) {
|
||
|
|
__vfp_save_state(&guest_ctxt->vfp);
|
||
|
|
__vfp_restore_state(&host_ctxt->vfp);
|
||
|
|
}
|
||
|
|
|
||
|
|
write_sysreg(fpexc, VFP_FPEXC);
|
||
|
|
|
||
|
|
return exit_code;
|
||
|
|
}
|
||
|
|
|
||
|
|
static const char * const __hyp_panic_string[] = {
|
||
|
|
[ARM_EXCEPTION_RESET] = "\nHYP panic: RST PC:%08x CPSR:%08x",
|
||
|
|
[ARM_EXCEPTION_UNDEFINED] = "\nHYP panic: UNDEF PC:%08x CPSR:%08x",
|
||
|
|
[ARM_EXCEPTION_SOFTWARE] = "\nHYP panic: SVC PC:%08x CPSR:%08x",
|
||
|
|
[ARM_EXCEPTION_PREF_ABORT] = "\nHYP panic: PABRT PC:%08x CPSR:%08x",
|
||
|
|
[ARM_EXCEPTION_DATA_ABORT] = "\nHYP panic: DABRT PC:%08x ADDR:%08x",
|
||
|
|
[ARM_EXCEPTION_IRQ] = "\nHYP panic: IRQ PC:%08x CPSR:%08x",
|
||
|
|
[ARM_EXCEPTION_FIQ] = "\nHYP panic: FIQ PC:%08x CPSR:%08x",
|
||
|
|
[ARM_EXCEPTION_HVC] = "\nHYP panic: HVC PC:%08x CPSR:%08x",
|
||
|
|
};
|
||
|
|
|
||
|
|
void __hyp_text __noreturn __hyp_panic(int cause)
|
||
|
|
{
|
||
|
|
u32 elr = read_special(ELR_hyp);
|
||
|
|
u32 val;
|
||
|
|
|
||
|
|
if (cause == ARM_EXCEPTION_DATA_ABORT)
|
||
|
|
val = read_sysreg(HDFAR);
|
||
|
|
else
|
||
|
|
val = read_special(SPSR);
|
||
|
|
|
||
|
|
if (read_sysreg(VTTBR)) {
|
||
|
|
struct kvm_vcpu *vcpu;
|
||
|
|
struct kvm_cpu_context *host_ctxt;
|
||
|
|
|
||
|
|
vcpu = (struct kvm_vcpu *)read_sysreg(HTPIDR);
|
||
|
|
host_ctxt = kern_hyp_va(vcpu->arch.host_cpu_context);
|
||
|
|
__timer_disable_traps(vcpu);
|
||
|
|
__deactivate_traps(vcpu);
|
||
|
|
__deactivate_vm(vcpu);
|
||
|
|
__banked_restore_state(host_ctxt);
|
||
|
|
__sysreg_restore_state(host_ctxt);
|
||
|
|
}
|
||
|
|
|
||
|
|
/* Call panic for real */
|
||
|
|
__hyp_do_panic(__hyp_panic_string[cause], elr, val);
|
||
|
|
|
||
|
|
unreachable();
|
||
|
|
}
|