imrcpan/freebsd
1
0 Fork 0
mirror of https://github.com/freebsd/freebsd-src.git synced 2026-10-28 11:25:21 +01:00
Code Issues Packages Projects Releases Wiki Activity

arm64/vmm: Make remaining registers use hypctx_*_sys_reg

Browse Source 
Move vgic, timer and trapframe registers into sys_regs to handle them
in the same way as all the other registers.

Signed-off-by:	Kajetan Puchalski <[email protected]>
Reviewed by:	andrew
Sponsored by:	Arm Ltd
Differential Revision:	https://reviews.freebsd.org/D56556
This commit is contained in:
Kajetan Puchalski authored and Andrew Turner committed 2026-07-06 16:18:30 +01:00
1 parent ee2642678a
commit 35164034e3
12 files changed
+301 -222
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

sys/arm64/vmm/arm64.h
+42 -4
View File
@@ -56,7 +56,14 @@ struct vgic_v3_cpu;
#define DBGWCR_EL1(n) (DBGWCR0_EL1 + MIN(n, 15))
#define DBGWVR_EL1(n) (DBGWVR0_EL1 + MIN(n, 15))
#define HOST_ICH_LR_EL2(n) (HOST_ICH_LR0_EL2 + MIN(n, VGIC_ICH_LR_NUM_MAX-1))
#define HOST_ICH_AP0R_EL2(n) (HOST_ICH_AP0R0_EL2 + MIN(n, VGIC_ICH_APR_NUM_MAX-1))
#define HOST_ICH_AP1R_EL2(n) (HOST_ICH_AP1R0_EL2 + MIN(n, VGIC_ICH_APR_NUM_MAX-1))
#define GPR_X(n) (GPR_X0 + MIN(n, 30))
enum hypctx_sysreg {
HOST_SPSR_EL2,
HOST_ESR_EL2,
CSSELR_EL1, /* Cache Size Selection Register */
MDCCINT_EL1, /* Monitor DCC Interrupt Enable Register */
PAR_EL1, /* Physical Address Register */
@@ -120,6 +127,28 @@ enum hypctx_sysreg {
HOST_FAR_EL2, /* Fault Address Register */
HOST_HPFAR_EL2, /* Hypervisor IPA Fault Address Register */
HOST_ICH_EISR_EL2, /* End of Interrupt Status Register */
HOST_ICH_ELRSR_EL2, /* Empty List Register Status Register */
HOST_ICH_HCR_EL2, /* Hyp Control Register */
HOST_ICH_MISR_EL2, /* Maintenance Interrupt State Register */
HOST_ICH_VMCR_EL2, /* Virtual Machine Control Register */
/*
* The List Registers are part of the VM context and are modified on a
* world switch. They need to be allocated statically so they are
* mapped in the EL2 translation tables when struct hypctx is mapped.
*/
HOST_ICH_LR0_EL2,
HOST_ICH_LR_MAX_EL2 = HOST_ICH_LR0_EL2 + (VGIC_ICH_LR_NUM_MAX - 1),
/* Active Priorities Registers for Group 0 and 1 interrupts */
HOST_ICH_AP0R0_EL2,
HOST_ICH_AP0R_MAX_EL2 = HOST_ICH_AP0R0_EL2 + (VGIC_ICH_APR_NUM_MAX - 1),
HOST_ICH_AP1R0_EL2,
HOST_ICH_AP1R_MAX_EL2 = HOST_ICH_AP1R0_EL2 + (VGIC_ICH_APR_NUM_MAX - 1),
HOST_CNTHCTL_EL2,
HOST_CNTVOFF_EL2,
NR_NON_VNCR_REGS,
/* VNCR Registers */
@@ -217,10 +246,17 @@ enum hypctx_sysreg {
/* Virtual-address-sized registers */
VA_REGS_START,
/* Do not reorder these without matching asm changes */
GPR_LR,
GPR_X0,
GPR_X30 = GPR_X0 + 30,
/* These can be reordered freely */
SP_EL0, /* Stack pointer */
TPIDR_EL0, /* EL0 Software ID Register */
TPIDRRO_EL0, /* Read-only Thread ID Register */
TPIDR_EL1, /* EL1 Software ID Register */
HOST_SP_EL1,
HOST_ELR_EL2,
VA_REGS_END,
};
@@ -229,7 +265,6 @@ enum hypctx_sysreg {
* Per-vCPU hypervisor state.
*/
struct hypctx {
struct trapframe tf;
/* Virtual-address-sized registers */
uint64_t va_regs[VA_REGS_END - VA_REGS_START - 1];
/* Non-VNCR register state */
@@ -238,7 +273,6 @@ struct hypctx {
struct hyp *hyp;
struct vcpu *vcpu;
struct vtimer vtimer;
struct vtimer_cpu vtimer_cpu;
uint64_t setcaps; /* Currently enabled capabilities. */
@@ -247,7 +281,10 @@ struct hypctx {
uint64_t debug_spsr; /* Saved guest SPSR */
uint64_t debug_mdscr; /* Saved guest MDSCR */
struct vgic_v3_regs vgic_v3_regs;
struct {
uint16_t ich_lr_num;
uint16_t ich_apr_num;
} vgic_v3;
struct vgic_v3_cpu *vgic_cpu;
bool has_exception;
bool dbg_oslock;
@@ -277,6 +314,8 @@ struct hypctx {
(&hypctx->va_regs[reg - VA_REGS_START - 1])
#endif
/* Calling this with markers like reg=VNCR_START or reg=NR_NON_VNCR_REGS
is a bad idea */
static inline uint64_t *
hypctx_sys_reg(struct hypctx *hypctx, int reg /* enum hypctx_sysreg */)
{
@@ -285,7 +324,6 @@ hypctx_sys_reg(struct hypctx *hypctx, int reg /* enum hypctx_sysreg */)
return (__hypctx_va_sysreg(hypctx, reg));
if (reg > VNCR_START)
return (__hypctx_vncr_sysreg(hypctx, reg));
/* Calling this with reg=VNCR_START or reg=NR_NON_VNCR_REGS is a bad idea */
return (&hypctx->sys_regs[reg]);
}
sys/arm64/vmm/hyp_genassym.c
+35
View File
@@ -0,0 +1,35 @@
/*
* SPDX-License-Identifier: BSD-2-Clause
*
* Copyright (c) 2026 Arm Ltd
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include <sys/param.h>
#include <sys/assym.h>
#include <sys/proc.h>
#include "arm64.h"
ASSYM(HYPCTX_GPR_LR, offsetof(struct hypctx, va_regs) + (GPR_LR - VA_REGS_START - 1) * sizeof(uint64_t));
ASSYM(HYPCTX_GPR_X0, offsetof(struct hypctx, va_regs) + (GPR_X0 - VA_REGS_START - 1) * sizeof(uint64_t));
sys/arm64/vmm/io/vgic_v3.c
+24 -22
View File
@@ -503,7 +503,7 @@ vgic_v3_cpuinit(device_t dev, struct hypctx *hypctx)
*
* Maintenance interrupts are disabled.
*/
hypctx->vgic_v3_regs.ich_hcr_el2 = ICH_HCR_EL2_En;
hypctx_write_sys_reg(hypctx, HOST_ICH_HCR_EL2, ICH_HCR_EL2_En);
/*
* Configure the Interrupt Controller Virtual Machine Control Register.
@@ -518,20 +518,21 @@ vgic_v3_cpuinit(device_t dev, struct hypctx *hypctx)
* ICH_VMCR_EL2_VENG0: virtual Group 0 interrupts enabled.
* ICH_VMCR_EL2_VENG1: virtual Group 1 interrupts enabled.
*/
hypctx->vgic_v3_regs.ich_vmcr_el2 =
hypctx_write_sys_reg(hypctx, HOST_ICH_VMCR_EL2,
(virt_features.min_prio << ICH_VMCR_EL2_VPMR_SHIFT) |
ICH_VMCR_EL2_VBPR1_NO_PREEMPTION | ICH_VMCR_EL2_VBPR0_NO_PREEMPTION;
hypctx->vgic_v3_regs.ich_vmcr_el2 &= ~ICH_VMCR_EL2_VEOIM;
hypctx->vgic_v3_regs.ich_vmcr_el2 |= ICH_VMCR_EL2_VENG0 |
ICH_VMCR_EL2_VBPR1_NO_PREEMPTION |
ICH_VMCR_EL2_VBPR0_NO_PREEMPTION);
*hypctx_sys_reg(hypctx, HOST_ICH_VMCR_EL2) &= ~ICH_VMCR_EL2_VEOIM;
*hypctx_sys_reg(hypctx, HOST_ICH_VMCR_EL2) |= ICH_VMCR_EL2_VENG0 |
ICH_VMCR_EL2_VENG1;
hypctx->vgic_v3_regs.ich_lr_num = virt_features.ich_lr_num;
for (i = 0; i < hypctx->vgic_v3_regs.ich_lr_num; i++)
hypctx->vgic_v3_regs.ich_lr_el2[i] = 0UL;
hypctx->vgic_v3.ich_lr_num = virt_features.ich_lr_num;
for (i = 0; i < hypctx->vgic_v3.ich_lr_num; i++)
hypctx_write_sys_reg(hypctx, HOST_ICH_LR_EL2(i), 0UL);
vgic_cpu->ich_lr_used = 0;
TAILQ_INIT(&vgic_cpu->irq_act_pend);
hypctx->vgic_v3_regs.ich_apr_num = virt_features.ich_apr_num;
hypctx->vgic_v3.ich_apr_num = virt_features.ich_apr_num;
}
static void
@@ -2118,7 +2119,7 @@ vgic_v3_flush_hwstate(device_t dev, struct hypctx *hypctx)
*/
mtx_lock_spin(&vgic_cpu->lr_mtx);
hypctx->vgic_v3_regs.ich_hcr_el2 &= ~ICH_HCR_EL2_UIE;
*hypctx_sys_reg(hypctx, HOST_ICH_HCR_EL2) &= ~ICH_HCR_EL2_UIE;
/* Exit early if there are no buffered interrupts */
if (TAILQ_EMPTY(&vgic_cpu->irq_act_pend))
@@ -2128,33 +2129,34 @@ vgic_v3_flush_hwstate(device_t dev, struct hypctx *hypctx)
__func__, vgic_cpu->ich_lr_used));
i = 0;
hypctx->vgic_v3_regs.ich_elrsr_el2 =
(1u << hypctx->vgic_v3_regs.ich_lr_num) - 1;
hypctx_write_sys_reg(hypctx, HOST_ICH_ELRSR_EL2,
(1u << hypctx->vgic_v3.ich_lr_num) - 1);
TAILQ_FOREACH(irq, &vgic_cpu->irq_act_pend, act_pend_list) {
/* No free list register, stop searching for IRQs */
if (i == hypctx->vgic_v3_regs.ich_lr_num)
if (i == hypctx->vgic_v3.ich_lr_num)
break;
if (!irq->enabled)
continue;
hypctx->vgic_v3_regs.ich_lr_el2[i] = ICH_LR_EL2_GROUP1 |
((uint64_t)irq->priority << ICH_LR_EL2_PRIO_SHIFT) |
irq->irq;
hypctx_write_sys_reg(hypctx, HOST_ICH_LR_EL2(i),
ICH_LR_EL2_GROUP1 |
((uint64_t)irq->priority << ICH_LR_EL2_PRIO_SHIFT) |
irq->irq);
if (irq->active) {
hypctx->vgic_v3_regs.ich_lr_el2[i] |=
*hypctx_sys_reg(hypctx, HOST_ICH_LR_EL2(i)) |=
ICH_LR_EL2_STATE_ACTIVE;
}
#ifdef notyet
/* TODO: Check why this is needed */
if ((irq->config & _MASK) == LEVEL)
hypctx->vgic_v3_regs.ich_lr_el2[i] |= ICH_LR_EL2_EOI;
*hypctx_sys_reg(hypctx, HOST_ICH_LR_EL2(i)) |= ICH_LR_EL2_EOI;
#endif
if (!irq->active && vgic_v3_irq_pending(irq)) {
hypctx->vgic_v3_regs.ich_lr_el2[i] |=
*hypctx_sys_reg(hypctx, HOST_ICH_LR_EL2(i)) |=
ICH_LR_EL2_STATE_PENDING;
/*
@@ -2196,8 +2198,8 @@ vgic_v3_sync_hwstate(device_t dev, struct hypctx *hypctx)
* access unlocked.
*/
for (i = 0; i < vgic_cpu->ich_lr_used; i++) {
lr = hypctx->vgic_v3_regs.ich_lr_el2[i];
hypctx->vgic_v3_regs.ich_lr_el2[i] = 0;
lr = hypctx_read_sys_reg(hypctx, HOST_ICH_LR_EL2(i));
hypctx_write_sys_reg(hypctx, HOST_ICH_LR_EL2(i), 0);
irq = vgic_v3_get_irq(hypctx->hyp, vcpu_vcpuid(hypctx->vcpu),
ICH_LR_EL2_VINTID(lr));
@@ -2244,7 +2246,7 @@ vgic_v3_sync_hwstate(device_t dev, struct hypctx *hypctx)
vgic_v3_release_irq(irq);
}
hypctx->vgic_v3_regs.ich_hcr_el2 &= ~ICH_HCR_EL2_EOICOUNT_MASK;
*hypctx_sys_reg(hypctx, HOST_ICH_HCR_EL2) &= ~ICH_HCR_EL2_EOICOUNT_MASK;
vgic_cpu->ich_lr_used = 0;
}
sys/arm64/vmm/io/vgic_v3.h
-22
View File
@@ -32,26 +32,4 @@
#define VGIC_ICH_LR_NUM_MAX 16
#define VGIC_ICH_APR_NUM_MAX 4
/* Registers accessed by EL2 */
struct vgic_v3_regs {
uint32_t ich_eisr_el2; /* End of Interrupt Status Register */
uint32_t ich_elrsr_el2; /* Empty List register Status Register (ICH_ELRSR_EL2) */
uint32_t ich_hcr_el2; /* Hyp Control Register */
uint32_t ich_misr_el2; /* Maintenance Interrupt State Register */
uint32_t ich_vmcr_el2; /* Virtual Machine Control Register */
/*
* The List Registers are part of the VM context and are modified on a
* world switch. They need to be allocated statically so they are
* mapped in the EL2 translation tables when struct hypctx is mapped.
*/
uint64_t ich_lr_el2[VGIC_ICH_LR_NUM_MAX];
uint16_t ich_lr_num;
/* Active Priorities Registers for Group 0 and 1 interrupts */
uint16_t ich_apr_num;
uint32_t ich_ap0r_el2[VGIC_ICH_APR_NUM_MAX];
uint32_t ich_ap1r_el2[VGIC_ICH_APR_NUM_MAX];
};
#endif /* !_VMM_VGIC_V3_H_ */
sys/arm64/vmm/io/vtimer.c
+18 -18
View File
@@ -97,7 +97,7 @@ vtimer_virtual_timer_intr(void *arg)
}
cntpct_el0 = READ_SPECIALREG(cntpct_el0) -
hypctx->vtimer.cntvoff_el2;
hypctx_read_sys_reg(hypctx, HOST_CNTVOFF_EL2);
if (hypctx->vtimer_cpu.virt_timer.cntx_cval_el0 < cntpct_el0)
vgic_inject_irq(hypctx->hyp, vcpu_vcpuid(hypctx->vcpu),
GT_VIRT_IRQ, true);
@@ -186,17 +186,17 @@ vtimer_cpuinit(struct hypctx *hypctx)
*
* TODO: Don't trap when FEAT_ECV is present
*/
hypctx->vtimer.cnthctl_el2 =
hypctx_write_sys_reg(hypctx, HOST_CNTHCTL_EL2,
CNTHCTL_E2H_EL0VCTEN_NOTRAP |
CNTHCTL_E2H_EL0VTEN_NOTRAP;
CNTHCTL_E2H_EL0VTEN_NOTRAP);
if (ecv_poff) {
hypctx->vtimer.cnthctl_el2 |=
*hypctx_sys_reg(hypctx, HOST_CNTHCTL_EL2) |=
CNTHCTL_E2H_EL0PCTEN_NOTRAP |
CNTHCTL_E2H_EL0PTEN_NOTRAP |
CNTHCTL_E2H_EL1PCTEN_NOTRAP |
CNTHCTL_E2H_EL1PTEN_NOTRAP;
} else {
hypctx->vtimer.cnthctl_el2 |=
*hypctx_sys_reg(hypctx, HOST_CNTHCTL_EL2) |=
CNTHCTL_E2H_EL0PCTEN_TRAP |
CNTHCTL_E2H_EL0PTEN_TRAP |
CNTHCTL_E2H_EL1PCTEN_TRAP |
@@ -209,20 +209,20 @@ vtimer_cpuinit(struct hypctx *hypctx)
* CNTHCTL_EL1PCTEN: trap access to CNTPCT_EL0
*/
if (ecv_poff) {
hypctx->vtimer.cnthctl_el2 =
hypctx_write_sys_reg(hypctx, HOST_CNTHCTL_EL2,
CNTHCTL_EL1PCTEN_NOTRAP |
CNTHCTL_EL1PCEN_NOTRAP;
CNTHCTL_EL1PCEN_NOTRAP);
} else {
hypctx->vtimer.cnthctl_el2 =
hypctx_write_sys_reg(hypctx, HOST_CNTHCTL_EL2,
CNTHCTL_EL1PCTEN_TRAP |
CNTHCTL_EL1PCEN_TRAP;
CNTHCTL_EL1PCEN_TRAP);
}
}
if (ecv_poff)
hypctx->vtimer.cnthctl_el2 |= CNTHCTL_ECV_EN;
*hypctx_sys_reg(hypctx, HOST_CNTHCTL_EL2) |= CNTHCTL_ECV_EN;
hypctx->vtimer.cntvoff_el2 = hypctx->hyp->cntvoff_el2;
hypctx_write_sys_reg(hypctx, HOST_CNTVOFF_EL2, hypctx->hyp->cntvoff_el2);
}
void
@@ -280,10 +280,10 @@ vtimer_sync_hwstate(struct hypctx *hypctx)
uint64_t cntpct_el0;
cntpct_el0 = READ_SPECIALREG(cntpct_el0) -
hypctx->vtimer.cntvoff_el2;
hypctx_read_sys_reg(hypctx, HOST_CNTVOFF_EL2);
vtime_sync_timer(hypctx, &hypctx->vtimer_cpu.virt_timer, cntpct_el0);
/* If FEAT_ECV_POFF is in use then we need to sync the physical timer */
if ((hypctx->vtimer.cnthctl_el2 & CNTHCTL_ECV_EN) != 0) {
if ((hypctx_read_sys_reg(hypctx, HOST_CNTHCTL_EL2) & CNTHCTL_ECV_EN) != 0) {
vtime_sync_timer(hypctx, &hypctx->vtimer_cpu.phys_timer,
cntpct_el0);
}
@@ -322,7 +322,7 @@ vtimer_schedule_irq(struct hypctx *hypctx, bool phys)
else
timer = &hypctx->vtimer_cpu.virt_timer;
cntpct_el0 = READ_SPECIALREG(cntpct_el0) -
hypctx->vtimer.cntvoff_el2;
hypctx_read_sys_reg(hypctx, HOST_CNTVOFF_EL2);
if (timer->cntx_cval_el0 < cntpct_el0) {
/* Timer set in the past, trigger interrupt */
vgic_inject_irq(hypctx->hyp, vcpu_vcpuid(hypctx->vcpu),
@@ -379,7 +379,7 @@ vtimer_phys_ctl_read(struct vcpu *vcpu, uint64_t *rval, void *arg)
hypctx = vcpu_get_cookie(vcpu);
vtimer_cpu = &hypctx->vtimer_cpu;
cntpct_el0 = READ_SPECIALREG(cntpct_el0) - hypctx->vtimer.cntvoff_el2;
cntpct_el0 = READ_SPECIALREG(cntpct_el0) - hypctx_read_sys_reg(hypctx, HOST_CNTVOFF_EL2);
if (vtimer_cpu->phys_timer.cntx_cval_el0 < cntpct_el0)
/* Timer condition met */
*rval = vtimer_cpu->phys_timer.cntx_ctl_el0 | CNTP_CTL_ISTATUS;
@@ -422,7 +422,7 @@ vtimer_phys_cnt_read(struct vcpu *vcpu, uint64_t *rval, void *arg)
struct hypctx *hypctx;
hypctx = vcpu_get_cookie(vcpu);
*rval = READ_SPECIALREG(cntpct_el0) - hypctx->vtimer.cntvoff_el2;
*rval = READ_SPECIALREG(cntpct_el0) - hypctx_read_sys_reg(hypctx, HOST_CNTVOFF_EL2);
return (0);
}
@@ -485,7 +485,7 @@ vtimer_phys_tval_read(struct vcpu *vcpu, uint64_t *rval, void *arg)
*rval = (uint32_t)RES1;
} else {
cntpct_el0 = READ_SPECIALREG(cntpct_el0) -
hypctx->vtimer.cntvoff_el2;
hypctx_read_sys_reg(hypctx, HOST_CNTVOFF_EL2);
*rval = vtimer_cpu->phys_timer.cntx_cval_el0 - cntpct_el0;
}
@@ -502,7 +502,7 @@ vtimer_phys_tval_write(struct vcpu *vcpu, uint64_t wval, void *arg)
hypctx = vcpu_get_cookie(vcpu);
vtimer_cpu = &hypctx->vtimer_cpu;
cntpct_el0 = READ_SPECIALREG(cntpct_el0) - hypctx->vtimer.cntvoff_el2;
cntpct_el0 = READ_SPECIALREG(cntpct_el0) - hypctx_read_sys_reg(hypctx, HOST_CNTVOFF_EL2);
vtimer_cpu->phys_timer.cntx_cval_el0 = (int32_t)wval + cntpct_el0;
vtimer_remove_irq(hypctx, vcpu);
sys/arm64/vmm/io/vtimer.h
-5
View File
@@ -37,11 +37,6 @@
struct hyp;
struct hypctx;
struct vtimer {
uint64_t cnthctl_el2;
uint64_t cntvoff_el2;
};
struct vtimer_timer {
struct callout callout;
struct mtx mtx;
sys/arm64/vmm/vmm.c
+3 -3
View File
@@ -862,13 +862,13 @@ vm_handle_smccc_call(struct vcpu *vcpu, struct vm_exit *vme, bool *retu)
hypctx = vcpu_get_cookie(vcpu);
if ((hypctx->tf.tf_esr & ESR_ELx_ISS_MASK) != 0)
if ((hypctx_read_sys_reg(hypctx, HOST_ESR_EL2) & ESR_ELx_ISS_MASK) != 0)
return (1);
vme->exitcode = VM_EXITCODE_SMCCC;
vme->u.smccc_call.func_id = hypctx->tf.tf_x[0];
vme->u.smccc_call.func_id = hypctx_read_sys_reg(hypctx, GPR_X(0));
for (i = 0; i < nitems(vme->u.smccc_call.args); i++)
vme->u.smccc_call.args[i] = hypctx->tf.tf_x[i + 1];
vme->u.smccc_call.args[i] = hypctx_read_sys_reg(hypctx, GPR_X(i + 1));
*retu = true;
return (0);
sys/arm64/vmm/vmm_arm64.c
+63 -47
View File
@@ -659,7 +659,7 @@ arm64_gen_inst_emul_data(struct hypctx *hypctx, uint32_t esr_iss,
~(TTBR_ASID_MASK | TTBR_CnP);
paging->tcr_el1 = hypctx_read_sys_reg(hypctx, TCR_EL1);
paging->tcr2_el1 = hypctx_read_sys_reg(hypctx, TCR2_EL1);
paging->flags = hypctx->tf.tf_spsr & (PSR_M_MASK | PSR_M_32);
paging->flags = hypctx_read_sys_reg(hypctx, HOST_SPSR_EL2) & (PSR_M_MASK | PSR_M_32);
if ((hypctx_read_sys_reg(hypctx, SCTLR_EL1) & SCTLR_M) != 0)
paging->flags |= VM_GP_MMU_ENABLED;
}
@@ -685,7 +685,7 @@ raise_data_insn_abort(struct hypctx *hypctx, uint64_t far, bool dabort, int fsc)
{
uint64_t esr;
if ((hypctx->tf.tf_spsr & PSR_M_MASK) == PSR_M_EL0t)
if ((hypctx_read_sys_reg(hypctx, HOST_SPSR_EL2) & PSR_M_MASK) == PSR_M_EL0t)
esr = EXCP_INSN_ABORT_L << ESR_ELx_EC_SHIFT;
else
esr = EXCP_INSN_ABORT << ESR_ELx_EC_SHIFT;
@@ -693,7 +693,7 @@ raise_data_insn_abort(struct hypctx *hypctx, uint64_t far, bool dabort, int fsc)
if (dabort)
esr |= EXCP_DATA_ABORT_L << ESR_ELx_EC_SHIFT;
/* Set the IL bit if set by hardware */
esr |= hypctx->tf.tf_esr & ESR_ELx_IL;
esr |= hypctx_read_sys_reg(hypctx, HOST_ESR_EL2) & ESR_ELx_IL;
vmmops_exception(hypctx, esr | fsc, far);
}
@@ -705,8 +705,8 @@ handle_el1_sync_excp(struct hypctx *hypctx, struct vm_exit *vme_ret,
uint64_t gpa;
uint32_t esr_ec, esr_iss;
esr_ec = ESR_ELx_EXCEPTION(hypctx->tf.tf_esr);
esr_iss = hypctx->tf.tf_esr & ESR_ELx_ISS_MASK;
esr_ec = ESR_ELx_EXCEPTION(hypctx_read_sys_reg(hypctx, HOST_ESR_EL2));
esr_iss = hypctx_read_sys_reg(hypctx, HOST_ESR_EL2) & ESR_ELx_ISS_MASK;
switch (esr_ec) {
case EXCP_UNKNOWN:
@@ -715,7 +715,7 @@ handle_el1_sync_excp(struct hypctx *hypctx, struct vm_exit *vme_ret,
vme_ret->exitcode = VM_EXITCODE_HYP;
break;
case EXCP_TRAP_WFI_WFE:
if ((hypctx->tf.tf_esr & 0x3) == 0) { /* WFI */
if ((hypctx_read_sys_reg(hypctx, HOST_ESR_EL2) & 0x3) == 0) { /* WFI */
vmm_stat_incr(hypctx->vcpu, VMEXIT_WFI, 1);
vme_ret->exitcode = VM_EXITCODE_WFI;
} else {
@@ -744,7 +744,7 @@ handle_el1_sync_excp(struct hypctx *hypctx, struct vm_exit *vme_ret,
case EXCP_DATA_ABORT_L:
vmm_stat_incr(hypctx->vcpu, esr_ec == EXCP_DATA_ABORT_L ?
VMEXIT_DATA_ABORT : VMEXIT_INSN_ABORT, 1);
switch (hypctx->tf.tf_esr & ISS_DATA_DFSC_MASK) {
switch (hypctx_read_sys_reg(hypctx, HOST_ESR_EL2) & ISS_DATA_DFSC_MASK) {
case ISS_DATA_DFSC_TF_L0:
case ISS_DATA_DFSC_TF_L1:
case ISS_DATA_DFSC_TF_L2:
@@ -769,7 +769,7 @@ handle_el1_sync_excp(struct hypctx *hypctx, struct vm_exit *vme_ret,
if (vm_mem_allocated(hypctx->vcpu, gpa)) {
vme_ret->exitcode = VM_EXITCODE_PAGING;
vme_ret->inst_length = 0;
vme_ret->u.paging.esr = hypctx->tf.tf_esr;
vme_ret->u.paging.esr = hypctx_read_sys_reg(hypctx, HOST_ESR_EL2);
vme_ret->u.paging.gpa = gpa;
} else if (esr_ec == EXCP_INSN_ABORT_L) {
/*
@@ -1094,7 +1094,7 @@ fault:
int
vmmops_run(void *vcpui, register_t pc, pmap_t pmap, struct vm_eventinfo *evinfo)
{
uint64_t excp_type, vbar_el1;
uint64_t excp_type, vbar_el1, new_spsr;
int handled;
register_t daif;
struct hyp *hyp;
@@ -1108,36 +1108,43 @@ vmmops_run(void *vcpui, register_t pc, pmap_t pmap, struct vm_eventinfo *evinfo)
vcpu = hypctx->vcpu;
vme = vm_exitinfo(vcpu);
hypctx->tf.tf_elr = (uint64_t)pc;
hypctx_write_sys_reg(hypctx, HOST_ELR_EL2, (uint64_t)pc);
for (;;) {
if (hypctx->has_exception) {
hypctx->has_exception = false;
hypctx_write_sys_reg(hypctx, ELR_EL1, hypctx->tf.tf_elr);
hypctx_write_sys_reg(hypctx, ELR_EL1,
hypctx_read_sys_reg(hypctx, HOST_ELR_EL2));
mode = hypctx->tf.tf_spsr & (PSR_M_MASK | PSR_M_32);
mode = hypctx_read_sys_reg(hypctx, HOST_SPSR_EL2) &
(PSR_M_MASK | PSR_M_32);
vbar_el1 = hypctx_read_sys_reg(hypctx, VBAR_EL1);
if (mode == PSR_M_EL1t) {
hypctx->tf.tf_elr = vbar_el1 + 0x0;
hypctx_write_sys_reg(hypctx, HOST_ELR_EL2,
vbar_el1 + 0x0);
} else if (mode == PSR_M_EL1h) {
hypctx->tf.tf_elr = vbar_el1 + 0x200;
hypctx_write_sys_reg(hypctx, HOST_ELR_EL2,
vbar_el1 + 0x200);
} else if ((mode & PSR_M_32) == PSR_M_64) {
/* 64-bit EL0 */
hypctx->tf.tf_elr = vbar_el1 + 0x400;
hypctx_write_sys_reg(hypctx, HOST_ELR_EL2,
vbar_el1 + 0x400);
} else {
/* 32-bit EL0 */
hypctx->tf.tf_elr = vbar_el1 + 0x600;
hypctx_write_sys_reg(hypctx, HOST_ELR_EL2,
vbar_el1 + 0x600);
}
/* Set the new spsr */
hypctx_write_sys_reg(hypctx, SPSR_EL1, hypctx->tf.tf_spsr);
new_spsr = hypctx_read_sys_reg(hypctx, HOST_SPSR_EL2);
hypctx_write_sys_reg(hypctx, SPSR_EL1, new_spsr);
/* Set the new cpsr */
hypctx->tf.tf_spsr = hypctx_read_sys_reg(hypctx,
SPSR_EL1) & PSR_FLAGS;
hypctx->tf.tf_spsr |= PSR_DAIF | PSR_M_EL1h;
hypctx_write_sys_reg(hypctx, HOST_SPSR_EL2,
new_spsr & PSR_FLAGS);
*hypctx_sys_reg(hypctx, HOST_SPSR_EL2) |= PSR_DAIF |
PSR_M_EL1h;
/*
* Update fields that may change on exeption entry
@@ -1145,12 +1152,15 @@ vmmops_run(void *vcpui, register_t pc, pmap_t pmap, struct vm_eventinfo *evinfo)
*/
if ((hypctx_read_sys_reg(hypctx, SCTLR_EL1) &
SCTLR_SPAN) == 0)
hypctx->tf.tf_spsr |= PSR_PAN;
*hypctx_sys_reg(hypctx,
HOST_SPSR_EL2) |= PSR_PAN;
if ((hypctx_read_sys_reg(hypctx, SCTLR_EL1) &
SCTLR_DSSBS) == 0)
hypctx->tf.tf_spsr &= ~PSR_SSBS;
*hypctx_sys_reg(hypctx,
HOST_SPSR_EL2) &= ~PSR_SSBS;
else
hypctx->tf.tf_spsr |= PSR_SSBS;
*hypctx_sys_reg(hypctx,
HOST_SPSR_EL2) |= PSR_SSBS;
}
daif = intr_disable();
@@ -1170,7 +1180,8 @@ vmmops_run(void *vcpui, register_t pc, pmap_t pmap, struct vm_eventinfo *evinfo)
/* Activate the stage2 pmap so the vmid is valid */
pmap_activate_vm(pmap);
hypctx_write_sys_reg(hypctx, HOST_VTTBR_EL2, pmap_to_ttbr0(pmap));
hypctx_write_sys_reg(hypctx, HOST_VTTBR_EL2,
pmap_to_ttbr0(pmap));
/*
* TODO: What happens if a timer interrupt is asserted exactly
@@ -1195,10 +1206,10 @@ vmmops_run(void *vcpui, register_t pc, pmap_t pmap, struct vm_eventinfo *evinfo)
if (excp_type == EXCP_TYPE_MAINT_IRQ)
continue;
vme->pc = hypctx->tf.tf_elr;
vme->pc = hypctx_read_sys_reg(hypctx, HOST_ELR_EL2);
vme->inst_length = INSN_SIZE;
vme->u.hyp.exception_nr = excp_type;
vme->u.hyp.esr_el2 = hypctx->tf.tf_esr;
vme->u.hyp.esr_el2 = hypctx_read_sys_reg(hypctx, HOST_ESR_EL2);
vme->u.hyp.far_el2 = hypctx_read_sys_reg(hypctx, HOST_FAR_EL2);
vme->u.hyp.hpfar_el2 = hypctx_read_sys_reg(hypctx, HOST_HPFAR_EL2);
@@ -1209,7 +1220,7 @@ vmmops_run(void *vcpui, register_t pc, pmap_t pmap, struct vm_eventinfo *evinfo)
break;
else
/* Resume guest execution from the next instruction. */
hypctx->tf.tf_elr += vme->inst_length;
*hypctx_sys_reg(hypctx, HOST_ELR_EL2) += vme->inst_length;
}
return (0);
@@ -1270,15 +1281,15 @@ hypctx_regptr(struct hypctx *hypctx, int reg)
{
switch (reg) {
case VM_REG_GUEST_X0 ... VM_REG_GUEST_X29:
return (&hypctx->tf.tf_x[reg]);
return hypctx_sys_reg(hypctx, GPR_X(reg));
case VM_REG_GUEST_LR:
return (&hypctx->tf.tf_lr);
return hypctx_sys_reg(hypctx, GPR_LR);
case VM_REG_GUEST_SP:
return (&hypctx->tf.tf_sp);
return hypctx_sys_reg(hypctx, HOST_SP_EL1);
case VM_REG_GUEST_CPSR:
return (&hypctx->tf.tf_spsr);
return hypctx_sys_reg(hypctx, HOST_SPSR_EL2);
case VM_REG_GUEST_PC:
return (&hypctx->tf.tf_elr);
return hypctx_sys_reg(hypctx, HOST_ELR_EL2);
case VM_REG_GUEST_SCTLR_EL1:
return hypctx_sys_reg(hypctx, SCTLR_EL1);
case VM_REG_GUEST_TTBR0_EL1:
@@ -1385,6 +1396,7 @@ vmmops_setcap(void *vcpui, int num, int val)
{
struct hypctx *hypctx = vcpui;
int ret;
uint64_t host_spsr_el2;
ret = 0;
@@ -1402,38 +1414,42 @@ vmmops_setcap(void *vcpui, int num, int val)
break;
if (val != 0) {
hypctx->debug_spsr |= (hypctx->tf.tf_spsr & PSR_SS);
hypctx->debug_mdscr |=
(hypctx_read_sys_reg(hypctx, MDSCR_EL1) & MDSCR_SS);
hypctx->debug_spsr |= (hypctx_read_sys_reg(hypctx,
HOST_SPSR_EL2) & PSR_SS);
hypctx->debug_mdscr |= (hypctx_read_sys_reg(hypctx,
MDSCR_EL1) & MDSCR_SS);
hypctx->tf.tf_spsr |= PSR_SS;
*hypctx_sys_reg(hypctx, HOST_SPSR_EL2) |= PSR_SS;
*hypctx_sys_reg(hypctx, MDSCR_EL1) |= MDSCR_SS;
*hypctx_sys_reg(hypctx, HOST_MDCR_EL2) |= MDCR_EL2_TDE;
} else {
hypctx->tf.tf_spsr &= ~PSR_SS;
hypctx->tf.tf_spsr |= hypctx->debug_spsr;
*hypctx_sys_reg(hypctx, HOST_SPSR_EL2) &= ~PSR_SS;
*hypctx_sys_reg(hypctx,
HOST_SPSR_EL2) |= hypctx->debug_spsr;
hypctx->debug_spsr &= ~PSR_SS;
*hypctx_sys_reg(hypctx, MDSCR_EL1) &= ~MDSCR_SS;
*hypctx_sys_reg(hypctx, MDSCR_EL1) |= hypctx->debug_mdscr;
*hypctx_sys_reg(hypctx,
MDSCR_EL1) |= hypctx->debug_mdscr;
hypctx->debug_mdscr &= ~MDSCR_SS;
if ((hypctx->setcaps & (1ul << VM_CAP_BRK_EXIT)) == 0)
*hypctx_sys_reg(hypctx, HOST_MDCR_EL2) &= ~MDCR_EL2_TDE;
*hypctx_sys_reg(hypctx,
HOST_MDCR_EL2) &= ~MDCR_EL2_TDE;
}
break;
case VM_CAP_MASK_HWINTR:
if ((val != 0) == ((hypctx->setcaps & (1ul << num)) != 0))
break;
host_spsr_el2 = hypctx_read_sys_reg(hypctx, HOST_SPSR_EL2);
if (val != 0) {
hypctx->debug_spsr |= (hypctx->tf.tf_spsr &
(PSR_I | PSR_F));
hypctx->tf.tf_spsr |= PSR_I | PSR_F;
hypctx->debug_spsr |= (host_spsr_el2 & (PSR_I | PSR_F));
host_spsr_el2 |= PSR_I | PSR_F;
} else {
hypctx->tf.tf_spsr &= ~(PSR_I | PSR_F);
hypctx->tf.tf_spsr |= (hypctx->debug_spsr &
(PSR_I | PSR_F));
host_spsr_el2 &= ~(PSR_I | PSR_F);
host_spsr_el2 |= (hypctx->debug_spsr & (PSR_I | PSR_F));
hypctx->debug_spsr &= ~(PSR_I | PSR_F);
}
hypctx_write_sys_reg(hypctx, HOST_SPSR_EL2, host_spsr_el2);
break;
default:
ret = ENOENT;
sys/arm64/vmm/vmm_hyp.c
+74 -61
View File
@@ -174,22 +174,27 @@ vmm_hyp_reg_store_pmu_debug(struct hypctx *hypctx, bool guest)
static void
vmm_hyp_reg_store_vgic(struct hypctx *hypctx, bool guest)
{
hypctx->vgic_v3_regs.ich_hcr_el2 = READ_SPECIALREG(ich_hcr_el2);
hypctx->vgic_v3_regs.ich_vmcr_el2 = READ_SPECIALREG(ich_vmcr_el2);
hypctx_write_sys_reg(hypctx, HOST_ICH_HCR_EL2,
READ_SPECIALREG(ich_hcr_el2));
hypctx_write_sys_reg(hypctx, HOST_ICH_VMCR_EL2,
READ_SPECIALREG(ich_vmcr_el2));
if (!guest)
return;
/* Store the GICv3 registers */
hypctx->vgic_v3_regs.ich_eisr_el2 = READ_SPECIALREG(ich_eisr_el2);
hypctx->vgic_v3_regs.ich_elrsr_el2 = READ_SPECIALREG(ich_elrsr_el2);
hypctx->vgic_v3_regs.ich_misr_el2 = READ_SPECIALREG(ich_misr_el2);
hypctx_write_sys_reg(hypctx, HOST_ICH_EISR_EL2,
READ_SPECIALREG(ich_eisr_el2));
hypctx_write_sys_reg(hypctx, HOST_ICH_ELRSR_EL2,
READ_SPECIALREG(ich_elrsr_el2));
hypctx_write_sys_reg(hypctx, HOST_ICH_MISR_EL2,
READ_SPECIALREG(ich_misr_el2));
switch (hypctx->vgic_v3_regs.ich_lr_num - 1) {
#define STORE_LR(x) \
case x: \
hypctx->vgic_v3_regs.ich_lr_el2[x] = \
READ_SPECIALREG(ich_lr ## x ##_el2)
switch (hypctx->vgic_v3.ich_lr_num - 1) {
#define STORE_LR(x) \
case x: \
hypctx_write_sys_reg(hypctx, HOST_ICH_LR_EL2(x), \
READ_SPECIALREG(ich_lr ## x ##_el2))
STORE_LR(15);
STORE_LR(14);
STORE_LR(13);
@@ -210,13 +215,13 @@ case x: \
#undef STORE_LR
}
switch (hypctx->vgic_v3_regs.ich_apr_num - 1) {
#define STORE_APR(x) \
case x: \
hypctx->vgic_v3_regs.ich_ap0r_el2[x] = \
READ_SPECIALREG(ich_ap0r ## x ##_el2); \
hypctx->vgic_v3_regs.ich_ap1r_el2[x] = \
READ_SPECIALREG(ich_ap1r ## x ##_el2)
switch (hypctx->vgic_v3.ich_apr_num - 1) {
#define STORE_APR(x) \
case x: \
hypctx_write_sys_reg(hypctx, HOST_ICH_AP0R_EL2(x), \
READ_SPECIALREG(ich_ap0r ## x ##_el2)); \
hypctx_write_sys_reg(hypctx, HOST_ICH_AP1R_EL2(x), \
READ_SPECIALREG(ich_ap1r ## x ##_el2))
STORE_APR(3);
STORE_APR(2);
STORE_APR(1);
@@ -242,13 +247,14 @@ vmm_hyp_reg_store_timer(struct hypctx *hypctx, struct hyp *hyp, bool guest)
} else {
hypctx->vtimer_cpu.cntkctl_el1 =
READ_SPECIALREG(cntkctl_el1);
hypctx->vtimer.cnthctl_el2 =
READ_SPECIALREG(cnthctl_el2);
hypctx->vtimer.cntvoff_el2 =
READ_SPECIALREG(cntvoff_el2);
hypctx_write_sys_reg(hypctx, HOST_CNTHCTL_EL2,
READ_SPECIALREG(cnthctl_el2));
hypctx_write_sys_reg(hypctx, HOST_CNTVOFF_EL2,
READ_SPECIALREG(cntvoff_el2));
}
ecv_poff = (hypctx->vtimer.cnthctl_el2 & CNTHCTL_ECV_EN) != 0;
ecv_poff = (hypctx_read_sys_reg(hypctx, HOST_CNTHCTL_EL2) &
CNTHCTL_ECV_EN) != 0;
if (guest_or_nonvhe(guest) && ecv_poff) {
/*
@@ -267,16 +273,17 @@ static void
vmm_hyp_reg_store_special(struct hypctx *hypctx, struct hyp *hyp, bool guest)
{
/* Store the special to from the trapframe */
hypctx->tf.tf_sp = READ_SPECIALREG(sp_el1);
hypctx->tf.tf_elr = READ_SPECIALREG(elr_el2);
hypctx->tf.tf_spsr = READ_SPECIALREG(spsr_el2);
hypctx_write_sys_reg(hypctx, HOST_SP_EL1, READ_SPECIALREG(sp_el1));
hypctx_write_sys_reg(hypctx, HOST_ELR_EL2, READ_SPECIALREG(elr_el2));
hypctx_write_sys_reg(hypctx, HOST_SPSR_EL2, READ_SPECIALREG(spsr_el2));
if (guest) {
hypctx->tf.tf_esr = READ_SPECIALREG(esr_el2);
hypctx_write_sys_reg(hypctx, HOST_ESR_EL2,
READ_SPECIALREG(esr_el2));
hypctx_write_sys_reg(hypctx, HOST_FAR_EL2,
READ_SPECIALREG(far_el2));
hypctx_write_sys_reg(hypctx,
PAR_EL1, READ_SPECIALREG(par_el1));
hypctx_write_sys_reg(hypctx, PAR_EL1,
READ_SPECIALREG(par_el1));
}
/* Store the guest special registers */
@@ -439,9 +446,9 @@ vmm_hyp_reg_restore_special(struct hypctx *hypctx, struct hyp *hyp, bool guest)
hypctx_read_sys_reg(hypctx, HOST_VMPIDR_EL2));
/* Load the special regs from the trapframe */
WRITE_SPECIALREG(sp_el1, hypctx->tf.tf_sp);
WRITE_SPECIALREG(elr_el2, hypctx->tf.tf_elr);
WRITE_SPECIALREG(spsr_el2, hypctx->tf.tf_spsr);
WRITE_SPECIALREG(sp_el1, hypctx_read_sys_reg(hypctx, HOST_SP_EL1));
WRITE_SPECIALREG(elr_el2, hypctx_read_sys_reg(hypctx, HOST_ELR_EL2));
WRITE_SPECIALREG(spsr_el2, hypctx_read_sys_reg(hypctx, HOST_SPSR_EL2));
}
static void
@@ -581,10 +588,13 @@ static void
vmm_hyp_reg_restore_timer(struct hypctx *hypctx, struct hyp *hyp, bool guest)
{
bool ecv_poff;
ecv_poff = (hypctx->vtimer.cnthctl_el2 & CNTHCTL_ECV_EN) != 0;
ecv_poff = (hypctx_read_sys_reg(hypctx, HOST_CNTHCTL_EL2) &
CNTHCTL_ECV_EN) != 0;
WRITE_SPECIALREG(cnthctl_el2, hypctx->vtimer.cnthctl_el2);
WRITE_SPECIALREG(cntvoff_el2, hypctx->vtimer.cntvoff_el2);
WRITE_SPECIALREG(cnthctl_el2,
hypctx_read_sys_reg(hypctx, HOST_CNTHCTL_EL2));
WRITE_SPECIALREG(cntvoff_el2,
hypctx_read_sys_reg(hypctx, HOST_CNTVOFF_EL2));
if (guest) {
WRITE_SPECIALREG(EL1_REG(CNTKCTL),
@@ -600,7 +610,7 @@ vmm_hyp_reg_restore_timer(struct hypctx *hypctx, struct hyp *hyp, bool guest)
* to keep in sync.
*/
WRITE_SPECIALREG(CNTPOFF_EL2_REG,
hypctx->vtimer.cntvoff_el2);
hypctx_read_sys_reg(hypctx, HOST_CNTVOFF_EL2));
isb();
}
} else {
@@ -623,18 +633,19 @@ static void
vmm_hyp_reg_restore_vgic(struct hypctx *hypctx, bool guest)
{
/* Load the GICv3 registers */
WRITE_SPECIALREG(ich_hcr_el2, hypctx->vgic_v3_regs.ich_hcr_el2);
WRITE_SPECIALREG(ich_hcr_el2,
hypctx_read_sys_reg(hypctx, HOST_ICH_HCR_EL2));
WRITE_SPECIALREG(ich_vmcr_el2,
hypctx->vgic_v3_regs.ich_vmcr_el2);
hypctx_read_sys_reg(hypctx, HOST_ICH_VMCR_EL2));
if (!guest)
return;
switch (hypctx->vgic_v3_regs.ich_lr_num - 1) {
#define LOAD_LR(x) \
case x: \
WRITE_SPECIALREG(ich_lr ## x ##_el2, \
hypctx->vgic_v3_regs.ich_lr_el2[x])
switch (hypctx->vgic_v3.ich_lr_num - 1) {
#define LOAD_LR(x) \
case x: \
WRITE_SPECIALREG(ich_lr ## x ##_el2, \
hypctx_read_sys_reg(hypctx, HOST_ICH_LR_EL2(x)))
LOAD_LR(15);
LOAD_LR(14);
LOAD_LR(13);
@@ -655,18 +666,18 @@ case x: \
#undef LOAD_LR
}
switch (hypctx->vgic_v3_regs.ich_apr_num - 1) {
#define LOAD_APR(x) \
case x: \
WRITE_SPECIALREG(ich_ap0r ## x ##_el2, \
hypctx->vgic_v3_regs.ich_ap0r_el2[x]); \
WRITE_SPECIALREG(ich_ap1r ## x ##_el2, \
hypctx->vgic_v3_regs.ich_ap1r_el2[x])
LOAD_APR(3);
LOAD_APR(2);
LOAD_APR(1);
default:
LOAD_APR(0);
switch (hypctx->vgic_v3.ich_apr_num - 1) {
#define LOAD_APR(x) \
case x: \
WRITE_SPECIALREG(ich_ap0r ## x ##_el2, \
hypctx_read_sys_reg(hypctx, HOST_ICH_AP0R_EL2(x))); \
WRITE_SPECIALREG(ich_ap1r ## x ##_el2, \
hypctx_read_sys_reg(hypctx, HOST_ICH_AP1R_EL2(x)))
LOAD_APR(3);
LOAD_APR(2);
LOAD_APR(1);
default:
LOAD_APR(0);
#undef LOAD_APR
}
}
@@ -693,11 +704,12 @@ static void
vmm_hyp_handle_guest_exit(struct hypctx *hypctx, uint64_t *ret)
{
bool hpfar_valid;
uint64_t s1e1r, hpfar_el2;
uint64_t s1e1r, hpfar_el2, host_esr_el2, host_far_el2;
hpfar_valid = true;
host_esr_el2 = hypctx_read_sys_reg(hypctx, HOST_ESR_EL2);
if (*ret == EXCP_TYPE_EL1_SYNC) {
switch (ESR_ELx_EXCEPTION(hypctx->tf.tf_esr)) {
switch (ESR_ELx_EXCEPTION(host_esr_el2)) {
case EXCP_INSN_ABORT_L:
case EXCP_DATA_ABORT_L:
/*
@@ -713,9 +725,9 @@ vmm_hyp_handle_guest_exit(struct hypctx *hypctx, uint64_t *ret)
*
* TODO: Add a case for Arm erratum 834220.
*/
if ((hypctx->tf.tf_esr & ISS_DATA_S1PTW) != 0)
if ((host_esr_el2 & ISS_DATA_S1PTW) != 0)
break;
switch (hypctx->tf.tf_esr & ISS_DATA_DFSC_MASK) {
switch (host_esr_el2 & ISS_DATA_DFSC_MASK) {
case ISS_DATA_DFSC_PF_L1:
case ISS_DATA_DFSC_PF_L2:
case ISS_DATA_DFSC_PF_L3:
@@ -726,14 +738,15 @@ vmm_hyp_handle_guest_exit(struct hypctx *hypctx, uint64_t *ret)
}
}
if (hpfar_valid) {
hypctx_write_sys_reg(hypctx, HOST_HPFAR_EL2, READ_SPECIALREG(hpfar_el2));
hypctx_write_sys_reg(hypctx, HOST_HPFAR_EL2,
READ_SPECIALREG(hpfar_el2));
} else {
/*
* TODO: There is a risk the at instruction could cause an
* exception here. We should handle it & return a failure.
*/
s1e1r =
arm64_address_translate_s1e1r(hypctx_read_sys_reg(hypctx, HOST_FAR_EL2));
host_far_el2 = hypctx_read_sys_reg(hypctx, HOST_FAR_EL2);
s1e1r = arm64_address_translate_s1e1r(host_far_el2);
if (PAR_SUCCESS(s1e1r)) {
hpfar_el2 = (s1e1r & PAR_PA_MASK) >> PAR_PA_SHIFT;
hpfar_el2 <<= HPFAR_EL2_FIPA_SHIFT;
sys/arm64/vmm/vmm_hyp_exception.S
+33 -32
View File
@@ -35,6 +35,7 @@
#include <machine/hypervisor.h>
#include "assym.inc"
#include "hyp_assym.h"
#include "hyp.h"
.macro save_host_registers
@@ -86,26 +87,26 @@
/* Restore the hypctx pointer */
mrs x0, tpidr_el2
stp x2, x3, [x0, #(TF_X + 2 * 8)]
stp x4, x5, [x0, #(TF_X + 4 * 8)]
stp x6, x7, [x0, #(TF_X + 6 * 8)]
stp x8, x9, [x0, #(TF_X + 8 * 8)]
stp x10, x11, [x0, #(TF_X + 10 * 8)]
stp x12, x13, [x0, #(TF_X + 12 * 8)]
stp x14, x15, [x0, #(TF_X + 14 * 8)]
stp x16, x17, [x0, #(TF_X + 16 * 8)]
stp x18, x19, [x0, #(TF_X + 18 * 8)]
stp x20, x21, [x0, #(TF_X + 20 * 8)]
stp x22, x23, [x0, #(TF_X + 22 * 8)]
stp x24, x25, [x0, #(TF_X + 24 * 8)]
stp x26, x27, [x0, #(TF_X + 26 * 8)]
stp x28, x29, [x0, #(TF_X + 28 * 8)]
stp x2, x3, [x0, #(HYPCTX_GPR_X0 + (2 * 8))]
stp x4, x5, [x0, #(HYPCTX_GPR_X0 + (4 * 8))]
stp x6, x7, [x0, #(HYPCTX_GPR_X0 + (6 * 8))]
stp x8, x9, [x0, #(HYPCTX_GPR_X0 + (8 * 8))]
stp x10, x11, [x0, #(HYPCTX_GPR_X0 + (10 * 8))]
stp x12, x13, [x0, #(HYPCTX_GPR_X0 + (12 * 8))]
stp x14, x15, [x0, #(HYPCTX_GPR_X0 + (14 * 8))]
stp x16, x17, [x0, #(HYPCTX_GPR_X0 + (16 * 8))]
stp x18, x19, [x0, #(HYPCTX_GPR_X0 + (18 * 8))]
stp x20, x21, [x0, #(HYPCTX_GPR_X0 + (20 * 8))]
stp x22, x23, [x0, #(HYPCTX_GPR_X0 + (22 * 8))]
stp x24, x25, [x0, #(HYPCTX_GPR_X0 + (24 * 8))]
stp x26, x27, [x0, #(HYPCTX_GPR_X0 + (26 * 8))]
stp x28, x29, [x0, #(HYPCTX_GPR_X0 + (28 * 8))]
str lr, [x0, #(TF_LR)]
str lr, [x0, #(HYPCTX_GPR_LR * 8)]
/* Restore the saved x0 & x1 and save them */
ldp x2, x3, [sp], #(2 * 8)
stp x2, x3, [x0, #(TF_X + 0 * 8)]
stp x2, x3, [x0, #(HYPCTX_GPR_X0 + (0 * 8))]
.endm
.macro restore_guest_registers
@@ -113,25 +114,25 @@
* Copy the guest x0 and x1 to the stack so we can restore them
* after loading the other registers.
*/
ldp x2, x3, [x0, #(TF_X + 0 * 8)]
ldp x2, x3, [x0, #(HYPCTX_GPR_X0 + (0 * 8))]
stp x2, x3, [sp, #-(2 * 8)]!
ldr lr, [x0, #(TF_LR)]
ldr lr, [x0, #(HYPCTX_GPR_LR * 8)]
ldp x28, x29, [x0, #(TF_X + 28 * 8)]
ldp x26, x27, [x0, #(TF_X + 26 * 8)]
ldp x24, x25, [x0, #(TF_X + 24 * 8)]
ldp x22, x23, [x0, #(TF_X + 22 * 8)]
ldp x20, x21, [x0, #(TF_X + 20 * 8)]
ldp x18, x19, [x0, #(TF_X + 18 * 8)]
ldp x16, x17, [x0, #(TF_X + 16 * 8)]
ldp x14, x15, [x0, #(TF_X + 14 * 8)]
ldp x12, x13, [x0, #(TF_X + 12 * 8)]
ldp x10, x11, [x0, #(TF_X + 10 * 8)]
ldp x8, x9, [x0, #(TF_X + 8 * 8)]
ldp x6, x7, [x0, #(TF_X + 6 * 8)]
ldp x4, x5, [x0, #(TF_X + 4 * 8)]
ldp x2, x3, [x0, #(TF_X + 2 * 8)]
ldp x28, x29, [x0, #(HYPCTX_GPR_X0 + (28 * 8))]
ldp x26, x27, [x0, #(HYPCTX_GPR_X0 + (26 * 8))]
ldp x24, x25, [x0, #(HYPCTX_GPR_X0 + (24 * 8))]
ldp x22, x23, [x0, #(HYPCTX_GPR_X0 + (22 * 8))]
ldp x20, x21, [x0, #(HYPCTX_GPR_X0 + (20 * 8))]
ldp x18, x19, [x0, #(HYPCTX_GPR_X0 + (18 * 8))]
ldp x16, x17, [x0, #(HYPCTX_GPR_X0 + (16 * 8))]
ldp x14, x15, [x0, #(HYPCTX_GPR_X0 + (14 * 8))]
ldp x12, x13, [x0, #(HYPCTX_GPR_X0 + (12 * 8))]
ldp x10, x11, [x0, #(HYPCTX_GPR_X0 + (10 * 8))]
ldp x8, x9, [x0, #(HYPCTX_GPR_X0 + (8 * 8))]
ldp x6, x7, [x0, #(HYPCTX_GPR_X0 + (6 * 8))]
ldp x4, x5, [x0, #(HYPCTX_GPR_X0 + (4 * 8))]
ldp x2, x3, [x0, #(HYPCTX_GPR_X0 + (2 * 8))]
ldp x0, x1, [sp], #(2 * 8)
.endm
sys/arm64/vmm/vmm_reset.c
+2 -4
View File
@@ -53,8 +53,6 @@ reset_vm_el01_regs(void *vcpu)
el2ctx = vcpu;
set_arch_unknown(el2ctx->tf);
/*
* Guest starts with:
* ~SCTLR_M: MMU off
@@ -152,9 +150,9 @@ reset_vm_el2_regs(void *vcpu)
* Disable interrupts in the guest. The guest OS will re-enable
* them.
*/
el2ctx->tf.tf_spsr = PSR_D | PSR_A | PSR_I | PSR_F;
hypctx_write_sys_reg(el2ctx, HOST_SPSR_EL2, PSR_D | PSR_A | PSR_I | PSR_F);
/* Use the EL1 stack when taking exceptions to EL1 */
el2ctx->tf.tf_spsr |= PSR_M_EL1h;
*hypctx_sys_reg(el2ctx, HOST_SPSR_EL2) |= PSR_M_EL1h;
/* FEAT_FGT traps */
if ((el2ctx->hyp->feats & HYP_FEAT_FGT) != 0) {
sys/modules/vmm/Makefile
+7 -4
View File
@@ -23,7 +23,7 @@ SRCS+= vmm.c \
.if ${MACHINE_CPUARCH} == "aarch64"
CFLAGS+= -I${SRCTOP}/sys/${MACHINE}/vmm/io
DPSRCS+= assym.inc
DPSRCS+= assym.inc hyp_genassym.c hyp_assym.h
# TODO: Add the new EL2 code
SRCS+= vmm_arm64.c \
@@ -47,6 +47,12 @@ CLEANFILES+= vmm_nvhe_exception.o vmm_nvhe.o
CLEANFILES+= vmm_hyp_blob.elf.full
CLEANFILES+= vmm_hyp_blob.elf vmm_hyp_blob.bin
hyp_assym.h: hyp_genassym.o
sh ${SYSDIR}/kern/genassym.sh hyp_genassym.o > ${.TARGET}
hyp_genassym.o: offset.inc
${CC} -c ${NOSAN_CFLAGS:N-flto*:N-fno-common} -fcommon ${.IMPSRC}
vmm_nvhe_exception.o: vmm_nvhe_exception.S vmm_hyp_exception.S
${CC} -c -x assembler-with-cpp -DLOCORE \
${NOSAN_CFLAGS:N-mbranch-protection*} ${.IMPSRC} -o ${.TARGET} -fpie
@@ -142,9 +148,6 @@ svm_support.o:
${CC} -c -x assembler-with-cpp -DLOCORE ${CFLAGS} \
${.IMPSRC} -o ${.TARGET}
hyp_genassym.o: offset.inc
${CC} -c ${NOSAN_CFLAGS:N-flto*:N-fno-common} -fcommon ${.IMPSRC}
vmx_genassym.o: offset.inc
${CC} -c ${NOSAN_CFLAGS:N-flto*:N-fno-common} -fcommon ${.IMPSRC}