VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR3/NEMR3Native-linux-armv8.cpp@ 104729

Last change on this file since 104729 was 104729, checked in by vboxsync, 6 months ago

VMM/NEMR3Native-linux-armv8.cpp: Sync the PAuth system registers, bugref:10391
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 62.2 KB
Line 
1/* $Id: NEMR3Native-linux-armv8.cpp 104729 2024-05-20 16:20:11Z vboxsync $ */
2/** @file
3 * NEM - Native execution manager, native ring-3 Linux backend arm64 version.
4 */
5
6/*
7 * Copyright (C) 2024 Oracle and/or its affiliates.
8 *
9 * This file is part of VirtualBox base platform packages, as
10 * available from https://www.virtualbox.org.
11 *
12 * This program is free software; you can redistribute it and/or
13 * modify it under the terms of the GNU General Public License
14 * as published by the Free Software Foundation, in version 3 of the
15 * License.
16 *
17 * This program is distributed in the hope that it will be useful, but
18 * WITHOUT ANY WARRANTY; without even the implied warranty of
19 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20 * General Public License for more details.
21 *
22 * You should have received a copy of the GNU General Public License
23 * along with this program; if not, see <https://www.gnu.org/licenses>.
24 *
25 * SPDX-License-Identifier: GPL-3.0-only
26 */
27
28
29/*********************************************************************************************************************************
30* Header Files *
31*********************************************************************************************************************************/
32#define LOG_GROUP LOG_GROUP_NEM
33#define VMCPU_INCL_CPUM_GST_CTX
34#include <VBox/vmm/nem.h>
35#include <VBox/vmm/iem.h>
36#include <VBox/vmm/em.h>
37#include <VBox/vmm/gic.h>
38#include <VBox/vmm/pdm.h>
39#include <VBox/vmm/trpm.h>
40#include "NEMInternal.h"
41#include <VBox/vmm/vmcc.h>
42
43#include <iprt/alloca.h>
44#include <iprt/string.h>
45#include <iprt/system.h>
46#include <iprt/armv8.h>
47
48#include <iprt/formats/arm-psci.h>
49
50#include <errno.h>
51#include <unistd.h>
52#include <sys/ioctl.h>
53#include <sys/fcntl.h>
54#include <sys/mman.h>
55#include <linux/kvm.h>
56
57
58/*********************************************************************************************************************************
59* Defined Constants And Macros *
60*********************************************************************************************************************************/
61
62/** Core register group. */
63#define KVM_ARM64_REG_CORE_GROUP UINT64_C(0x6030000000100000)
64/** System register group. */
65#define KVM_ARM64_REG_SYS_GROUP UINT64_C(0x6030000000130000)
66/** System register group. */
67#define KVM_ARM64_REG_SIMD_GROUP UINT64_C(0x6040000000100050)
68/** FP register group. */
69#define KVM_ARM64_REG_FP_GROUP UINT64_C(0x6020000000100000)
70
71#define KVM_ARM64_REG_CORE_CREATE(a_idReg) (KVM_ARM64_REG_CORE_GROUP | ((uint64_t)(a_idReg) & 0xffff))
72#define KVM_ARM64_REG_GPR(a_iGpr) KVM_ARM64_REG_CORE_CREATE((a_iGpr) << 1)
73#define KVM_ARM64_REG_SP_EL0 KVM_ARM64_REG_CORE_CREATE(0x3e)
74#define KVM_ARM64_REG_PC KVM_ARM64_REG_CORE_CREATE(0x40)
75#define KVM_ARM64_REG_PSTATE KVM_ARM64_REG_CORE_CREATE(0x42)
76#define KVM_ARM64_REG_SP_EL1 KVM_ARM64_REG_CORE_CREATE(0x44)
77#define KVM_ARM64_REG_ELR_EL1 KVM_ARM64_REG_CORE_CREATE(0x46)
78#define KVM_ARM64_REG_SPSR_EL1 KVM_ARM64_REG_CORE_CREATE(0x48)
79#define KVM_ARM64_REG_SPSR_ABT KVM_ARM64_REG_CORE_CREATE(0x4a)
80#define KVM_ARM64_REG_SPSR_UND KVM_ARM64_REG_CORE_CREATE(0x4c)
81#define KVM_ARM64_REG_SPSR_IRQ KVM_ARM64_REG_CORE_CREATE(0x4e)
82#define KVM_ARM64_REG_SPSR_FIQ KVM_ARM64_REG_CORE_CREATE(0x50)
83
84/** This maps to our IPRT representation of system register IDs, yay! */
85#define KVM_ARM64_REG_SYS_CREATE(a_idSysReg) (KVM_ARM64_REG_SYS_GROUP | ((uint64_t)(a_idSysReg) & 0xffff))
86
87#define KVM_ARM64_REG_SIMD_CREATE(a_iVecReg) (KVM_ARM64_REG_SIMD_GROUP | (((uint64_t)(a_iVecReg) << 2) & 0xffff))
88
89#define KVM_ARM64_REG_FP_CREATE(a_idReg) (KVM_ARM64_REG_FP_GROUP | ((uint64_t)(a_idReg) & 0xffff))
90#define KVM_ARM64_REG_FP_FPSR KVM_ARM64_REG_FP_CREATE(0xd4)
91#define KVM_ARM64_REG_FP_FPCR KVM_ARM64_REG_FP_CREATE(0xd5)
92
93
94/*********************************************************************************************************************************
95* Structures and Typedefs *
96*********************************************************************************************************************************/
97
98
99/*********************************************************************************************************************************
100* Global Variables *
101*********************************************************************************************************************************/
102/** The general registers. */
103static const struct
104{
105 uint64_t idKvmReg;
106 uint32_t fCpumExtrn;
107 uint32_t offCpumCtx;
108} s_aCpumRegs[] =
109{
110#define CPUM_GREG_EMIT_X0_X3(a_Idx) { KVM_ARM64_REG_GPR(a_Idx), CPUMCTX_EXTRN_X ## a_Idx, RT_UOFFSETOF(CPUMCTX, aGRegs[a_Idx].x) }
111#define CPUM_GREG_EMIT_X4_X28(a_Idx) { KVM_ARM64_REG_GPR(a_Idx), CPUMCTX_EXTRN_X4_X28, RT_UOFFSETOF(CPUMCTX, aGRegs[a_Idx].x) }
112 CPUM_GREG_EMIT_X0_X3(0),
113 CPUM_GREG_EMIT_X0_X3(1),
114 CPUM_GREG_EMIT_X0_X3(2),
115 CPUM_GREG_EMIT_X0_X3(3),
116 CPUM_GREG_EMIT_X4_X28(4),
117 CPUM_GREG_EMIT_X4_X28(5),
118 CPUM_GREG_EMIT_X4_X28(6),
119 CPUM_GREG_EMIT_X4_X28(7),
120 CPUM_GREG_EMIT_X4_X28(8),
121 CPUM_GREG_EMIT_X4_X28(9),
122 CPUM_GREG_EMIT_X4_X28(10),
123 CPUM_GREG_EMIT_X4_X28(11),
124 CPUM_GREG_EMIT_X4_X28(12),
125 CPUM_GREG_EMIT_X4_X28(13),
126 CPUM_GREG_EMIT_X4_X28(14),
127 CPUM_GREG_EMIT_X4_X28(15),
128 CPUM_GREG_EMIT_X4_X28(16),
129 CPUM_GREG_EMIT_X4_X28(17),
130 CPUM_GREG_EMIT_X4_X28(18),
131 CPUM_GREG_EMIT_X4_X28(19),
132 CPUM_GREG_EMIT_X4_X28(20),
133 CPUM_GREG_EMIT_X4_X28(21),
134 CPUM_GREG_EMIT_X4_X28(22),
135 CPUM_GREG_EMIT_X4_X28(23),
136 CPUM_GREG_EMIT_X4_X28(24),
137 CPUM_GREG_EMIT_X4_X28(25),
138 CPUM_GREG_EMIT_X4_X28(26),
139 CPUM_GREG_EMIT_X4_X28(27),
140 CPUM_GREG_EMIT_X4_X28(28),
141 { KVM_ARM64_REG_GPR(29), CPUMCTX_EXTRN_FP, RT_UOFFSETOF(CPUMCTX, aGRegs[29].x) },
142 { KVM_ARM64_REG_GPR(30), CPUMCTX_EXTRN_LR, RT_UOFFSETOF(CPUMCTX, aGRegs[30].x) },
143 { KVM_ARM64_REG_PC, CPUMCTX_EXTRN_PC, RT_UOFFSETOF(CPUMCTX, Pc.u64) },
144#undef CPUM_GREG_EMIT_X0_X3
145#undef CPUM_GREG_EMIT_X4_X28
146};
147/** SIMD/FP registers. */
148static const struct
149{
150 uint64_t idKvmReg;
151 uint32_t offCpumCtx;
152} s_aCpumFpRegs[] =
153{
154#define CPUM_VREG_EMIT(a_Idx) { KVM_ARM64_REG_SIMD_CREATE(a_Idx), RT_UOFFSETOF(CPUMCTX, aVRegs[a_Idx].v) }
155 CPUM_VREG_EMIT(0),
156 CPUM_VREG_EMIT(1),
157 CPUM_VREG_EMIT(2),
158 CPUM_VREG_EMIT(3),
159 CPUM_VREG_EMIT(4),
160 CPUM_VREG_EMIT(5),
161 CPUM_VREG_EMIT(6),
162 CPUM_VREG_EMIT(7),
163 CPUM_VREG_EMIT(8),
164 CPUM_VREG_EMIT(9),
165 CPUM_VREG_EMIT(10),
166 CPUM_VREG_EMIT(11),
167 CPUM_VREG_EMIT(12),
168 CPUM_VREG_EMIT(13),
169 CPUM_VREG_EMIT(14),
170 CPUM_VREG_EMIT(15),
171 CPUM_VREG_EMIT(16),
172 CPUM_VREG_EMIT(17),
173 CPUM_VREG_EMIT(18),
174 CPUM_VREG_EMIT(19),
175 CPUM_VREG_EMIT(20),
176 CPUM_VREG_EMIT(21),
177 CPUM_VREG_EMIT(22),
178 CPUM_VREG_EMIT(23),
179 CPUM_VREG_EMIT(24),
180 CPUM_VREG_EMIT(25),
181 CPUM_VREG_EMIT(26),
182 CPUM_VREG_EMIT(27),
183 CPUM_VREG_EMIT(28),
184 CPUM_VREG_EMIT(29),
185 CPUM_VREG_EMIT(30),
186 CPUM_VREG_EMIT(31)
187#undef CPUM_VREG_EMIT
188};
189/** System registers. */
190static const struct
191{
192 uint64_t idKvmReg;
193 uint32_t fCpumExtrn;
194 uint32_t offCpumCtx;
195} s_aCpumSysRegs[] =
196{
197 { KVM_ARM64_REG_SP_EL0, CPUMCTX_EXTRN_SP, RT_UOFFSETOF(CPUMCTX, aSpReg[0].u64) },
198 { KVM_ARM64_REG_SP_EL1, CPUMCTX_EXTRN_SP, RT_UOFFSETOF(CPUMCTX, aSpReg[1].u64) },
199 { KVM_ARM64_REG_SPSR_EL1, CPUMCTX_EXTRN_SPSR, RT_UOFFSETOF(CPUMCTX, Spsr.u64) },
200 { KVM_ARM64_REG_ELR_EL1, CPUMCTX_EXTRN_ELR, RT_UOFFSETOF(CPUMCTX, Elr.u64) },
201 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_SCTRL_EL1), CPUMCTX_EXTRN_SCTLR_TCR_TTBR, RT_UOFFSETOF(CPUMCTX, Sctlr.u64) },
202 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_TCR_EL1), CPUMCTX_EXTRN_SCTLR_TCR_TTBR, RT_UOFFSETOF(CPUMCTX, Tcr.u64) },
203 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_TTBR0_EL1), CPUMCTX_EXTRN_SCTLR_TCR_TTBR, RT_UOFFSETOF(CPUMCTX, Ttbr0.u64) },
204 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_TTBR1_EL1), CPUMCTX_EXTRN_SCTLR_TCR_TTBR, RT_UOFFSETOF(CPUMCTX, Ttbr1.u64) },
205 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_VBAR_EL1), CPUMCTX_EXTRN_SYSREG_MISC, RT_UOFFSETOF(CPUMCTX, VBar.u64) },
206 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_AFSR0_EL1), CPUMCTX_EXTRN_SYSREG_MISC, RT_UOFFSETOF(CPUMCTX, Afsr0.u64) },
207 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_AFSR1_EL1), CPUMCTX_EXTRN_SYSREG_MISC, RT_UOFFSETOF(CPUMCTX, Afsr1.u64) },
208 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_AMAIR_EL1), CPUMCTX_EXTRN_SYSREG_MISC, RT_UOFFSETOF(CPUMCTX, Amair.u64) },
209 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_CNTKCTL_EL1), CPUMCTX_EXTRN_SYSREG_MISC, RT_UOFFSETOF(CPUMCTX, CntKCtl.u64) },
210 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_CONTEXTIDR_EL1), CPUMCTX_EXTRN_SYSREG_MISC, RT_UOFFSETOF(CPUMCTX, ContextIdr.u64) },
211 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_CPACR_EL1), CPUMCTX_EXTRN_SYSREG_MISC, RT_UOFFSETOF(CPUMCTX, Cpacr.u64) },
212 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_CSSELR_EL1), CPUMCTX_EXTRN_SYSREG_MISC, RT_UOFFSETOF(CPUMCTX, Csselr.u64) },
213 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_ESR_EL1), CPUMCTX_EXTRN_SYSREG_MISC, RT_UOFFSETOF(CPUMCTX, Esr.u64) },
214 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_FAR_EL1), CPUMCTX_EXTRN_SYSREG_MISC, RT_UOFFSETOF(CPUMCTX, Far.u64) },
215 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_MAIR_EL1), CPUMCTX_EXTRN_SYSREG_MISC, RT_UOFFSETOF(CPUMCTX, Mair.u64) },
216 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_PAR_EL1), CPUMCTX_EXTRN_SYSREG_MISC, RT_UOFFSETOF(CPUMCTX, Par.u64) },
217 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_TPIDRRO_EL0), CPUMCTX_EXTRN_SYSREG_MISC, RT_UOFFSETOF(CPUMCTX, TpIdrRoEl0.u64) },
218 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_TPIDR_EL0), CPUMCTX_EXTRN_SYSREG_MISC, RT_UOFFSETOF(CPUMCTX, aTpIdr[0].u64) },
219 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_TPIDR_EL1), CPUMCTX_EXTRN_SYSREG_MISC, RT_UOFFSETOF(CPUMCTX, aTpIdr[1].u64) },
220 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_MDCCINT_EL1), CPUMCTX_EXTRN_SYSREG_MISC, RT_UOFFSETOF(CPUMCTX, MDccInt.u64) }
221};
222/** Debug system registers. */
223static const struct
224{
225 uint64_t idKvmReg;
226 uint32_t offCpumCtx;
227} s_aCpumDbgRegs[] =
228{
229#define CPUM_DBGREG_EMIT(a_BorW, a_Idx) \
230 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_DBG ## a_BorW ## CRn_EL1(a_Idx)), RT_UOFFSETOF(CPUMCTX, a ## a_BorW ## p[a_Idx].Ctrl.u64) }, \
231 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_DBG ## a_BorW ## VRn_EL1(a_Idx)), RT_UOFFSETOF(CPUMCTX, a ## a_BorW ## p[a_Idx].Value.u64) }
232 /* Breakpoint registers. */
233 CPUM_DBGREG_EMIT(B, 0),
234 CPUM_DBGREG_EMIT(B, 1),
235 CPUM_DBGREG_EMIT(B, 2),
236 CPUM_DBGREG_EMIT(B, 3),
237 CPUM_DBGREG_EMIT(B, 4),
238 CPUM_DBGREG_EMIT(B, 5),
239 CPUM_DBGREG_EMIT(B, 6),
240 CPUM_DBGREG_EMIT(B, 7),
241 CPUM_DBGREG_EMIT(B, 8),
242 CPUM_DBGREG_EMIT(B, 9),
243 CPUM_DBGREG_EMIT(B, 10),
244 CPUM_DBGREG_EMIT(B, 11),
245 CPUM_DBGREG_EMIT(B, 12),
246 CPUM_DBGREG_EMIT(B, 13),
247 CPUM_DBGREG_EMIT(B, 14),
248 CPUM_DBGREG_EMIT(B, 15),
249 /* Watchpoint registers. */
250 CPUM_DBGREG_EMIT(W, 0),
251 CPUM_DBGREG_EMIT(W, 1),
252 CPUM_DBGREG_EMIT(W, 2),
253 CPUM_DBGREG_EMIT(W, 3),
254 CPUM_DBGREG_EMIT(W, 4),
255 CPUM_DBGREG_EMIT(W, 5),
256 CPUM_DBGREG_EMIT(W, 6),
257 CPUM_DBGREG_EMIT(W, 7),
258 CPUM_DBGREG_EMIT(W, 8),
259 CPUM_DBGREG_EMIT(W, 9),
260 CPUM_DBGREG_EMIT(W, 10),
261 CPUM_DBGREG_EMIT(W, 11),
262 CPUM_DBGREG_EMIT(W, 12),
263 CPUM_DBGREG_EMIT(W, 13),
264 CPUM_DBGREG_EMIT(W, 14),
265 CPUM_DBGREG_EMIT(W, 15),
266 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_MDSCR_EL1), RT_UOFFSETOF(CPUMCTX, Mdscr.u64) }
267#undef CPUM_DBGREG_EMIT
268};
269/** PAuth key system registers. */
270static const struct
271{
272 uint64_t idKvmReg;
273 uint32_t offCpumCtx;
274} s_aCpumPAuthKeyRegs[] =
275{
276 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_APDAKeyLo_EL1), RT_UOFFSETOF(CPUMCTX, Apda.Low.u64) },
277 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_APDAKeyHi_EL1), RT_UOFFSETOF(CPUMCTX, Apda.High.u64) },
278 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_APDBKeyLo_EL1), RT_UOFFSETOF(CPUMCTX, Apdb.Low.u64) },
279 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_APDBKeyHi_EL1), RT_UOFFSETOF(CPUMCTX, Apdb.High.u64) },
280 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_APGAKeyLo_EL1), RT_UOFFSETOF(CPUMCTX, Apga.Low.u64) },
281 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_APGAKeyHi_EL1), RT_UOFFSETOF(CPUMCTX, Apga.High.u64) },
282 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_APIAKeyLo_EL1), RT_UOFFSETOF(CPUMCTX, Apia.Low.u64) },
283 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_APIAKeyHi_EL1), RT_UOFFSETOF(CPUMCTX, Apia.High.u64) },
284 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_APIBKeyLo_EL1), RT_UOFFSETOF(CPUMCTX, Apib.Low.u64) },
285 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_APIBKeyHi_EL1), RT_UOFFSETOF(CPUMCTX, Apib.High.u64) }
286};
287/** ID registers. */
288static const struct
289{
290 uint64_t idKvmReg;
291 uint32_t offIdStruct;
292} s_aIdRegs[] =
293{
294 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_ID_AA64DFR0_EL1), RT_UOFFSETOF(CPUMIDREGS, u64RegIdAa64Dfr0El1) },
295 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_ID_AA64DFR1_EL1), RT_UOFFSETOF(CPUMIDREGS, u64RegIdAa64Dfr1El1) },
296 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_ID_AA64ISAR0_EL1), RT_UOFFSETOF(CPUMIDREGS, u64RegIdAa64Isar0El1) },
297 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_ID_AA64ISAR1_EL1), RT_UOFFSETOF(CPUMIDREGS, u64RegIdAa64Isar1El1) },
298 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_ID_AA64MMFR0_EL1), RT_UOFFSETOF(CPUMIDREGS, u64RegIdAa64Mmfr0El1) },
299 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_ID_AA64MMFR1_EL1), RT_UOFFSETOF(CPUMIDREGS, u64RegIdAa64Mmfr1El1) },
300 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_ID_AA64MMFR2_EL1), RT_UOFFSETOF(CPUMIDREGS, u64RegIdAa64Mmfr2El1) },
301 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_ID_AA64PFR0_EL1), RT_UOFFSETOF(CPUMIDREGS, u64RegIdAa64Pfr0El1) },
302 { KVM_ARM64_REG_SYS_CREATE(ARMV8_AARCH64_SYSREG_ID_AA64PFR1_EL1), RT_UOFFSETOF(CPUMIDREGS, u64RegIdAa64Pfr1El1) }
303};
304
305
306
307/* Forward declarations of things called by the template. */
308static int nemR3LnxInitSetupVm(PVM pVM, PRTERRINFO pErrInfo);
309
310
311/* Instantiate the common bits we share with the x86 KVM backend. */
312#include "NEMR3NativeTemplate-linux.cpp.h"
313
314
315/**
316 * Queries and logs the supported register list from KVM.
317 *
318 * @returns VBox status code.
319 * @param fdVCpu The file descriptor number of vCPU 0.
320 */
321static int nemR3LnxLogRegList(int fdVCpu)
322{
323 struct KVM_REG_LIST
324 {
325 uint64_t cRegs;
326 uint64_t aRegs[1024];
327 } RegList; RT_ZERO(RegList);
328
329 RegList.cRegs = RT_ELEMENTS(RegList.aRegs);
330 int rcLnx = ioctl(fdVCpu, KVM_GET_REG_LIST, &RegList);
331 if (rcLnx != 0)
332 return RTErrConvertFromErrno(errno);
333
334 LogRel(("NEM: KVM vCPU registers:\n"));
335
336 for (uint32_t i = 0; i < RegList.cRegs; i++)
337 LogRel(("NEM: %36s: %#RX64\n", "Unknown" /** @todo */, RegList.aRegs[i]));
338
339 return VINF_SUCCESS;
340}
341
342
343/**
344 * Sets the given attribute in KVM to the given value.
345 *
346 * @returns VBox status code.
347 * @param pVM The VM instance.
348 * @param u32Grp The device attribute group being set.
349 * @param u32Attr The actual attribute inside the group being set.
350 * @param pvAttrVal Where the attribute value to set.
351 * @param pszAttribute Attribute description for logging.
352 * @param pErrInfo Optional error information.
353 */
354static int nemR3LnxSetAttribute(PVM pVM, uint32_t u32Grp, uint32_t u32Attr, const void *pvAttrVal, const char *pszAttribute,
355 PRTERRINFO pErrInfo)
356{
357 struct kvm_device_attr DevAttr;
358
359 DevAttr.flags = 0;
360 DevAttr.group = u32Grp;
361 DevAttr.attr = u32Attr;
362 DevAttr.addr = (uintptr_t)pvAttrVal;
363 int rcLnx = ioctl(pVM->nem.s.fdVm, KVM_HAS_DEVICE_ATTR, &DevAttr);
364 if (rcLnx < 0)
365 return RTErrInfoSetF(pErrInfo, RTErrConvertFromErrno(errno),
366 N_("KVM error: KVM doesn't support setting the attribute \"%s\" (%d)"),
367 pszAttribute, errno);
368
369 rcLnx = ioctl(pVM->nem.s.fdVm, KVM_SET_DEVICE_ATTR, &DevAttr);
370 if (rcLnx < 0)
371 return RTErrInfoSetF(pErrInfo, RTErrConvertFromErrno(errno),
372 N_("KVM error: Setting the attribute \"%s\" for KVM failed (%d)"),
373 pszAttribute, errno);
374
375 return VINF_SUCCESS;
376}
377
378
379DECL_FORCE_INLINE(int) nemR3LnxKvmSetQueryReg(PVMCPUCC pVCpu, bool fQuery, uint64_t idKvmReg, const void *pv)
380{
381 struct kvm_one_reg Reg;
382 Reg.id = idKvmReg;
383 Reg.addr = (uintptr_t)pv;
384
385 /*
386 * Who thought that this API was a good idea? Supporting to query/set just one register
387 * at a time is horribly inefficient.
388 */
389 int rcLnx = ioctl(pVCpu->nem.s.fdVCpu, fQuery ? KVM_GET_ONE_REG : KVM_SET_ONE_REG, &Reg);
390 if (!rcLnx)
391 return 0;
392
393 return RTErrConvertFromErrno(-rcLnx);
394}
395
396DECLINLINE(int) nemR3LnxKvmQueryRegU64(PVMCPUCC pVCpu, uint64_t idKvmReg, uint64_t *pu64)
397{
398 return nemR3LnxKvmSetQueryReg(pVCpu, true /*fQuery*/, idKvmReg, pu64);
399}
400
401
402DECLINLINE(int) nemR3LnxKvmQueryRegU32(PVMCPUCC pVCpu, uint64_t idKvmReg, uint32_t *pu32)
403{
404 return nemR3LnxKvmSetQueryReg(pVCpu, true /*fQuery*/, idKvmReg, pu32);
405}
406
407
408DECLINLINE(int) nemR3LnxKvmQueryRegPV(PVMCPUCC pVCpu, uint64_t idKvmReg, void *pv)
409{
410 return nemR3LnxKvmSetQueryReg(pVCpu, true /*fQuery*/, idKvmReg, pv);
411}
412
413
414DECLINLINE(int) nemR3LnxKvmSetRegU64(PVMCPUCC pVCpu, uint64_t idKvmReg, const uint64_t *pu64)
415{
416 return nemR3LnxKvmSetQueryReg(pVCpu, false /*fQuery*/, idKvmReg, pu64);
417}
418
419
420DECLINLINE(int) nemR3LnxKvmSetRegU32(PVMCPUCC pVCpu, uint64_t idKvmReg, const uint32_t *pu32)
421{
422 return nemR3LnxKvmSetQueryReg(pVCpu, false /*fQuery*/, idKvmReg, pu32);
423}
424
425
426DECLINLINE(int) nemR3LnxKvmSetRegPV(PVMCPUCC pVCpu, uint64_t idKvmReg, const void *pv)
427{
428 return nemR3LnxKvmSetQueryReg(pVCpu, false /*fQuery*/, idKvmReg, pv);
429}
430
431
432/**
433 * Does the early setup of a KVM VM.
434 *
435 * @returns VBox status code.
436 * @param pVM The cross context VM structure.
437 * @param pErrInfo Where to always return error info.
438 */
439static int nemR3LnxInitSetupVm(PVM pVM, PRTERRINFO pErrInfo)
440{
441 AssertReturn(pVM->nem.s.fdVm != -1, RTErrInfoSet(pErrInfo, VERR_WRONG_ORDER, "Wrong initalization order"));
442
443 /*
444 * Create the VCpus.
445 */
446 for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
447 {
448 PVMCPU pVCpu = pVM->apCpusR3[idCpu];
449
450 /* Create it. */
451 pVCpu->nem.s.fdVCpu = ioctl(pVM->nem.s.fdVm, KVM_CREATE_VCPU, (unsigned long)idCpu);
452 if (pVCpu->nem.s.fdVCpu < 0)
453 return RTErrInfoSetF(pErrInfo, VERR_NEM_VM_CREATE_FAILED, "KVM_CREATE_VCPU failed for VCpu #%u: %d", idCpu, errno);
454
455 /* Map the KVM_RUN area. */
456 pVCpu->nem.s.pRun = (struct kvm_run *)mmap(NULL, pVM->nem.s.cbVCpuMmap, PROT_READ | PROT_WRITE, MAP_SHARED,
457 pVCpu->nem.s.fdVCpu, 0 /*offset*/);
458 if ((void *)pVCpu->nem.s.pRun == MAP_FAILED)
459 return RTErrInfoSetF(pErrInfo, VERR_NEM_VM_CREATE_FAILED, "mmap failed for VCpu #%u: %d", idCpu, errno);
460
461 /* Initialize the vCPU. */
462 struct kvm_vcpu_init VCpuInit; RT_ZERO(VCpuInit);
463 VCpuInit.target = KVM_ARM_TARGET_GENERIC_V8;
464 /** @todo Enable features. */
465 if (ioctl(pVCpu->nem.s.fdVCpu, KVM_ARM_VCPU_INIT, &VCpuInit) != 0)
466 return RTErrInfoSetF(pErrInfo, VERR_NEM_VM_CREATE_FAILED, "KVM_ARM_VCPU_INIT failed for VCpu #%u: %d", idCpu, errno);
467
468#if 0
469 uint32_t fFeatures = 0; /** @todo SVE */
470 if (ioctl(pVCpu->nem.s.fdVCpu, KVM_ARM_VCPU_FINALIZE, &fFeatures) != 0)
471 return RTErrInfoSetF(pErrInfo, VERR_NEM_VM_CREATE_FAILED, "KVM_ARM_VCPU_FINALIZE failed for VCpu #%u: %d", idCpu, errno);
472#endif
473
474 if (idCpu == 0)
475 {
476 /* Query the supported register list and log it. */
477 int rc = nemR3LnxLogRegList(pVCpu->nem.s.fdVCpu);
478 if (RT_FAILURE(rc))
479 return RTErrInfoSetF(pErrInfo, VERR_NEM_VM_CREATE_FAILED, "Querying the supported register list failed with %Rrc", rc);
480
481 /* Need to query the ID registers and populate CPUM. */
482 CPUMIDREGS IdRegs; RT_ZERO(IdRegs);
483 for (uint32_t i = 0; i < RT_ELEMENTS(s_aIdRegs); i++)
484 {
485 uint64_t *pu64 = (uint64_t *)((uint8_t *)&IdRegs + s_aIdRegs[i].offIdStruct);
486 rc = nemR3LnxKvmQueryRegU64(pVCpu, s_aIdRegs[i].idKvmReg, pu64);
487 if (RT_FAILURE(rc))
488 return VMSetError(pVM, VERR_NEM_VM_CREATE_FAILED, RT_SRC_POS,
489 "Querying register %#x failed: %Rrc", s_aIdRegs[i].idKvmReg, rc);
490 }
491
492 rc = CPUMR3PopulateFeaturesByIdRegisters(pVM, &IdRegs);
493 if (RT_FAILURE(rc))
494 return rc;
495 }
496 }
497
498 /*
499 * Setup the SMCCC filter to get exits for PSCI related
500 * guest calls (to support SMP, power off and reset).
501 */
502 struct kvm_smccc_filter SmcccPsciFilter; RT_ZERO(SmcccPsciFilter);
503 SmcccPsciFilter.base = ARM_PSCI_FUNC_ID_CREATE_FAST_64(ARM_PSCI_FUNC_ID_PSCI_VERSION);
504 SmcccPsciFilter.nr_functions = ARM_PSCI_FUNC_ID_CREATE_FAST_64(ARM_PSCI_FUNC_ID_SYSTEM_RESET) - SmcccPsciFilter.base + 1;
505 SmcccPsciFilter.action = KVM_SMCCC_FILTER_FWD_TO_USER;
506 int rc = nemR3LnxSetAttribute(pVM, KVM_ARM_VM_SMCCC_CTRL, KVM_ARM_VM_SMCCC_FILTER, &SmcccPsciFilter,
507 "KVM_ARM_VM_SMCCC_FILTER", pErrInfo);
508 if (RT_FAILURE(rc))
509 return rc;
510
511 SmcccPsciFilter.base = ARM_PSCI_FUNC_ID_CREATE_FAST_32(ARM_PSCI_FUNC_ID_PSCI_VERSION);
512 SmcccPsciFilter.nr_functions = ARM_PSCI_FUNC_ID_CREATE_FAST_32(ARM_PSCI_FUNC_ID_SYSTEM_RESET) - SmcccPsciFilter.base + 1;
513 SmcccPsciFilter.action = KVM_SMCCC_FILTER_FWD_TO_USER;
514 rc = nemR3LnxSetAttribute(pVM, KVM_ARM_VM_SMCCC_CTRL, KVM_ARM_VM_SMCCC_FILTER, &SmcccPsciFilter,
515 "KVM_ARM_VM_SMCCC_FILTER", pErrInfo);
516 if (RT_FAILURE(rc))
517 return rc;
518
519 return VINF_SUCCESS;
520}
521
522
523int nemR3NativeInitCompleted(PVM pVM, VMINITCOMPLETED enmWhat)
524{
525 /*
526 * Make RTThreadPoke work again (disabled for avoiding unnecessary
527 * critical section issues in ring-0).
528 */
529 if (enmWhat == VMINITCOMPLETED_RING3)
530 VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ALL_AT_ONCE, nemR3LnxFixThreadPoke, NULL);
531
532 return VINF_SUCCESS;
533}
534
535
536/*********************************************************************************************************************************
537* CPU State *
538*********************************************************************************************************************************/
539
540/**
541 * Sets the given general purpose register to the given value.
542 *
543 * @param pVCpu The cross context virtual CPU structure of the
544 * calling EMT.
545 * @param uReg The register index.
546 * @param f64BitReg Flag whether to operate on a 64-bit or 32-bit register.
547 * @param fSignExtend Flag whether to sign extend the value.
548 * @param u64Val The value.
549 */
550DECLINLINE(void) nemR3LnxSetGReg(PVMCPU pVCpu, uint8_t uReg, bool f64BitReg, bool fSignExtend, uint64_t u64Val)
551{
552 AssertReturnVoid(uReg < 31);
553
554 if (f64BitReg)
555 pVCpu->cpum.GstCtx.aGRegs[uReg].x = fSignExtend ? (int64_t)u64Val : u64Val;
556 else
557 pVCpu->cpum.GstCtx.aGRegs[uReg].w = fSignExtend ? (int32_t)u64Val : u64Val; /** @todo Does this clear the upper half on real hardware? */
558
559 /* Mark the register as not extern anymore. */
560 switch (uReg)
561 {
562 case 0:
563 pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_X0;
564 break;
565 case 1:
566 pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_X1;
567 break;
568 case 2:
569 pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_X2;
570 break;
571 case 3:
572 pVCpu->cpum.GstCtx.fExtrn &= ~CPUMCTX_EXTRN_X3;
573 break;
574 default:
575 AssertRelease(!(pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_X4_X28));
576 /** @todo We need to import all missing registers in order to clear this flag (or just set it in HV from here). */
577 }
578}
579
580
581/**
582 * Gets the given general purpose register and returns the value.
583 *
584 * @returns Value from the given register.
585 * @param pVCpu The cross context virtual CPU structure of the
586 * calling EMT.
587 * @param uReg The register index.
588 */
589DECLINLINE(uint64_t) nemR3LnxGetGReg(PVMCPU pVCpu, uint8_t uReg)
590{
591 AssertReturn(uReg <= ARMV8_AARCH64_REG_ZR, 0);
592
593 if (uReg == ARMV8_AARCH64_REG_ZR)
594 return 0;
595
596 /** @todo Import the register if extern. */
597 //AssertRelease(!(pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_GPRS_MASK));
598
599 return pVCpu->cpum.GstCtx.aGRegs[uReg].x;
600}
601
602/**
603 * Worker that imports selected state from KVM.
604 */
605static int nemHCLnxImportState(PVMCPUCC pVCpu, uint64_t fWhat, PCPUMCTX pCtx)
606{
607 fWhat &= pVCpu->cpum.GstCtx.fExtrn;
608 if (!fWhat)
609 return VINF_SUCCESS;
610
611#if 0
612 hv_return_t hrc = hv_vcpu_get_sys_reg(pVCpu->nem.s.hVCpu, HV_SYS_REG_CNTV_CTL_EL0, &pVCpu->cpum.GstCtx.CntvCtlEl0);
613 if (hrc == HV_SUCCESS)
614 hrc = hv_vcpu_get_sys_reg(pVCpu->nem.s.hVCpu, HV_SYS_REG_CNTV_CVAL_EL0, &pVCpu->cpum.GstCtx.CntvCValEl0);
615#endif
616
617 int rc = VINF_SUCCESS;
618 if (fWhat & (CPUMCTX_EXTRN_GPRS_MASK | CPUMCTX_EXTRN_FP | CPUMCTX_EXTRN_LR | CPUMCTX_EXTRN_PC))
619 {
620 for (uint32_t i = 0; i < RT_ELEMENTS(s_aCpumRegs); i++)
621 {
622 if (s_aCpumRegs[i].fCpumExtrn & fWhat)
623 {
624 uint64_t *pu64 = (uint64_t *)((uint8_t *)&pVCpu->cpum.GstCtx + s_aCpumRegs[i].offCpumCtx);
625 rc |= nemR3LnxKvmQueryRegU64(pVCpu, s_aCpumRegs[i].idKvmReg, pu64);
626 }
627 }
628 }
629
630 if ( rc == VINF_SUCCESS
631 && (fWhat & CPUMCTX_EXTRN_FPCR))
632 {
633 uint32_t u32Tmp;
634 rc |= nemR3LnxKvmQueryRegU32(pVCpu, KVM_ARM64_REG_FP_FPCR, &u32Tmp);
635 if (rc == VINF_SUCCESS)
636 pVCpu->cpum.GstCtx.fpcr = u32Tmp;
637 }
638
639 if ( rc == VINF_SUCCESS
640 && (fWhat & CPUMCTX_EXTRN_FPSR))
641 {
642 uint32_t u32Tmp;
643 rc |= nemR3LnxKvmQueryRegU32(pVCpu, KVM_ARM64_REG_FP_FPSR, &u32Tmp);
644 if (rc == VINF_SUCCESS)
645 pVCpu->cpum.GstCtx.fpsr = u32Tmp;
646 }
647
648 if ( rc == VINF_SUCCESS
649 && (fWhat & CPUMCTX_EXTRN_V0_V31))
650 {
651 /* SIMD/FP registers. */
652 for (uint32_t i = 0; i < RT_ELEMENTS(s_aCpumFpRegs); i++)
653 {
654 void *pu128 = (void *)((uint8_t *)&pVCpu->cpum.GstCtx + s_aCpumFpRegs[i].offCpumCtx);
655 rc |= nemR3LnxKvmQueryRegPV(pVCpu, s_aCpumFpRegs[i].idKvmReg, pu128);
656 }
657 }
658
659 if ( rc == VINF_SUCCESS
660 && (fWhat & CPUMCTX_EXTRN_SYSREG_DEBUG))
661 {
662 /* Debug registers. */
663 for (uint32_t i = 0; i < RT_ELEMENTS(s_aCpumDbgRegs); i++)
664 {
665 uint64_t *pu64 = (uint64_t *)((uint8_t *)&pVCpu->cpum.GstCtx + s_aCpumDbgRegs[i].offCpumCtx);
666 rc |= nemR3LnxKvmQueryRegU64(pVCpu, s_aCpumDbgRegs[i].idKvmReg, pu64);
667 }
668 }
669
670 if ( rc == VINF_SUCCESS
671 && (fWhat & CPUMCTX_EXTRN_SYSREG_PAUTH_KEYS))
672 {
673 /* PAuth registers. */
674 for (uint32_t i = 0; i < RT_ELEMENTS(s_aCpumPAuthKeyRegs); i++)
675 {
676 uint64_t *pu64 = (uint64_t *)((uint8_t *)&pVCpu->cpum.GstCtx + s_aCpumPAuthKeyRegs[i].offCpumCtx);
677 rc |= nemR3LnxKvmQueryRegU64(pVCpu, s_aCpumPAuthKeyRegs[i].idKvmReg, pu64);
678 }
679 }
680
681 if ( rc == VINF_SUCCESS
682 && (fWhat & (CPUMCTX_EXTRN_SPSR | CPUMCTX_EXTRN_ELR | CPUMCTX_EXTRN_SP | CPUMCTX_EXTRN_SCTLR_TCR_TTBR | CPUMCTX_EXTRN_SYSREG_MISC)))
683 {
684 /* System registers. */
685 for (uint32_t i = 0; i < RT_ELEMENTS(s_aCpumSysRegs); i++)
686 {
687 if (s_aCpumSysRegs[i].fCpumExtrn & fWhat)
688 {
689 uint64_t *pu64 = (uint64_t *)((uint8_t *)&pVCpu->cpum.GstCtx + s_aCpumSysRegs[i].offCpumCtx);
690 rc |= nemR3LnxKvmQueryRegU64(pVCpu, s_aCpumSysRegs[i].idKvmReg, pu64);
691 }
692 }
693 }
694
695 if ( rc == VINF_SUCCESS
696 && (fWhat & CPUMCTX_EXTRN_PSTATE))
697 {
698 uint64_t u64Tmp;
699 rc |= nemR3LnxKvmQueryRegU64(pVCpu, KVM_ARM64_REG_PSTATE, &u64Tmp);
700 if (rc == VINF_SUCCESS)
701 pVCpu->cpum.GstCtx.fPState = (uint32_t)u64Tmp;
702
703 }
704
705 /*
706 * Update the external mask.
707 */
708 pCtx->fExtrn &= ~fWhat;
709 pVCpu->cpum.GstCtx.fExtrn &= ~fWhat;
710 if (!(pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_ALL))
711 pVCpu->cpum.GstCtx.fExtrn = 0;
712
713 return VINF_SUCCESS;
714}
715
716
717/**
718 * Interface for importing state on demand (used by IEM).
719 *
720 * @returns VBox status code.
721 * @param pVCpu The cross context CPU structure.
722 * @param fWhat What to import, CPUMCTX_EXTRN_XXX.
723 */
724VMM_INT_DECL(int) NEMImportStateOnDemand(PVMCPUCC pVCpu, uint64_t fWhat)
725{
726 STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatImportOnDemand);
727 return nemHCLnxImportState(pVCpu, fWhat, &pVCpu->cpum.GstCtx);
728}
729
730
731/**
732 * Exports state to KVM.
733 */
734static int nemHCLnxExportState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
735{
736 uint64_t const fExtrn = ~pCtx->fExtrn & CPUMCTX_EXTRN_ALL;
737 Assert((~fExtrn & CPUMCTX_EXTRN_ALL) != CPUMCTX_EXTRN_ALL); RT_NOREF(fExtrn);
738
739 RT_NOREF(pVM);
740 int rc = VINF_SUCCESS;
741 if ( (pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_GPRS_MASK | CPUMCTX_EXTRN_FP | CPUMCTX_EXTRN_LR | CPUMCTX_EXTRN_PC))
742 != (CPUMCTX_EXTRN_GPRS_MASK | CPUMCTX_EXTRN_FP | CPUMCTX_EXTRN_LR | CPUMCTX_EXTRN_PC))
743 {
744 for (uint32_t i = 0; i < RT_ELEMENTS(s_aCpumRegs); i++)
745 {
746 if (!(s_aCpumRegs[i].fCpumExtrn & pVCpu->cpum.GstCtx.fExtrn))
747 {
748 const uint64_t *pu64 = (const uint64_t *)((uint8_t *)&pVCpu->cpum.GstCtx + s_aCpumRegs[i].offCpumCtx);
749 rc |= nemR3LnxKvmSetRegU64(pVCpu, s_aCpumRegs[i].idKvmReg, pu64);
750 }
751 }
752 }
753
754 if ( rc == VINF_SUCCESS
755 && !(pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_FPCR))
756 {
757 uint32_t u32Tmp = pVCpu->cpum.GstCtx.fpcr;
758 rc |= nemR3LnxKvmSetRegU32(pVCpu, KVM_ARM64_REG_FP_FPCR, &u32Tmp);
759 }
760
761 if ( rc == VINF_SUCCESS
762 && !(pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_FPSR))
763 {
764 uint32_t u32Tmp = pVCpu->cpum.GstCtx.fpsr;
765 rc |= nemR3LnxKvmSetRegU32(pVCpu, KVM_ARM64_REG_FP_FPSR, &u32Tmp);
766 }
767
768 if ( rc == VINF_SUCCESS
769 && !(pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_V0_V31))
770 {
771 /* SIMD/FP registers. */
772 for (uint32_t i = 0; i < RT_ELEMENTS(s_aCpumFpRegs); i++)
773 {
774 void *pu128 = (void *)((uint8_t *)&pVCpu->cpum.GstCtx + s_aCpumFpRegs[i].offCpumCtx);
775 rc |= nemR3LnxKvmSetRegPV(pVCpu, s_aCpumFpRegs[i].idKvmReg, pu128);
776 }
777 }
778
779 if ( rc == VINF_SUCCESS
780 && !(pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_SYSREG_DEBUG))
781 {
782 /* Debug registers. */
783 for (uint32_t i = 0; i < RT_ELEMENTS(s_aCpumDbgRegs); i++)
784 {
785 uint64_t *pu64 = (uint64_t *)((uint8_t *)&pVCpu->cpum.GstCtx + s_aCpumDbgRegs[i].offCpumCtx);
786 rc |= nemR3LnxKvmSetRegU64(pVCpu, s_aCpumDbgRegs[i].idKvmReg, pu64);
787 }
788 }
789
790 if ( rc == VINF_SUCCESS
791 && !(pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_SYSREG_PAUTH_KEYS))
792 {
793 /* PAuth registers. */
794 for (uint32_t i = 0; i < RT_ELEMENTS(s_aCpumPAuthKeyRegs); i++)
795 {
796 uint64_t *pu64 = (uint64_t *)((uint8_t *)&pVCpu->cpum.GstCtx + s_aCpumPAuthKeyRegs[i].offCpumCtx);
797 rc |= nemR3LnxKvmSetRegU64(pVCpu, s_aCpumPAuthKeyRegs[i].idKvmReg, pu64);
798 }
799 }
800
801 if ( rc == VINF_SUCCESS
802 && (pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_SPSR | CPUMCTX_EXTRN_ELR | CPUMCTX_EXTRN_SP | CPUMCTX_EXTRN_SCTLR_TCR_TTBR | CPUMCTX_EXTRN_SYSREG_MISC))
803 != (CPUMCTX_EXTRN_SPSR | CPUMCTX_EXTRN_ELR | CPUMCTX_EXTRN_SP | CPUMCTX_EXTRN_SCTLR_TCR_TTBR | CPUMCTX_EXTRN_SYSREG_MISC))
804 {
805 /* System registers. */
806 for (uint32_t i = 0; i < RT_ELEMENTS(s_aCpumSysRegs); i++)
807 {
808 if (!(s_aCpumSysRegs[i].fCpumExtrn & pVCpu->cpum.GstCtx.fExtrn))
809 {
810 uint64_t *pu64 = (uint64_t *)((uint8_t *)&pVCpu->cpum.GstCtx + s_aCpumSysRegs[i].offCpumCtx);
811 rc |= nemR3LnxKvmSetRegU64(pVCpu, s_aCpumSysRegs[i].idKvmReg, pu64);
812 }
813 }
814 }
815
816 if ( rc == VINF_SUCCESS
817 && !(pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_PSTATE))
818 {
819 uint64_t u64Tmp = pVCpu->cpum.GstCtx.fPState;
820 rc = nemR3LnxKvmSetRegU64(pVCpu, KVM_ARM64_REG_PSTATE, &u64Tmp);
821 }
822
823 /*
824 * KVM now owns all the state.
825 */
826 pCtx->fExtrn = CPUMCTX_EXTRN_KEEPER_NEM | CPUMCTX_EXTRN_ALL;
827 return VINF_SUCCESS;
828}
829
830
831/**
832 * Query the CPU tick counter and optionally the TSC_AUX MSR value.
833 *
834 * @returns VBox status code.
835 * @param pVCpu The cross context CPU structure.
836 * @param pcTicks Where to return the CPU tick count.
837 * @param puAux Where to return the TSC_AUX register value.
838 */
839VMM_INT_DECL(int) NEMHCQueryCpuTick(PVMCPUCC pVCpu, uint64_t *pcTicks, uint32_t *puAux)
840{
841 STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatQueryCpuTick);
842 // KVM_GET_CLOCK?
843 RT_NOREF(pVCpu, pcTicks, puAux);
844 return VINF_SUCCESS;
845}
846
847
848/**
849 * Resumes CPU clock (TSC) on all virtual CPUs.
850 *
851 * This is called by TM when the VM is started, restored, resumed or similar.
852 *
853 * @returns VBox status code.
854 * @param pVM The cross context VM structure.
855 * @param pVCpu The cross context CPU structure of the calling EMT.
856 * @param uPausedTscValue The TSC value at the time of pausing.
857 */
858VMM_INT_DECL(int) NEMHCResumeCpuTickOnAll(PVMCC pVM, PVMCPUCC pVCpu, uint64_t uPausedTscValue)
859{
860 // KVM_SET_CLOCK?
861 RT_NOREF(pVM, pVCpu, uPausedTscValue);
862 return VINF_SUCCESS;
863}
864
865
866VMM_INT_DECL(uint32_t) NEMHCGetFeatures(PVMCC pVM)
867{
868 RT_NOREF(pVM);
869 return NEM_FEAT_F_NESTED_PAGING
870 | NEM_FEAT_F_FULL_GST_EXEC;
871}
872
873
874
875/*********************************************************************************************************************************
876* Execution *
877*********************************************************************************************************************************/
878
879
880VMMR3_INT_DECL(bool) NEMR3CanExecuteGuest(PVM pVM, PVMCPU pVCpu)
881{
882 RT_NOREF(pVM, pVCpu);
883 Assert(VM_IS_NEM_ENABLED(pVM));
884 return true;
885}
886
887
888bool nemR3NativeSetSingleInstruction(PVM pVM, PVMCPU pVCpu, bool fEnable)
889{
890 NOREF(pVM); NOREF(pVCpu); NOREF(fEnable);
891 return false;
892}
893
894
895void nemR3NativeNotifyFF(PVM pVM, PVMCPU pVCpu, uint32_t fFlags)
896{
897 int rc = RTThreadPoke(pVCpu->hThread);
898 LogFlow(("nemR3NativeNotifyFF: #%u -> %Rrc\n", pVCpu->idCpu, rc));
899 AssertRC(rc);
900 RT_NOREF(pVM, fFlags);
901}
902
903
904DECLHIDDEN(bool) nemR3NativeNotifyDebugEventChanged(PVM pVM, bool fUseDebugLoop)
905{
906 RT_NOREF(pVM, fUseDebugLoop);
907 return false;
908}
909
910
911DECLHIDDEN(bool) nemR3NativeNotifyDebugEventChangedPerCpu(PVM pVM, PVMCPU pVCpu, bool fUseDebugLoop)
912{
913 RT_NOREF(pVM, pVCpu, fUseDebugLoop);
914 return false;
915}
916
917
918DECL_FORCE_INLINE(int) nemR3LnxKvmUpdateIntrState(PVM pVM, PVMCPU pVCpu, bool fIrq, bool fAsserted)
919{
920 struct kvm_irq_level IrqLvl;
921
922 LogFlowFunc(("pVM=%p pVCpu=%p fIrq=%RTbool fAsserted=%RTbool\n",
923 pVM, pVCpu, fIrq, fAsserted));
924
925 IrqLvl.irq = ((uint32_t)KVM_ARM_IRQ_TYPE_CPU << 24) /* Directly drives CPU interrupt lines. */
926 | (pVCpu->idCpu & 0xff) << 16
927 | (fIrq ? 0 : 1);
928 IrqLvl.level = fAsserted ? 1 : 0;
929 int rcLnx = ioctl(pVM->nem.s.fdVm, KVM_IRQ_LINE, &IrqLvl);
930 AssertReturn(rcLnx == 0, VERR_NEM_IPE_9);
931
932 return VINF_SUCCESS;
933}
934
935
936/**
937 * Deals with pending interrupt FFs prior to executing guest code.
938 */
939static VBOXSTRICTRC nemHCLnxHandleInterruptFF(PVM pVM, PVMCPU pVCpu)
940{
941 LogFlowFunc(("pVCpu=%p{.idCpu=%u} fIrq=%RTbool fFiq=%RTbool\n",
942 pVCpu, pVCpu->idCpu,
943 VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_IRQ),
944 VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_FIQ)));
945
946 bool fIrq = VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_IRQ);
947 bool fFiq = VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_FIQ);
948
949 /* Update the pending interrupt state. */
950 if (fIrq != pVCpu->nem.s.fIrqLastSeen)
951 {
952 int rc = nemR3LnxKvmUpdateIntrState(pVM, pVCpu, true /*fIrq*/, fIrq);
953 AssertRCReturn(rc, VERR_NEM_IPE_9);
954 pVCpu->nem.s.fIrqLastSeen = fIrq;
955 }
956
957 if (fFiq != pVCpu->nem.s.fIrqLastSeen)
958 {
959 int rc = nemR3LnxKvmUpdateIntrState(pVM, pVCpu, false /*fIrq*/, fFiq);
960 AssertRCReturn(rc, VERR_NEM_IPE_9);
961 pVCpu->nem.s.fFiqLastSeen = fFiq;
962 }
963
964 return VINF_SUCCESS;
965}
966
967
968#if 0
969/**
970 * Handles KVM_EXIT_INTERNAL_ERROR.
971 */
972static VBOXSTRICTRC nemR3LnxHandleInternalError(PVMCPU pVCpu, struct kvm_run *pRun)
973{
974 Log(("NEM: KVM_EXIT_INTERNAL_ERROR! suberror=%#x (%d) ndata=%u data=%.*Rhxs\n", pRun->internal.suberror,
975 pRun->internal.suberror, pRun->internal.ndata, sizeof(pRun->internal.data), &pRun->internal.data[0]));
976
977 /*
978 * Deal with each suberror, returning if we don't want IEM to handle it.
979 */
980 switch (pRun->internal.suberror)
981 {
982 case KVM_INTERNAL_ERROR_EMULATION:
983 {
984 EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_INTERNAL_ERROR_EMULATION),
985 pRun->s.regs.regs.rip + pRun->s.regs.sregs.cs.base, ASMReadTSC());
986 STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitInternalErrorEmulation);
987 break;
988 }
989
990 case KVM_INTERNAL_ERROR_SIMUL_EX:
991 case KVM_INTERNAL_ERROR_DELIVERY_EV:
992 case KVM_INTERNAL_ERROR_UNEXPECTED_EXIT_REASON:
993 default:
994 {
995 EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_INTERNAL_ERROR_FATAL),
996 pRun->s.regs.regs.rip + pRun->s.regs.sregs.cs.base, ASMReadTSC());
997 STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitInternalErrorFatal);
998 const char *pszName;
999 switch (pRun->internal.suberror)
1000 {
1001 case KVM_INTERNAL_ERROR_EMULATION: pszName = "KVM_INTERNAL_ERROR_EMULATION"; break;
1002 case KVM_INTERNAL_ERROR_SIMUL_EX: pszName = "KVM_INTERNAL_ERROR_SIMUL_EX"; break;
1003 case KVM_INTERNAL_ERROR_DELIVERY_EV: pszName = "KVM_INTERNAL_ERROR_DELIVERY_EV"; break;
1004 case KVM_INTERNAL_ERROR_UNEXPECTED_EXIT_REASON: pszName = "KVM_INTERNAL_ERROR_UNEXPECTED_EXIT_REASON"; break;
1005 default: pszName = "unknown"; break;
1006 }
1007 LogRel(("NEM: KVM_EXIT_INTERNAL_ERROR! suberror=%#x (%s) ndata=%u data=%.*Rhxs\n", pRun->internal.suberror, pszName,
1008 pRun->internal.ndata, sizeof(pRun->internal.data), &pRun->internal.data[0]));
1009 return VERR_NEM_IPE_0;
1010 }
1011 }
1012
1013 /*
1014 * Execute instruction in IEM and try get on with it.
1015 */
1016 Log2(("nemR3LnxHandleInternalError: Executing instruction at %04x:%08RX64 in IEM\n",
1017 pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip));
1018 VBOXSTRICTRC rcStrict = nemHCLnxImportState(pVCpu,
1019 IEM_CPUMCTX_EXTRN_MUST_MASK | CPUMCTX_EXTRN_INHIBIT_INT
1020 | CPUMCTX_EXTRN_INHIBIT_NMI,
1021 &pVCpu->cpum.GstCtx, pRun);
1022 if (RT_SUCCESS(rcStrict))
1023 rcStrict = IEMExecOne(pVCpu);
1024 return rcStrict;
1025}
1026#endif
1027
1028
1029/**
1030 * Handles KVM_EXIT_MMIO.
1031 */
1032static VBOXSTRICTRC nemHCLnxHandleExitMmio(PVMCC pVM, PVMCPUCC pVCpu, struct kvm_run *pRun)
1033{
1034 /*
1035 * Input validation.
1036 */
1037 Assert(pRun->mmio.len <= sizeof(pRun->mmio.data));
1038 Assert(pRun->mmio.is_write <= 1);
1039
1040#if 0
1041 /*
1042 * We cannot easily act on the exit history here, because the MMIO port
1043 * exit is stateful and the instruction will be completed in the next
1044 * KVM_RUN call. There seems no way to circumvent this.
1045 */
1046 EMHistoryAddExit(pVCpu,
1047 pRun->mmio.is_write
1048 ? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_MMIO_WRITE)
1049 : EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_MMIO_READ),
1050 pRun->s.regs.regs.pc, ASMReadTSC());
1051#else
1052 RT_NOREF(pVCpu);
1053#endif
1054
1055 /*
1056 * Do the requested job.
1057 */
1058 VBOXSTRICTRC rcStrict;
1059 if (pRun->mmio.is_write)
1060 {
1061 rcStrict = PGMPhysWrite(pVM, pRun->mmio.phys_addr, pRun->mmio.data, pRun->mmio.len, PGMACCESSORIGIN_HM);
1062 Log4(("MmioExit/%u:WRITE %#x LB %u, %.*Rhxs -> rcStrict=%Rrc\n",
1063 pVCpu->idCpu,
1064 pRun->mmio.phys_addr, pRun->mmio.len, pRun->mmio.len, pRun->mmio.data, VBOXSTRICTRC_VAL(rcStrict) ));
1065 }
1066 else
1067 {
1068 rcStrict = PGMPhysRead(pVM, pRun->mmio.phys_addr, pRun->mmio.data, pRun->mmio.len, PGMACCESSORIGIN_HM);
1069 Log4(("MmioExit/%u: READ %#x LB %u -> %.*Rhxs rcStrict=%Rrc\n",
1070 pVCpu->idCpu,
1071 pRun->mmio.phys_addr, pRun->mmio.len, pRun->mmio.len, pRun->mmio.data, VBOXSTRICTRC_VAL(rcStrict) ));
1072 }
1073 return rcStrict;
1074}
1075
1076
1077/**
1078 * Handles KVM_EXIT_HYPERCALL.
1079 */
1080static VBOXSTRICTRC nemHCLnxHandleExitHypercall(PVMCC pVM, PVMCPUCC pVCpu, struct kvm_run *pRun)
1081{
1082#if 0
1083 /*
1084 * We cannot easily act on the exit history here, because the MMIO port
1085 * exit is stateful and the instruction will be completed in the next
1086 * KVM_RUN call. There seems no way to circumvent this.
1087 */
1088 EMHistoryAddExit(pVCpu,
1089 pRun->mmio.is_write
1090 ? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_MMIO_WRITE)
1091 : EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_MMIO_READ),
1092 pRun->s.regs.regs.pc, ASMReadTSC());
1093#else
1094 RT_NOREF(pVCpu);
1095#endif
1096
1097 /*
1098 * Do the requested job.
1099 */
1100 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
1101
1102 /** @todo Raise exception to EL1 if PSCI not configured. */
1103 /** @todo Need a generic mechanism here to pass this to, GIM maybe?. */
1104 uint32_t uFunId = pRun->hypercall.nr;
1105 bool fHvc64 = RT_BOOL(uFunId & ARM_SMCCC_FUNC_ID_64BIT); RT_NOREF(fHvc64);
1106 uint32_t uEntity = ARM_SMCCC_FUNC_ID_ENTITY_GET(uFunId);
1107 uint32_t uFunNum = ARM_SMCCC_FUNC_ID_NUM_GET(uFunId);
1108 if (uEntity == ARM_SMCCC_FUNC_ID_ENTITY_STD_SEC_SERVICE)
1109 {
1110 rcStrict = nemHCLnxImportState(pVCpu, CPUMCTX_EXTRN_X0 | CPUMCTX_EXTRN_X1 | CPUMCTX_EXTRN_X2 | CPUMCTX_EXTRN_X3,
1111 &pVCpu->cpum.GstCtx);
1112 if (rcStrict != VINF_SUCCESS)
1113 return rcStrict;
1114
1115 switch (uFunNum)
1116 {
1117 case ARM_PSCI_FUNC_ID_PSCI_VERSION:
1118 nemR3LnxSetGReg(pVCpu, ARMV8_AARCH64_REG_X0, false /*f64BitReg*/, false /*fSignExtend*/, ARM_PSCI_FUNC_ID_PSCI_VERSION_SET(1, 2));
1119 break;
1120 case ARM_PSCI_FUNC_ID_SYSTEM_OFF:
1121 rcStrict = VMR3PowerOff(pVM->pUVM);
1122 break;
1123 case ARM_PSCI_FUNC_ID_SYSTEM_RESET:
1124 case ARM_PSCI_FUNC_ID_SYSTEM_RESET2:
1125 {
1126 bool fHaltOnReset;
1127 int rc = CFGMR3QueryBool(CFGMR3GetChild(CFGMR3GetRoot(pVM), "PDM"), "HaltOnReset", &fHaltOnReset);
1128 if (RT_SUCCESS(rc) && fHaltOnReset)
1129 {
1130 Log(("nemHCLnxHandleExitHypercall: Halt On Reset!\n"));
1131 rcStrict = VINF_EM_HALT;
1132 }
1133 else
1134 {
1135 /** @todo pVM->pdm.s.fResetFlags = fFlags; */
1136 VM_FF_SET(pVM, VM_FF_RESET);
1137 rcStrict = VINF_EM_RESET;
1138 }
1139 break;
1140 }
1141 case ARM_PSCI_FUNC_ID_CPU_ON:
1142 {
1143 uint64_t u64TgtCpu = nemR3LnxGetGReg(pVCpu, ARMV8_AARCH64_REG_X1);
1144 RTGCPHYS GCPhysExecAddr = nemR3LnxGetGReg(pVCpu, ARMV8_AARCH64_REG_X2);
1145 uint64_t u64CtxId = nemR3LnxGetGReg(pVCpu, ARMV8_AARCH64_REG_X3);
1146 VMMR3CpuOn(pVM, u64TgtCpu & 0xff, GCPhysExecAddr, u64CtxId);
1147 nemR3LnxSetGReg(pVCpu, ARMV8_AARCH64_REG_X0, true /*f64BitReg*/, false /*fSignExtend*/, ARM_PSCI_STS_SUCCESS);
1148 break;
1149 }
1150 case ARM_PSCI_FUNC_ID_PSCI_FEATURES:
1151 {
1152 uint32_t u32FunNum = (uint32_t)nemR3LnxGetGReg(pVCpu, ARMV8_AARCH64_REG_X1);
1153 switch (u32FunNum)
1154 {
1155 case ARM_PSCI_FUNC_ID_PSCI_VERSION:
1156 case ARM_PSCI_FUNC_ID_SYSTEM_OFF:
1157 case ARM_PSCI_FUNC_ID_SYSTEM_RESET:
1158 case ARM_PSCI_FUNC_ID_SYSTEM_RESET2:
1159 case ARM_PSCI_FUNC_ID_CPU_ON:
1160 nemR3LnxSetGReg(pVCpu, ARMV8_AARCH64_REG_X0,
1161 false /*f64BitReg*/, false /*fSignExtend*/,
1162 (uint64_t)ARM_PSCI_STS_SUCCESS);
1163 break;
1164 default:
1165 nemR3LnxSetGReg(pVCpu, ARMV8_AARCH64_REG_X0,
1166 false /*f64BitReg*/, false /*fSignExtend*/,
1167 (uint64_t)ARM_PSCI_STS_NOT_SUPPORTED);
1168 }
1169 break;
1170 }
1171 default:
1172 nemR3LnxSetGReg(pVCpu, ARMV8_AARCH64_REG_X0, false /*f64BitReg*/, false /*fSignExtend*/, (uint64_t)ARM_PSCI_STS_NOT_SUPPORTED);
1173 }
1174 }
1175 else
1176 nemR3LnxSetGReg(pVCpu, ARMV8_AARCH64_REG_X0, false /*f64BitReg*/, false /*fSignExtend*/, (uint64_t)ARM_PSCI_STS_NOT_SUPPORTED);
1177
1178
1179 return rcStrict;
1180}
1181
1182
1183static VBOXSTRICTRC nemHCLnxHandleExit(PVMCC pVM, PVMCPUCC pVCpu, struct kvm_run *pRun, bool *pfStatefulExit)
1184{
1185 STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitTotal);
1186
1187 if (pVCpu->nem.s.fIrqDeviceLvls != pRun->s.regs.device_irq_level)
1188 {
1189 uint64_t fChanged = pVCpu->nem.s.fIrqDeviceLvls ^ pRun->s.regs.device_irq_level;
1190
1191 if (fChanged & KVM_ARM_DEV_EL1_VTIMER)
1192 {
1193 TMCpuSetVTimerNextActivation(pVCpu, UINT64_MAX);
1194 GICPpiSet(pVCpu, pVM->nem.s.u32GicPpiVTimer, RT_BOOL(pRun->s.regs.device_irq_level & KVM_ARM_DEV_EL1_VTIMER));
1195 }
1196
1197 if (fChanged & KVM_ARM_DEV_EL1_PTIMER)
1198 {
1199 //TMCpuSetVTimerNextActivation(pVCpu, UINT64_MAX);
1200 GICPpiSet(pVCpu, pVM->nem.s.u32GicPpiVTimer, RT_BOOL(pRun->s.regs.device_irq_level & KVM_ARM_DEV_EL1_PTIMER));
1201 }
1202
1203 pVCpu->nem.s.fIrqDeviceLvls = pRun->s.regs.device_irq_level;
1204 }
1205
1206 switch (pRun->exit_reason)
1207 {
1208 case KVM_EXIT_EXCEPTION:
1209 AssertFailed();
1210 break;
1211
1212 case KVM_EXIT_MMIO:
1213 STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitMmio);
1214 *pfStatefulExit = true;
1215 return nemHCLnxHandleExitMmio(pVM, pVCpu, pRun);
1216
1217 case KVM_EXIT_INTR: /* EINTR */
1218 //EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_INTERRUPTED),
1219 // pRun->s.regs.regs.pc, ASMReadTSC());
1220 STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitIntr);
1221 Log5(("Intr/%u\n", pVCpu->idCpu));
1222 return VINF_SUCCESS;
1223
1224 case KVM_EXIT_HYPERCALL:
1225 STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitHypercall);
1226 return nemHCLnxHandleExitHypercall(pVM, pVCpu, pRun);
1227
1228#if 0
1229 case KVM_EXIT_DEBUG:
1230 STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitDebug);
1231 AssertFailed();
1232 break;
1233
1234 case KVM_EXIT_SYSTEM_EVENT:
1235 AssertFailed();
1236 break;
1237
1238 case KVM_EXIT_DIRTY_RING_FULL:
1239 AssertFailed();
1240 break;
1241 case KVM_EXIT_AP_RESET_HOLD:
1242 AssertFailed();
1243 break;
1244
1245
1246 case KVM_EXIT_SHUTDOWN:
1247 AssertFailed();
1248 break;
1249
1250 case KVM_EXIT_FAIL_ENTRY:
1251 LogRel(("NEM: KVM_EXIT_FAIL_ENTRY! hardware_entry_failure_reason=%#x cpu=%#x\n",
1252 pRun->fail_entry.hardware_entry_failure_reason, pRun->fail_entry.cpu));
1253 EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_FAILED_ENTRY),
1254 pRun->s.regs.regs.pc, ASMReadTSC());
1255 return VERR_NEM_IPE_1;
1256
1257 case KVM_EXIT_INTERNAL_ERROR:
1258 /* we're counting sub-reasons inside the function. */
1259 return nemR3LnxHandleInternalError(pVCpu, pRun);
1260#endif
1261
1262 /*
1263 * Foreign and unknowns.
1264 */
1265#if 0
1266 case KVM_EXIT_IO:
1267 AssertLogRelMsgFailedReturn(("KVM_EXIT_IO on VCpu #%u at %RX64!\n", pVCpu->idCpu, pRun->s.regs.pc), VERR_NEM_IPE_1);
1268 case KVM_EXIT_NMI:
1269 AssertLogRelMsgFailedReturn(("KVM_EXIT_NMI on VCpu #%u at %RX64!\n", pVCpu->idCpu, pRun->s.regs.pc), VERR_NEM_IPE_1);
1270 case KVM_EXIT_EPR:
1271 AssertLogRelMsgFailedReturn(("KVM_EXIT_EPR on VCpu #%u at %RX64!\n", pVCpu->idCpu, pRun->s.regs.pc), VERR_NEM_IPE_1);
1272 case KVM_EXIT_WATCHDOG:
1273 AssertLogRelMsgFailedReturn(("KVM_EXIT_WATCHDOG on VCpu #%u at %RX64!\n", pVCpu->idCpu, pRun->s.regs.pc), VERR_NEM_IPE_1);
1274 case KVM_EXIT_ARM_NISV:
1275 AssertLogRelMsgFailedReturn(("KVM_EXIT_ARM_NISV on VCpu #%u at %RX64!\n", pVCpu->idCpu, pRun->s.regs.pc), VERR_NEM_IPE_1);
1276 case KVM_EXIT_S390_STSI:
1277 AssertLogRelMsgFailedReturn(("KVM_EXIT_S390_STSI on VCpu #%u at %RX64!\n", pVCpu->idCpu, pRun->s.regs.pc), VERR_NEM_IPE_1);
1278 case KVM_EXIT_S390_TSCH:
1279 AssertLogRelMsgFailedReturn(("KVM_EXIT_S390_TSCH on VCpu #%u at %RX64!\n", pVCpu->idCpu, pRun->s.regs.pc), VERR_NEM_IPE_1);
1280 case KVM_EXIT_OSI:
1281 AssertLogRelMsgFailedReturn(("KVM_EXIT_OSI on VCpu #%u at %RX64!\n", pVCpu->idCpu, pRun->s.regs.pc), VERR_NEM_IPE_1);
1282 case KVM_EXIT_PAPR_HCALL:
1283 AssertLogRelMsgFailedReturn(("KVM_EXIT_PAPR_HCALL on VCpu #%u at %RX64!\n", pVCpu->idCpu, pRun->s.regs.pc), VERR_NEM_IPE_1);
1284 case KVM_EXIT_S390_UCONTROL:
1285 AssertLogRelMsgFailedReturn(("KVM_EXIT_S390_UCONTROL on VCpu #%u at %RX64!\n", pVCpu->idCpu, pRun->s.regs.pc), VERR_NEM_IPE_1);
1286 case KVM_EXIT_DCR:
1287 AssertLogRelMsgFailedReturn(("KVM_EXIT_DCR on VCpu #%u at %RX64!\n", pVCpu->idCpu, pRun->s.regs.pc), VERR_NEM_IPE_1);
1288 case KVM_EXIT_S390_SIEIC:
1289 AssertLogRelMsgFailedReturn(("KVM_EXIT_S390_SIEIC on VCpu #%u at %RX64!\n", pVCpu->idCpu, pRun->s.regs.pc), VERR_NEM_IPE_1);
1290 case KVM_EXIT_S390_RESET:
1291 AssertLogRelMsgFailedReturn(("KVM_EXIT_S390_RESET on VCpu #%u at %RX64!\n", pVCpu->idCpu, pRun->s.regs.pc), VERR_NEM_IPE_1);
1292 case KVM_EXIT_UNKNOWN:
1293 AssertLogRelMsgFailedReturn(("KVM_EXIT_UNKNOWN on VCpu #%u at %RX64!\n", pVCpu->idCpu, pRun->s.regs.pc), VERR_NEM_IPE_1);
1294 case KVM_EXIT_XEN:
1295 AssertLogRelMsgFailedReturn(("KVM_EXIT_XEN on VCpu #%u at %RX64!\n", pVCpu->idCpu, pRun->s.regs.pc), VERR_NEM_IPE_1);
1296#endif
1297 default:
1298 AssertLogRelMsgFailedReturn(("Unknown exit reason %u on VCpu #%u!\n", pRun->exit_reason, pVCpu->idCpu), VERR_NEM_IPE_1);
1299 }
1300 RT_NOREF(pVM, pVCpu);
1301 return VERR_NOT_IMPLEMENTED;
1302}
1303
1304
1305VBOXSTRICTRC nemR3NativeRunGC(PVM pVM, PVMCPU pVCpu)
1306{
1307 /*
1308 * Try switch to NEM runloop state.
1309 */
1310 if (VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC_NEM, VMCPUSTATE_STARTED))
1311 { /* likely */ }
1312 else
1313 {
1314 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC_NEM, VMCPUSTATE_STARTED_EXEC_NEM_CANCELED);
1315 LogFlow(("NEM/%u: returning immediately because canceled\n", pVCpu->idCpu));
1316 return VINF_SUCCESS;
1317 }
1318
1319 /*
1320 * The run loop.
1321 */
1322 struct kvm_run * const pRun = pVCpu->nem.s.pRun;
1323 const bool fSingleStepping = DBGFIsStepping(pVCpu);
1324 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
1325 bool fStatefulExit = false; /* For MMIO and IO exits. */
1326 for (unsigned iLoop = 0;; iLoop++)
1327 {
1328 /*
1329 * Sync the interrupt state.
1330 */
1331 rcStrict = nemHCLnxHandleInterruptFF(pVM, pVCpu);
1332 if (rcStrict == VINF_SUCCESS)
1333 { /* likely */ }
1334 else
1335 {
1336 LogFlow(("NEM/%u: breaking: nemHCLnxHandleInterruptFF -> %Rrc\n", pVCpu->idCpu, VBOXSTRICTRC_VAL(rcStrict) ));
1337 STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatBreakOnStatus);
1338 break;
1339 }
1340
1341 /*
1342 * Ensure KVM has the whole state.
1343 */
1344 if ((pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_ALL) != CPUMCTX_EXTRN_ALL)
1345 {
1346 int rc2 = nemHCLnxExportState(pVM, pVCpu, &pVCpu->cpum.GstCtx);
1347 AssertRCReturn(rc2, rc2);
1348 }
1349
1350 /*
1351 * Poll timers and run for a bit.
1352 *
1353 * With the VID approach (ring-0 or ring-3) we can specify a timeout here,
1354 * so we take the time of the next timer event and uses that as a deadline.
1355 * The rounding heuristics are "tuned" so that rhel5 (1K timer) will boot fine.
1356 */
1357 /** @todo See if we cannot optimize this TMTimerPollGIP by only redoing
1358 * the whole polling job when timers have changed... */
1359 uint64_t offDeltaIgnored;
1360 uint64_t const nsNextTimerEvt = TMTimerPollGIP(pVM, pVCpu, &offDeltaIgnored); NOREF(nsNextTimerEvt);
1361 if ( !VM_FF_IS_ANY_SET(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
1362 && !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
1363 {
1364 if (VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC_NEM_WAIT, VMCPUSTATE_STARTED_EXEC_NEM))
1365 {
1366 //LogFlow(("NEM/%u: Entry @ %04x:%08RX64 IF=%d EFL=%#RX64 SS:RSP=%04x:%08RX64 cr0=%RX64\n",
1367 // pVCpu->idCpu, pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip,
1368 // !!(pRun->s.regs.regs.rflags & X86_EFL_IF), pRun->s.regs.regs.rflags,
1369 // pRun->s.regs.sregs.ss.selector, pRun->s.regs.regs.rsp, pRun->s.regs.sregs.cr0));
1370 TMNotifyStartOfExecution(pVM, pVCpu);
1371
1372 int rcLnx = ioctl(pVCpu->nem.s.fdVCpu, KVM_RUN, 0UL);
1373
1374 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC_NEM, VMCPUSTATE_STARTED_EXEC_NEM_WAIT);
1375 TMNotifyEndOfExecution(pVM, pVCpu, ASMReadTSC());
1376
1377#if 0 //def LOG_ENABLED
1378 if (LogIsFlowEnabled())
1379 {
1380 struct kvm_mp_state MpState = {UINT32_MAX};
1381 ioctl(pVCpu->nem.s.fdVCpu, KVM_GET_MP_STATE, &MpState);
1382 LogFlow(("NEM/%u: Exit @ %04x:%08RX64 IF=%d EFL=%#RX64 CR8=%#x Reason=%#x IrqReady=%d Flags=%#x %#lx\n", pVCpu->idCpu,
1383 pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip, pRun->if_flag,
1384 pRun->s.regs.regs.rflags, pRun->s.regs.sregs.cr8, pRun->exit_reason,
1385 pRun->ready_for_interrupt_injection, pRun->flags, MpState.mp_state));
1386 }
1387#endif
1388 fStatefulExit = false;
1389 if (RT_LIKELY(rcLnx == 0 || errno == EINTR))
1390 {
1391 /*
1392 * Deal with the exit.
1393 */
1394 rcStrict = nemHCLnxHandleExit(pVM, pVCpu, pRun, &fStatefulExit);
1395 if (rcStrict == VINF_SUCCESS)
1396 { /* hopefully likely */ }
1397 else
1398 {
1399 LogFlow(("NEM/%u: breaking: nemHCLnxHandleExit -> %Rrc\n", pVCpu->idCpu, VBOXSTRICTRC_VAL(rcStrict) ));
1400 STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatBreakOnStatus);
1401 break;
1402 }
1403 }
1404 else
1405 {
1406 int rc2 = RTErrConvertFromErrno(errno);
1407 AssertLogRelMsgFailedReturn(("KVM_RUN failed: rcLnx=%d errno=%u rc=%Rrc\n", rcLnx, errno, rc2), rc2);
1408 }
1409
1410 /*
1411 * If no relevant FFs are pending, loop.
1412 */
1413 if ( !VM_FF_IS_ANY_SET( pVM, !fSingleStepping ? VM_FF_HP_R0_PRE_HM_MASK : VM_FF_HP_R0_PRE_HM_STEP_MASK)
1414 && !VMCPU_FF_IS_ANY_SET(pVCpu, !fSingleStepping ? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
1415 { /* likely */ }
1416 else
1417 {
1418
1419 /** @todo Try handle pending flags, not just return to EM loops. Take care
1420 * not to set important RCs here unless we've handled an exit. */
1421 LogFlow(("NEM/%u: breaking: pending FF (%#x / %#RX64)\n",
1422 pVCpu->idCpu, pVM->fGlobalForcedActions, (uint64_t)pVCpu->fLocalForcedActions));
1423 STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatBreakOnFFPost);
1424 break;
1425 }
1426 }
1427 else
1428 {
1429 LogFlow(("NEM/%u: breaking: canceled %d (pre exec)\n", pVCpu->idCpu, VMCPU_GET_STATE(pVCpu) ));
1430 STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatBreakOnCancel);
1431 break;
1432 }
1433 }
1434 else
1435 {
1436 LogFlow(("NEM/%u: breaking: pending FF (pre exec)\n", pVCpu->idCpu));
1437 STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatBreakOnFFPre);
1438 break;
1439 }
1440 } /* the run loop */
1441
1442
1443 /*
1444 * If the last exit was stateful, commit the state we provided before
1445 * returning to the EM loop so we have a consistent state and can safely
1446 * be rescheduled and whatnot. This may require us to make multiple runs
1447 * for larger MMIO and I/O operations. Sigh^3.
1448 *
1449 * Note! There is no 'ing way to reset the kernel side completion callback
1450 * for these stateful i/o exits. Very annoying interface.
1451 */
1452 /** @todo check how this works with string I/O and string MMIO. */
1453 if (fStatefulExit && RT_SUCCESS(rcStrict))
1454 {
1455 STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatFlushExitOnReturn);
1456 uint32_t const uOrgExit = pRun->exit_reason;
1457 for (uint32_t i = 0; ; i++)
1458 {
1459 pRun->immediate_exit = 1;
1460 int rcLnx = ioctl(pVCpu->nem.s.fdVCpu, KVM_RUN, 0UL);
1461 Log(("NEM/%u: Flushed stateful exit -> %d/%d exit_reason=%d\n", pVCpu->idCpu, rcLnx, errno, pRun->exit_reason));
1462 if (rcLnx == -1 && errno == EINTR)
1463 {
1464 switch (i)
1465 {
1466 case 0: STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatFlushExitOnReturn1Loop); break;
1467 case 1: STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatFlushExitOnReturn2Loops); break;
1468 case 2: STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatFlushExitOnReturn3Loops); break;
1469 default: STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatFlushExitOnReturn4PlusLoops); break;
1470 }
1471 break;
1472 }
1473 AssertLogRelMsgBreakStmt(rcLnx == 0 && pRun->exit_reason == uOrgExit,
1474 ("rcLnx=%d errno=%d exit_reason=%d uOrgExit=%d\n", rcLnx, errno, pRun->exit_reason, uOrgExit),
1475 rcStrict = VERR_NEM_IPE_6);
1476 VBOXSTRICTRC rcStrict2 = nemHCLnxHandleExit(pVM, pVCpu, pRun, &fStatefulExit);
1477 if (rcStrict2 == VINF_SUCCESS || rcStrict2 == rcStrict)
1478 { /* likely */ }
1479 else if (RT_FAILURE(rcStrict2))
1480 {
1481 rcStrict = rcStrict2;
1482 break;
1483 }
1484 else
1485 {
1486 AssertLogRelMsgBreakStmt(rcStrict == VINF_SUCCESS,
1487 ("rcStrict=%Rrc rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict), VBOXSTRICTRC_VAL(rcStrict2)),
1488 rcStrict = VERR_NEM_IPE_7);
1489 rcStrict = rcStrict2;
1490 }
1491 }
1492 pRun->immediate_exit = 0;
1493 }
1494
1495 /*
1496 * If the CPU is running, make sure to stop it before we try sync back the
1497 * state and return to EM. We don't sync back the whole state if we can help it.
1498 */
1499 if (!VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED, VMCPUSTATE_STARTED_EXEC_NEM))
1500 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED, VMCPUSTATE_STARTED_EXEC_NEM_CANCELED);
1501
1502 if (pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_ALL)
1503 {
1504 /* Try anticipate what we might need. */
1505 uint64_t fImport = IEM_CPUMCTX_EXTRN_MUST_MASK /*?*/;
1506 if ( (rcStrict >= VINF_EM_FIRST && rcStrict <= VINF_EM_LAST)
1507 || RT_FAILURE(rcStrict))
1508 fImport = CPUMCTX_EXTRN_ALL;
1509 else if (VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_IRQ | VMCPU_FF_INTERRUPT_FIQ))
1510 fImport |= IEM_CPUMCTX_EXTRN_XCPT_MASK;
1511
1512 if (pVCpu->cpum.GstCtx.fExtrn & fImport)
1513 {
1514 int rc2 = nemHCLnxImportState(pVCpu, fImport, &pVCpu->cpum.GstCtx);
1515 if (RT_SUCCESS(rc2))
1516 pVCpu->cpum.GstCtx.fExtrn &= ~fImport;
1517 else if (RT_SUCCESS(rcStrict))
1518 rcStrict = rc2;
1519 if (!(pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_ALL))
1520 pVCpu->cpum.GstCtx.fExtrn = 0;
1521 STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatImportOnReturn);
1522 }
1523 else
1524 STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatImportOnReturnSkipped);
1525 }
1526 else
1527 {
1528 pVCpu->cpum.GstCtx.fExtrn = 0;
1529 STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatImportOnReturnSkipped);
1530 }
1531
1532 LogFlow(("NEM/%u: %08RX64 => %Rrc\n", pVCpu->idCpu, pVCpu->cpum.GstCtx.Pc, VBOXSTRICTRC_VAL(rcStrict) ));
1533 return rcStrict;
1534}
1535
1536
1537/** @page pg_nem_linux_armv8 NEM/linux - Native Execution Manager, Linux.
1538 *
1539 * This is using KVM.
1540 *
1541 */
1542
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette