VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR3/NEMR3Native-linux.cpp@ 104870

Last change on this file since 104870 was 104740, checked in by vboxsync, 4 months ago

VMM/NEMR3Native-linux.cpp: Make it use the template code shared with the ARMv8 backend, bugref:9044
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 77.1 KB
Line 
1/* $Id: NEMR3Native-linux.cpp 104740 2024-05-20 18:33:12Z vboxsync $ */
2/** @file
3 * NEM - Native execution manager, native ring-3 Linux backend.
4 */
5
6/*
7 * Copyright (C) 2021-2023 Oracle and/or its affiliates.
8 *
9 * This file is part of VirtualBox base platform packages, as
10 * available from https://www.virtualbox.org.
11 *
12 * This program is free software; you can redistribute it and/or
13 * modify it under the terms of the GNU General Public License
14 * as published by the Free Software Foundation, in version 3 of the
15 * License.
16 *
17 * This program is distributed in the hope that it will be useful, but
18 * WITHOUT ANY WARRANTY; without even the implied warranty of
19 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20 * General Public License for more details.
21 *
22 * You should have received a copy of the GNU General Public License
23 * along with this program; if not, see <https://www.gnu.org/licenses>.
24 *
25 * SPDX-License-Identifier: GPL-3.0-only
26 */
27
28
29/*********************************************************************************************************************************
30* Header Files *
31*********************************************************************************************************************************/
32#define LOG_GROUP LOG_GROUP_NEM
33#define VMCPU_INCL_CPUM_GST_CTX
34#include <VBox/vmm/nem.h>
35#include <VBox/vmm/iem.h>
36#include <VBox/vmm/em.h>
37#include <VBox/vmm/apic.h>
38#include <VBox/vmm/pdm.h>
39#include <VBox/vmm/trpm.h>
40#include "NEMInternal.h"
41#include <VBox/vmm/vmcc.h>
42
43#include <iprt/alloca.h>
44#include <iprt/string.h>
45#include <iprt/system.h>
46#include <iprt/x86.h>
47
48#include <errno.h>
49#include <unistd.h>
50#include <sys/ioctl.h>
51#include <sys/fcntl.h>
52#include <sys/mman.h>
53#include <linux/kvm.h>
54
55
56/* Forward declarations of things called by the template. */
57static int nemR3LnxInitSetupVm(PVM pVM, PRTERRINFO pErrInfo);
58
59
60/* Instantiate the common bits we share with the ARMv8 KVM backend. */
61#include "NEMR3NativeTemplate-linux.cpp.h"
62
63
64
65/**
66 * Does the early setup of a KVM VM.
67 *
68 * @returns VBox status code.
69 * @param pVM The cross context VM structure.
70 * @param pErrInfo Where to always return error info.
71 */
72static int nemR3LnxInitSetupVm(PVM pVM, PRTERRINFO pErrInfo)
73{
74 AssertReturn(pVM->nem.s.fdVm != -1, RTErrInfoSet(pErrInfo, VERR_WRONG_ORDER, "Wrong initalization order"));
75
76 /*
77 * Enable user space MSRs and let us check everything KVM cannot handle.
78 * We will set up filtering later when ring-3 init has completed.
79 */
80 struct kvm_enable_cap CapEn =
81 {
82 KVM_CAP_X86_USER_SPACE_MSR, 0,
83 { KVM_MSR_EXIT_REASON_FILTER | KVM_MSR_EXIT_REASON_UNKNOWN | KVM_MSR_EXIT_REASON_INVAL, 0, 0, 0}
84 };
85 int rcLnx = ioctl(pVM->nem.s.fdVm, KVM_ENABLE_CAP, &CapEn);
86 if (rcLnx == -1)
87 return RTErrInfoSetF(pErrInfo, VERR_NEM_VM_CREATE_FAILED, "Failed to enable KVM_CAP_X86_USER_SPACE_MSR failed: %u", errno);
88
89 /*
90 * Create the VCpus.
91 */
92 for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
93 {
94 PVMCPU pVCpu = pVM->apCpusR3[idCpu];
95
96 /* Create it. */
97 pVCpu->nem.s.fdVCpu = ioctl(pVM->nem.s.fdVm, KVM_CREATE_VCPU, (unsigned long)idCpu);
98 if (pVCpu->nem.s.fdVCpu < 0)
99 return RTErrInfoSetF(pErrInfo, VERR_NEM_VM_CREATE_FAILED, "KVM_CREATE_VCPU failed for VCpu #%u: %d", idCpu, errno);
100
101 /* Map the KVM_RUN area. */
102 pVCpu->nem.s.pRun = (struct kvm_run *)mmap(NULL, pVM->nem.s.cbVCpuMmap, PROT_READ | PROT_WRITE, MAP_SHARED,
103 pVCpu->nem.s.fdVCpu, 0 /*offset*/);
104 if ((void *)pVCpu->nem.s.pRun == MAP_FAILED)
105 return RTErrInfoSetF(pErrInfo, VERR_NEM_VM_CREATE_FAILED, "mmap failed for VCpu #%u: %d", idCpu, errno);
106
107 /* We want all x86 registers and events on each exit. */
108 pVCpu->nem.s.pRun->kvm_valid_regs = KVM_SYNC_X86_REGS | KVM_SYNC_X86_SREGS | KVM_SYNC_X86_EVENTS;
109 }
110 return VINF_SUCCESS;
111}
112
113
114/**
115 * Update the CPUID leaves for a VCPU.
116 *
117 * The KVM_SET_CPUID2 call replaces any previous leaves, so we have to redo
118 * everything when there really just are single bit changes. That said, it
119 * looks like KVM update the XCR/XSAVE related stuff as well as the APIC enabled
120 * bit(s), so it should suffice if we do this at startup, I hope.
121 */
122static int nemR3LnxUpdateCpuIdsLeaves(PVM pVM, PVMCPU pVCpu)
123{
124 uint32_t cLeaves = 0;
125 PCCPUMCPUIDLEAF const paLeaves = CPUMR3CpuIdGetPtr(pVM, &cLeaves);
126 struct kvm_cpuid2 *pReq = (struct kvm_cpuid2 *)alloca(RT_UOFFSETOF_DYN(struct kvm_cpuid2, entries[cLeaves + 2]));
127
128 pReq->nent = cLeaves;
129 pReq->padding = 0;
130
131 for (uint32_t i = 0; i < cLeaves; i++)
132 {
133 CPUMGetGuestCpuId(pVCpu, paLeaves[i].uLeaf, paLeaves[i].uSubLeaf, -1 /*f64BitMode*/,
134 &pReq->entries[i].eax,
135 &pReq->entries[i].ebx,
136 &pReq->entries[i].ecx,
137 &pReq->entries[i].edx);
138 pReq->entries[i].function = paLeaves[i].uLeaf;
139 pReq->entries[i].index = paLeaves[i].uSubLeaf;
140 pReq->entries[i].flags = !paLeaves[i].fSubLeafMask ? 0 : KVM_CPUID_FLAG_SIGNIFCANT_INDEX;
141 pReq->entries[i].padding[0] = 0;
142 pReq->entries[i].padding[1] = 0;
143 pReq->entries[i].padding[2] = 0;
144 }
145
146 int rcLnx = ioctl(pVCpu->nem.s.fdVCpu, KVM_SET_CPUID2, pReq);
147 AssertLogRelMsgReturn(rcLnx == 0, ("rcLnx=%d errno=%d cLeaves=%#x\n", rcLnx, errno, cLeaves), RTErrConvertFromErrno(errno));
148
149 return VINF_SUCCESS;
150}
151
152
153int nemR3NativeInitCompleted(PVM pVM, VMINITCOMPLETED enmWhat)
154{
155 /*
156 * Make RTThreadPoke work again (disabled for avoiding unnecessary
157 * critical section issues in ring-0).
158 */
159 if (enmWhat == VMINITCOMPLETED_RING3)
160 VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ALL_AT_ONCE, nemR3LnxFixThreadPoke, NULL);
161
162 /*
163 * Configure CPUIDs after ring-3 init has been done.
164 */
165 if (enmWhat == VMINITCOMPLETED_RING3)
166 {
167 for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
168 {
169 int rc = nemR3LnxUpdateCpuIdsLeaves(pVM, pVM->apCpusR3[idCpu]);
170 AssertRCReturn(rc, rc);
171 }
172 }
173
174 /*
175 * Configure MSRs after ring-3 init is done.
176 *
177 * We only need to tell KVM which MSRs it can handle, as we already
178 * requested KVM_MSR_EXIT_REASON_FILTER, KVM_MSR_EXIT_REASON_UNKNOWN
179 * and KVM_MSR_EXIT_REASON_INVAL in nemR3LnxInitSetupVm, and here we
180 * will use KVM_MSR_FILTER_DEFAULT_DENY. So, all MSRs w/o a 1 in the
181 * bitmaps should be deferred to ring-3.
182 */
183 if (enmWhat == VMINITCOMPLETED_RING3)
184 {
185 struct kvm_msr_filter MsrFilters = {0}; /* Structure with a couple of implicit paddings on 64-bit systems. */
186 MsrFilters.flags = KVM_MSR_FILTER_DEFAULT_DENY;
187
188 unsigned iRange = 0;
189#define MSR_RANGE_BEGIN(a_uBase, a_uEnd, a_fFlags) \
190 AssertCompile(0x3000 <= KVM_MSR_FILTER_MAX_BITMAP_SIZE * 8); \
191 uint64_t RT_CONCAT(bm, a_uBase)[0x3000 / 64] = {0}; \
192 do { \
193 uint64_t * const pbm = RT_CONCAT(bm, a_uBase); \
194 uint32_t const uBase = UINT32_C(a_uBase); \
195 uint32_t const cMsrs = UINT32_C(a_uEnd) - UINT32_C(a_uBase); \
196 MsrFilters.ranges[iRange].base = UINT32_C(a_uBase); \
197 MsrFilters.ranges[iRange].nmsrs = cMsrs; \
198 MsrFilters.ranges[iRange].flags = (a_fFlags); \
199 MsrFilters.ranges[iRange].bitmap = (uint8_t *)&RT_CONCAT(bm, a_uBase)[0]
200#define MSR_RANGE_ADD(a_Msr) \
201 do { Assert((uint32_t)(a_Msr) - uBase < cMsrs); ASMBitSet(pbm, (uint32_t)(a_Msr) - uBase); } while (0)
202#define MSR_RANGE_END(a_cMinMsrs) \
203 /* optimize the range size before closing: */ \
204 uint32_t cBitmap = cMsrs / 64; \
205 while (cBitmap > ((a_cMinMsrs) + 63 / 64) && pbm[cBitmap - 1] == 0) \
206 cBitmap -= 1; \
207 MsrFilters.ranges[iRange].nmsrs = cBitmap * 64; \
208 iRange++; \
209 } while (0)
210
211 /* 1st Intel range: 0000_0000 to 0000_3000. */
212 MSR_RANGE_BEGIN(0x00000000, 0x00003000, KVM_MSR_FILTER_READ | KVM_MSR_FILTER_WRITE);
213 MSR_RANGE_ADD(MSR_IA32_TSC);
214 MSR_RANGE_ADD(MSR_IA32_SYSENTER_CS);
215 MSR_RANGE_ADD(MSR_IA32_SYSENTER_ESP);
216 MSR_RANGE_ADD(MSR_IA32_SYSENTER_EIP);
217 MSR_RANGE_ADD(MSR_IA32_CR_PAT);
218 /** @todo more? */
219 MSR_RANGE_END(64);
220
221 /* 1st AMD range: c000_0000 to c000_3000 */
222 MSR_RANGE_BEGIN(0xc0000000, 0xc0003000, KVM_MSR_FILTER_READ | KVM_MSR_FILTER_WRITE);
223 MSR_RANGE_ADD(MSR_K6_EFER);
224 MSR_RANGE_ADD(MSR_K6_STAR);
225 MSR_RANGE_ADD(MSR_K8_GS_BASE);
226 MSR_RANGE_ADD(MSR_K8_KERNEL_GS_BASE);
227 MSR_RANGE_ADD(MSR_K8_LSTAR);
228 MSR_RANGE_ADD(MSR_K8_CSTAR);
229 MSR_RANGE_ADD(MSR_K8_SF_MASK);
230 MSR_RANGE_ADD(MSR_K8_TSC_AUX);
231 /** @todo add more? */
232 MSR_RANGE_END(64);
233
234 /** @todo Specify other ranges too? Like hyper-V and KVM to make sure we get
235 * the MSR requests instead of KVM. */
236
237 int rcLnx = ioctl(pVM->nem.s.fdVm, KVM_X86_SET_MSR_FILTER, &MsrFilters);
238 if (rcLnx == -1)
239 return VMSetError(pVM, VERR_NEM_VM_CREATE_FAILED, RT_SRC_POS,
240 "Failed to enable KVM_X86_SET_MSR_FILTER failed: %u", errno);
241 }
242
243 return VINF_SUCCESS;
244}
245
246
247/*********************************************************************************************************************************
248* CPU State *
249*********************************************************************************************************************************/
250
251/**
252 * Worker that imports selected state from KVM.
253 */
254static int nemHCLnxImportState(PVMCPUCC pVCpu, uint64_t fWhat, PCPUMCTX pCtx, struct kvm_run *pRun)
255{
256 fWhat &= pVCpu->cpum.GstCtx.fExtrn;
257 if (!fWhat)
258 return VINF_SUCCESS;
259
260 /*
261 * Stuff that goes into kvm_run::s.regs.regs:
262 */
263 if (fWhat & (CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_GPRS_MASK))
264 {
265 if (fWhat & CPUMCTX_EXTRN_RIP)
266 pCtx->rip = pRun->s.regs.regs.rip;
267 if (fWhat & CPUMCTX_EXTRN_RFLAGS)
268 pCtx->rflags.u = pRun->s.regs.regs.rflags;
269
270 if (fWhat & CPUMCTX_EXTRN_RAX)
271 pCtx->rax = pRun->s.regs.regs.rax;
272 if (fWhat & CPUMCTX_EXTRN_RCX)
273 pCtx->rcx = pRun->s.regs.regs.rcx;
274 if (fWhat & CPUMCTX_EXTRN_RDX)
275 pCtx->rdx = pRun->s.regs.regs.rdx;
276 if (fWhat & CPUMCTX_EXTRN_RBX)
277 pCtx->rbx = pRun->s.regs.regs.rbx;
278 if (fWhat & CPUMCTX_EXTRN_RSP)
279 pCtx->rsp = pRun->s.regs.regs.rsp;
280 if (fWhat & CPUMCTX_EXTRN_RBP)
281 pCtx->rbp = pRun->s.regs.regs.rbp;
282 if (fWhat & CPUMCTX_EXTRN_RSI)
283 pCtx->rsi = pRun->s.regs.regs.rsi;
284 if (fWhat & CPUMCTX_EXTRN_RDI)
285 pCtx->rdi = pRun->s.regs.regs.rdi;
286 if (fWhat & CPUMCTX_EXTRN_R8_R15)
287 {
288 pCtx->r8 = pRun->s.regs.regs.r8;
289 pCtx->r9 = pRun->s.regs.regs.r9;
290 pCtx->r10 = pRun->s.regs.regs.r10;
291 pCtx->r11 = pRun->s.regs.regs.r11;
292 pCtx->r12 = pRun->s.regs.regs.r12;
293 pCtx->r13 = pRun->s.regs.regs.r13;
294 pCtx->r14 = pRun->s.regs.regs.r14;
295 pCtx->r15 = pRun->s.regs.regs.r15;
296 }
297 }
298
299 /*
300 * Stuff that goes into kvm_run::s.regs.sregs.
301 *
302 * Note! The apic_base can be ignored because we gets all MSR writes to it
303 * and VBox always keeps the correct value.
304 */
305 bool fMaybeChangedMode = false;
306 bool fUpdateCr3 = false;
307 if (fWhat & ( CPUMCTX_EXTRN_SREG_MASK | CPUMCTX_EXTRN_TABLE_MASK | CPUMCTX_EXTRN_CR_MASK
308 | CPUMCTX_EXTRN_EFER | CPUMCTX_EXTRN_APIC_TPR))
309 {
310 /** @todo what about Attr.n.u4LimitHigh? */
311#define NEM_LNX_IMPORT_SEG(a_CtxSeg, a_KvmSeg) do { \
312 (a_CtxSeg).u64Base = (a_KvmSeg).base; \
313 (a_CtxSeg).u32Limit = (a_KvmSeg).limit; \
314 (a_CtxSeg).ValidSel = (a_CtxSeg).Sel = (a_KvmSeg).selector; \
315 (a_CtxSeg).Attr.n.u4Type = (a_KvmSeg).type; \
316 (a_CtxSeg).Attr.n.u1DescType = (a_KvmSeg).s; \
317 (a_CtxSeg).Attr.n.u2Dpl = (a_KvmSeg).dpl; \
318 (a_CtxSeg).Attr.n.u1Present = (a_KvmSeg).present; \
319 (a_CtxSeg).Attr.n.u1Available = (a_KvmSeg).avl; \
320 (a_CtxSeg).Attr.n.u1Long = (a_KvmSeg).l; \
321 (a_CtxSeg).Attr.n.u1DefBig = (a_KvmSeg).db; \
322 (a_CtxSeg).Attr.n.u1Granularity = (a_KvmSeg).g; \
323 (a_CtxSeg).Attr.n.u1Unusable = (a_KvmSeg).unusable; \
324 (a_CtxSeg).fFlags = CPUMSELREG_FLAGS_VALID; \
325 CPUMSELREG_ARE_HIDDEN_PARTS_VALID(pVCpu, &(a_CtxSeg)); \
326 } while (0)
327
328 if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
329 {
330 if (fWhat & CPUMCTX_EXTRN_ES)
331 NEM_LNX_IMPORT_SEG(pCtx->es, pRun->s.regs.sregs.es);
332 if (fWhat & CPUMCTX_EXTRN_CS)
333 NEM_LNX_IMPORT_SEG(pCtx->cs, pRun->s.regs.sregs.cs);
334 if (fWhat & CPUMCTX_EXTRN_SS)
335 NEM_LNX_IMPORT_SEG(pCtx->ss, pRun->s.regs.sregs.ss);
336 if (fWhat & CPUMCTX_EXTRN_DS)
337 NEM_LNX_IMPORT_SEG(pCtx->ds, pRun->s.regs.sregs.ds);
338 if (fWhat & CPUMCTX_EXTRN_FS)
339 NEM_LNX_IMPORT_SEG(pCtx->fs, pRun->s.regs.sregs.fs);
340 if (fWhat & CPUMCTX_EXTRN_GS)
341 NEM_LNX_IMPORT_SEG(pCtx->gs, pRun->s.regs.sregs.gs);
342 }
343 if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
344 {
345 if (fWhat & CPUMCTX_EXTRN_GDTR)
346 {
347 pCtx->gdtr.pGdt = pRun->s.regs.sregs.gdt.base;
348 pCtx->gdtr.cbGdt = pRun->s.regs.sregs.gdt.limit;
349 }
350 if (fWhat & CPUMCTX_EXTRN_IDTR)
351 {
352 pCtx->idtr.pIdt = pRun->s.regs.sregs.idt.base;
353 pCtx->idtr.cbIdt = pRun->s.regs.sregs.idt.limit;
354 }
355 if (fWhat & CPUMCTX_EXTRN_LDTR)
356 NEM_LNX_IMPORT_SEG(pCtx->ldtr, pRun->s.regs.sregs.ldt);
357 if (fWhat & CPUMCTX_EXTRN_TR)
358 NEM_LNX_IMPORT_SEG(pCtx->tr, pRun->s.regs.sregs.tr);
359 }
360 if (fWhat & CPUMCTX_EXTRN_CR_MASK)
361 {
362 if (fWhat & CPUMCTX_EXTRN_CR0)
363 {
364 if (pVCpu->cpum.GstCtx.cr0 != pRun->s.regs.sregs.cr0)
365 {
366 CPUMSetGuestCR0(pVCpu, pRun->s.regs.sregs.cr0);
367 fMaybeChangedMode = true;
368 }
369 }
370 if (fWhat & CPUMCTX_EXTRN_CR2)
371 pCtx->cr2 = pRun->s.regs.sregs.cr2;
372 if (fWhat & CPUMCTX_EXTRN_CR3)
373 {
374 if (pCtx->cr3 != pRun->s.regs.sregs.cr3)
375 {
376 CPUMSetGuestCR3(pVCpu, pRun->s.regs.sregs.cr3);
377 fUpdateCr3 = true;
378 }
379 }
380 if (fWhat & CPUMCTX_EXTRN_CR4)
381 {
382 if (pCtx->cr4 != pRun->s.regs.sregs.cr4)
383 {
384 CPUMSetGuestCR4(pVCpu, pRun->s.regs.sregs.cr4);
385 fMaybeChangedMode = true;
386 }
387 }
388 }
389 if (fWhat & CPUMCTX_EXTRN_APIC_TPR)
390 APICSetTpr(pVCpu, (uint8_t)pRun->s.regs.sregs.cr8 << 4);
391 if (fWhat & CPUMCTX_EXTRN_EFER)
392 {
393 if (pCtx->msrEFER != pRun->s.regs.sregs.efer)
394 {
395 Log7(("NEM/%u: MSR EFER changed %RX64 -> %RX64\n", pVCpu->idCpu, pVCpu->cpum.GstCtx.msrEFER, pRun->s.regs.sregs.efer));
396 if ((pRun->s.regs.sregs.efer ^ pVCpu->cpum.GstCtx.msrEFER) & MSR_K6_EFER_NXE)
397 PGMNotifyNxeChanged(pVCpu, RT_BOOL(pRun->s.regs.sregs.efer & MSR_K6_EFER_NXE));
398 pCtx->msrEFER = pRun->s.regs.sregs.efer;
399 fMaybeChangedMode = true;
400 }
401 }
402#undef NEM_LNX_IMPORT_SEG
403 }
404
405 /*
406 * Debug registers.
407 */
408 if (fWhat & CPUMCTX_EXTRN_DR_MASK)
409 {
410 struct kvm_debugregs DbgRegs = {{0}};
411 int rc = ioctl(pVCpu->nem.s.fdVCpu, KVM_GET_DEBUGREGS, &DbgRegs);
412 AssertMsgReturn(rc == 0, ("rc=%d errno=%d\n", rc, errno), VERR_NEM_IPE_3);
413
414 if (fWhat & CPUMCTX_EXTRN_DR0_DR3)
415 {
416 pCtx->dr[0] = DbgRegs.db[0];
417 pCtx->dr[1] = DbgRegs.db[1];
418 pCtx->dr[2] = DbgRegs.db[2];
419 pCtx->dr[3] = DbgRegs.db[3];
420 }
421 if (fWhat & CPUMCTX_EXTRN_DR6)
422 pCtx->dr[6] = DbgRegs.dr6;
423 if (fWhat & CPUMCTX_EXTRN_DR7)
424 pCtx->dr[7] = DbgRegs.dr7;
425 }
426
427 /*
428 * FPU, SSE, AVX, ++.
429 */
430 if (fWhat & (CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX | CPUMCTX_EXTRN_OTHER_XSAVE | CPUMCTX_EXTRN_XCRx))
431 {
432 if (fWhat & (CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX | CPUMCTX_EXTRN_OTHER_XSAVE))
433 {
434 fWhat |= CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX | CPUMCTX_EXTRN_OTHER_XSAVE; /* we do all or nothing at all */
435
436 AssertCompile(sizeof(pCtx->XState) >= sizeof(struct kvm_xsave));
437 int rc = ioctl(pVCpu->nem.s.fdVCpu, KVM_GET_XSAVE, &pCtx->XState);
438 AssertMsgReturn(rc == 0, ("rc=%d errno=%d\n", rc, errno), VERR_NEM_IPE_3);
439 }
440
441 if (fWhat & CPUMCTX_EXTRN_XCRx)
442 {
443 struct kvm_xcrs Xcrs =
444 { /*.nr_xcrs = */ 2,
445 /*.flags = */ 0,
446 /*.xcrs= */ {
447 { /*.xcr =*/ 0, /*.reserved=*/ 0, /*.value=*/ pCtx->aXcr[0] },
448 { /*.xcr =*/ 1, /*.reserved=*/ 0, /*.value=*/ pCtx->aXcr[1] },
449 }
450 };
451
452 int rc = ioctl(pVCpu->nem.s.fdVCpu, KVM_GET_XCRS, &Xcrs);
453 AssertMsgReturn(rc == 0, ("rc=%d errno=%d\n", rc, errno), VERR_NEM_IPE_3);
454
455 pCtx->aXcr[0] = Xcrs.xcrs[0].value;
456 pCtx->aXcr[1] = Xcrs.xcrs[1].value;
457 }
458 }
459
460 /*
461 * MSRs.
462 */
463 if (fWhat & ( CPUMCTX_EXTRN_KERNEL_GS_BASE | CPUMCTX_EXTRN_SYSCALL_MSRS | CPUMCTX_EXTRN_SYSENTER_MSRS
464 | CPUMCTX_EXTRN_TSC_AUX | CPUMCTX_EXTRN_OTHER_MSRS))
465 {
466 union
467 {
468 struct kvm_msrs Core;
469 uint64_t padding[2 + sizeof(struct kvm_msr_entry) * 32];
470 } uBuf;
471 uint64_t *pauDsts[32];
472 uint32_t iMsr = 0;
473 PCPUMCTXMSRS const pCtxMsrs = CPUMQueryGuestCtxMsrsPtr(pVCpu);
474
475#define ADD_MSR(a_Msr, a_uValue) do { \
476 Assert(iMsr < 32); \
477 uBuf.Core.entries[iMsr].index = (a_Msr); \
478 uBuf.Core.entries[iMsr].reserved = 0; \
479 uBuf.Core.entries[iMsr].data = UINT64_MAX; \
480 pauDsts[iMsr] = &(a_uValue); \
481 iMsr += 1; \
482 } while (0)
483
484 if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
485 ADD_MSR(MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE);
486 if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
487 {
488 ADD_MSR(MSR_K6_STAR, pCtx->msrSTAR);
489 ADD_MSR(MSR_K8_LSTAR, pCtx->msrLSTAR);
490 ADD_MSR(MSR_K8_CSTAR, pCtx->msrCSTAR);
491 ADD_MSR(MSR_K8_SF_MASK, pCtx->msrSFMASK);
492 }
493 if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
494 {
495 ADD_MSR(MSR_IA32_SYSENTER_CS, pCtx->SysEnter.cs);
496 ADD_MSR(MSR_IA32_SYSENTER_EIP, pCtx->SysEnter.eip);
497 ADD_MSR(MSR_IA32_SYSENTER_ESP, pCtx->SysEnter.esp);
498 }
499 if (fWhat & CPUMCTX_EXTRN_TSC_AUX)
500 ADD_MSR(MSR_K8_TSC_AUX, pCtxMsrs->msr.TscAux);
501 if (fWhat & CPUMCTX_EXTRN_OTHER_MSRS)
502 {
503 ADD_MSR(MSR_IA32_CR_PAT, pCtx->msrPAT);
504 /** @todo What do we _have_ to add here?
505 * We also have: Mttr*, MiscEnable, FeatureControl. */
506 }
507
508 uBuf.Core.pad = 0;
509 uBuf.Core.nmsrs = iMsr;
510 int rc = ioctl(pVCpu->nem.s.fdVCpu, KVM_GET_MSRS, &uBuf);
511 AssertMsgReturn(rc == (int)iMsr,
512 ("rc=%d iMsr=%d (->%#x) errno=%d\n",
513 rc, iMsr, (uint32_t)rc < iMsr ? uBuf.Core.entries[rc].index : 0, errno),
514 VERR_NEM_IPE_3);
515
516 while (iMsr-- > 0)
517 *pauDsts[iMsr] = uBuf.Core.entries[iMsr].data;
518#undef ADD_MSR
519 }
520
521 /*
522 * Interruptibility state and pending interrupts.
523 */
524 if (fWhat & (CPUMCTX_EXTRN_INHIBIT_INT | CPUMCTX_EXTRN_INHIBIT_NMI))
525 {
526 fWhat |= CPUMCTX_EXTRN_INHIBIT_INT | CPUMCTX_EXTRN_INHIBIT_NMI; /* always do both, see export and interrupt FF handling */
527
528 struct kvm_vcpu_events KvmEvents = {0};
529 int rcLnx = ioctl(pVCpu->nem.s.fdVCpu, KVM_GET_VCPU_EVENTS, &KvmEvents);
530 AssertLogRelMsgReturn(rcLnx == 0, ("rcLnx=%d errno=%d\n", rcLnx, errno), VERR_NEM_IPE_3);
531
532 if (pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_RIP)
533 pVCpu->cpum.GstCtx.rip = pRun->s.regs.regs.rip;
534
535 CPUMUpdateInterruptShadowSsStiEx(&pVCpu->cpum.GstCtx,
536 RT_BOOL(KvmEvents.interrupt.shadow & KVM_X86_SHADOW_INT_MOV_SS),
537 RT_BOOL(KvmEvents.interrupt.shadow & KVM_X86_SHADOW_INT_STI),
538 pVCpu->cpum.GstCtx.rip);
539 CPUMUpdateInterruptInhibitingByNmi(&pVCpu->cpum.GstCtx, KvmEvents.nmi.masked != 0);
540
541 if (KvmEvents.interrupt.injected)
542 {
543 STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatImportPendingInterrupt);
544 TRPMAssertTrap(pVCpu, KvmEvents.interrupt.nr, !KvmEvents.interrupt.soft ? TRPM_HARDWARE_INT : TRPM_SOFTWARE_INT);
545 }
546
547 Assert(KvmEvents.nmi.injected == 0);
548 Assert(KvmEvents.nmi.pending == 0);
549 }
550
551 /*
552 * Update the external mask.
553 */
554 pCtx->fExtrn &= ~fWhat;
555 pVCpu->cpum.GstCtx.fExtrn &= ~fWhat;
556 if (!(pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_ALL))
557 pVCpu->cpum.GstCtx.fExtrn = 0;
558
559 /*
560 * We sometimes need to update PGM on the guest status.
561 */
562 if (!fMaybeChangedMode && !fUpdateCr3)
563 { /* likely */ }
564 else
565 {
566 /*
567 * Make sure we got all the state PGM might need.
568 */
569 Log7(("nemHCLnxImportState: fMaybeChangedMode=%d fUpdateCr3=%d fExtrnNeeded=%#RX64\n", fMaybeChangedMode, fUpdateCr3,
570 pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_CR4 | CPUMCTX_EXTRN_CR3 | CPUMCTX_EXTRN_EFER) ));
571 if (pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_CR4 | CPUMCTX_EXTRN_CR3 | CPUMCTX_EXTRN_EFER))
572 {
573 if (pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_CR0)
574 {
575 if (pVCpu->cpum.GstCtx.cr0 != pRun->s.regs.sregs.cr0)
576 {
577 CPUMSetGuestCR0(pVCpu, pRun->s.regs.sregs.cr0);
578 fMaybeChangedMode = true;
579 }
580 }
581 if (pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_CR3)
582 {
583 if (pCtx->cr3 != pRun->s.regs.sregs.cr3)
584 {
585 CPUMSetGuestCR3(pVCpu, pRun->s.regs.sregs.cr3);
586 fUpdateCr3 = true;
587 }
588 }
589 if (pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_CR4)
590 {
591 if (pCtx->cr4 != pRun->s.regs.sregs.cr4)
592 {
593 CPUMSetGuestCR4(pVCpu, pRun->s.regs.sregs.cr4);
594 fMaybeChangedMode = true;
595 }
596 }
597 if (fWhat & CPUMCTX_EXTRN_EFER)
598 {
599 if (pCtx->msrEFER != pRun->s.regs.sregs.efer)
600 {
601 Log7(("NEM/%u: MSR EFER changed %RX64 -> %RX64\n", pVCpu->idCpu, pVCpu->cpum.GstCtx.msrEFER, pRun->s.regs.sregs.efer));
602 if ((pRun->s.regs.sregs.efer ^ pVCpu->cpum.GstCtx.msrEFER) & MSR_K6_EFER_NXE)
603 PGMNotifyNxeChanged(pVCpu, RT_BOOL(pRun->s.regs.sregs.efer & MSR_K6_EFER_NXE));
604 pCtx->msrEFER = pRun->s.regs.sregs.efer;
605 fMaybeChangedMode = true;
606 }
607 }
608
609 pVCpu->cpum.GstCtx.fExtrn &= ~(CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_CR4 | CPUMCTX_EXTRN_CR3 | CPUMCTX_EXTRN_EFER);
610 if (!(pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_ALL))
611 pVCpu->cpum.GstCtx.fExtrn = 0;
612 }
613
614 /*
615 * Notify PGM about the changes.
616 */
617 if (fMaybeChangedMode)
618 {
619 int rc = PGMChangeMode(pVCpu, pVCpu->cpum.GstCtx.cr0, pVCpu->cpum.GstCtx.cr4,
620 pVCpu->cpum.GstCtx.msrEFER, false /*fForce*/);
621 AssertMsgReturn(rc == VINF_SUCCESS, ("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_NEM_IPE_1);
622 }
623
624 if (fUpdateCr3)
625 {
626 int rc = PGMUpdateCR3(pVCpu, pVCpu->cpum.GstCtx.cr3);
627 if (rc == VINF_SUCCESS)
628 { /* likely */ }
629 else
630 AssertMsgFailedReturn(("rc=%Rrc\n", rc), RT_FAILURE_NP(rc) ? rc : VERR_NEM_IPE_2);
631 }
632 }
633
634 return VINF_SUCCESS;
635}
636
637
638/**
639 * Interface for importing state on demand (used by IEM).
640 *
641 * @returns VBox status code.
642 * @param pVCpu The cross context CPU structure.
643 * @param fWhat What to import, CPUMCTX_EXTRN_XXX.
644 */
645VMM_INT_DECL(int) NEMImportStateOnDemand(PVMCPUCC pVCpu, uint64_t fWhat)
646{
647 STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatImportOnDemand);
648 return nemHCLnxImportState(pVCpu, fWhat, &pVCpu->cpum.GstCtx, pVCpu->nem.s.pRun);
649}
650
651
652/**
653 * Exports state to KVM.
654 */
655static int nemHCLnxExportState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, struct kvm_run *pRun)
656{
657 uint64_t const fExtrn = ~pCtx->fExtrn & CPUMCTX_EXTRN_ALL;
658 Assert((~fExtrn & CPUMCTX_EXTRN_ALL) != CPUMCTX_EXTRN_ALL);
659
660 /*
661 * Stuff that goes into kvm_run::s.regs.regs:
662 */
663 if (fExtrn & (CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_GPRS_MASK))
664 {
665 if (fExtrn & CPUMCTX_EXTRN_RIP)
666 pRun->s.regs.regs.rip = pCtx->rip;
667 if (fExtrn & CPUMCTX_EXTRN_RFLAGS)
668 pRun->s.regs.regs.rflags = pCtx->rflags.u;
669
670 if (fExtrn & CPUMCTX_EXTRN_RAX)
671 pRun->s.regs.regs.rax = pCtx->rax;
672 if (fExtrn & CPUMCTX_EXTRN_RCX)
673 pRun->s.regs.regs.rcx = pCtx->rcx;
674 if (fExtrn & CPUMCTX_EXTRN_RDX)
675 pRun->s.regs.regs.rdx = pCtx->rdx;
676 if (fExtrn & CPUMCTX_EXTRN_RBX)
677 pRun->s.regs.regs.rbx = pCtx->rbx;
678 if (fExtrn & CPUMCTX_EXTRN_RSP)
679 pRun->s.regs.regs.rsp = pCtx->rsp;
680 if (fExtrn & CPUMCTX_EXTRN_RBP)
681 pRun->s.regs.regs.rbp = pCtx->rbp;
682 if (fExtrn & CPUMCTX_EXTRN_RSI)
683 pRun->s.regs.regs.rsi = pCtx->rsi;
684 if (fExtrn & CPUMCTX_EXTRN_RDI)
685 pRun->s.regs.regs.rdi = pCtx->rdi;
686 if (fExtrn & CPUMCTX_EXTRN_R8_R15)
687 {
688 pRun->s.regs.regs.r8 = pCtx->r8;
689 pRun->s.regs.regs.r9 = pCtx->r9;
690 pRun->s.regs.regs.r10 = pCtx->r10;
691 pRun->s.regs.regs.r11 = pCtx->r11;
692 pRun->s.regs.regs.r12 = pCtx->r12;
693 pRun->s.regs.regs.r13 = pCtx->r13;
694 pRun->s.regs.regs.r14 = pCtx->r14;
695 pRun->s.regs.regs.r15 = pCtx->r15;
696 }
697 pRun->kvm_dirty_regs |= KVM_SYNC_X86_REGS;
698 }
699
700 /*
701 * Stuff that goes into kvm_run::s.regs.sregs:
702 *
703 * The APIC base register updating is a little suboptimal... But at least
704 * VBox always has the right base register value, so it's one directional.
705 */
706 uint64_t const uApicBase = APICGetBaseMsrNoCheck(pVCpu);
707 if ( (fExtrn & ( CPUMCTX_EXTRN_SREG_MASK | CPUMCTX_EXTRN_TABLE_MASK | CPUMCTX_EXTRN_CR_MASK
708 | CPUMCTX_EXTRN_EFER | CPUMCTX_EXTRN_APIC_TPR))
709 || uApicBase != pVCpu->nem.s.uKvmApicBase)
710 {
711 if ((pVCpu->nem.s.uKvmApicBase ^ uApicBase) & MSR_IA32_APICBASE_EN)
712 Log(("NEM/%u: APICBASE_EN changed %#010RX64 -> %#010RX64\n", pVCpu->idCpu, pVCpu->nem.s.uKvmApicBase, uApicBase));
713 pRun->s.regs.sregs.apic_base = uApicBase;
714 pVCpu->nem.s.uKvmApicBase = uApicBase;
715
716 if (fExtrn & CPUMCTX_EXTRN_APIC_TPR)
717 pRun->s.regs.sregs.cr8 = CPUMGetGuestCR8(pVCpu);
718
719#define NEM_LNX_EXPORT_SEG(a_KvmSeg, a_CtxSeg) do { \
720 (a_KvmSeg).base = (a_CtxSeg).u64Base; \
721 (a_KvmSeg).limit = (a_CtxSeg).u32Limit; \
722 (a_KvmSeg).selector = (a_CtxSeg).Sel; \
723 (a_KvmSeg).type = (a_CtxSeg).Attr.n.u4Type; \
724 (a_KvmSeg).s = (a_CtxSeg).Attr.n.u1DescType; \
725 (a_KvmSeg).dpl = (a_CtxSeg).Attr.n.u2Dpl; \
726 (a_KvmSeg).present = (a_CtxSeg).Attr.n.u1Present; \
727 (a_KvmSeg).avl = (a_CtxSeg).Attr.n.u1Available; \
728 (a_KvmSeg).l = (a_CtxSeg).Attr.n.u1Long; \
729 (a_KvmSeg).db = (a_CtxSeg).Attr.n.u1DefBig; \
730 (a_KvmSeg).g = (a_CtxSeg).Attr.n.u1Granularity; \
731 (a_KvmSeg).unusable = (a_CtxSeg).Attr.n.u1Unusable; \
732 (a_KvmSeg).padding = 0; \
733 } while (0)
734
735 if (fExtrn & CPUMCTX_EXTRN_SREG_MASK)
736 {
737 if (fExtrn & CPUMCTX_EXTRN_ES)
738 NEM_LNX_EXPORT_SEG(pRun->s.regs.sregs.es, pCtx->es);
739 if (fExtrn & CPUMCTX_EXTRN_CS)
740 NEM_LNX_EXPORT_SEG(pRun->s.regs.sregs.cs, pCtx->cs);
741 if (fExtrn & CPUMCTX_EXTRN_SS)
742 NEM_LNX_EXPORT_SEG(pRun->s.regs.sregs.ss, pCtx->ss);
743 if (fExtrn & CPUMCTX_EXTRN_DS)
744 NEM_LNX_EXPORT_SEG(pRun->s.regs.sregs.ds, pCtx->ds);
745 if (fExtrn & CPUMCTX_EXTRN_FS)
746 NEM_LNX_EXPORT_SEG(pRun->s.regs.sregs.fs, pCtx->fs);
747 if (fExtrn & CPUMCTX_EXTRN_GS)
748 NEM_LNX_EXPORT_SEG(pRun->s.regs.sregs.gs, pCtx->gs);
749 }
750 if (fExtrn & CPUMCTX_EXTRN_TABLE_MASK)
751 {
752 if (fExtrn & CPUMCTX_EXTRN_GDTR)
753 {
754 pRun->s.regs.sregs.gdt.base = pCtx->gdtr.pGdt;
755 pRun->s.regs.sregs.gdt.limit = pCtx->gdtr.cbGdt;
756 pRun->s.regs.sregs.gdt.padding[0] = 0;
757 pRun->s.regs.sregs.gdt.padding[1] = 0;
758 pRun->s.regs.sregs.gdt.padding[2] = 0;
759 }
760 if (fExtrn & CPUMCTX_EXTRN_IDTR)
761 {
762 pRun->s.regs.sregs.idt.base = pCtx->idtr.pIdt;
763 pRun->s.regs.sregs.idt.limit = pCtx->idtr.cbIdt;
764 pRun->s.regs.sregs.idt.padding[0] = 0;
765 pRun->s.regs.sregs.idt.padding[1] = 0;
766 pRun->s.regs.sregs.idt.padding[2] = 0;
767 }
768 if (fExtrn & CPUMCTX_EXTRN_LDTR)
769 NEM_LNX_EXPORT_SEG(pRun->s.regs.sregs.ldt, pCtx->ldtr);
770 if (fExtrn & CPUMCTX_EXTRN_TR)
771 NEM_LNX_EXPORT_SEG(pRun->s.regs.sregs.tr, pCtx->tr);
772 }
773 if (fExtrn & CPUMCTX_EXTRN_CR_MASK)
774 {
775 if (fExtrn & CPUMCTX_EXTRN_CR0)
776 pRun->s.regs.sregs.cr0 = pCtx->cr0;
777 if (fExtrn & CPUMCTX_EXTRN_CR2)
778 pRun->s.regs.sregs.cr2 = pCtx->cr2;
779 if (fExtrn & CPUMCTX_EXTRN_CR3)
780 pRun->s.regs.sregs.cr3 = pCtx->cr3;
781 if (fExtrn & CPUMCTX_EXTRN_CR4)
782 pRun->s.regs.sregs.cr4 = pCtx->cr4;
783 }
784 if (fExtrn & CPUMCTX_EXTRN_EFER)
785 pRun->s.regs.sregs.efer = pCtx->msrEFER;
786
787 RT_ZERO(pRun->s.regs.sregs.interrupt_bitmap); /* this is an alternative interrupt injection interface */
788
789 pRun->kvm_dirty_regs |= KVM_SYNC_X86_SREGS;
790 }
791
792 /*
793 * Debug registers.
794 */
795 if (fExtrn & CPUMCTX_EXTRN_DR_MASK)
796 {
797 struct kvm_debugregs DbgRegs = {{0}};
798
799 if ((fExtrn & CPUMCTX_EXTRN_DR_MASK) != CPUMCTX_EXTRN_DR_MASK)
800 {
801 /* Partial debug state, we must get DbgRegs first so we can merge: */
802 int rc = ioctl(pVCpu->nem.s.fdVCpu, KVM_GET_DEBUGREGS, &DbgRegs);
803 AssertMsgReturn(rc == 0, ("rc=%d errno=%d\n", rc, errno), VERR_NEM_IPE_3);
804 }
805
806 if (fExtrn & CPUMCTX_EXTRN_DR0_DR3)
807 {
808 DbgRegs.db[0] = pCtx->dr[0];
809 DbgRegs.db[1] = pCtx->dr[1];
810 DbgRegs.db[2] = pCtx->dr[2];
811 DbgRegs.db[3] = pCtx->dr[3];
812 }
813 if (fExtrn & CPUMCTX_EXTRN_DR6)
814 DbgRegs.dr6 = pCtx->dr[6];
815 if (fExtrn & CPUMCTX_EXTRN_DR7)
816 DbgRegs.dr7 = pCtx->dr[7];
817
818 int rc = ioctl(pVCpu->nem.s.fdVCpu, KVM_SET_DEBUGREGS, &DbgRegs);
819 AssertMsgReturn(rc == 0, ("rc=%d errno=%d\n", rc, errno), VERR_NEM_IPE_3);
820 }
821
822 /*
823 * FPU, SSE, AVX, ++.
824 */
825 if (fExtrn & (CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX | CPUMCTX_EXTRN_OTHER_XSAVE | CPUMCTX_EXTRN_XCRx))
826 {
827 if (fExtrn & (CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX | CPUMCTX_EXTRN_OTHER_XSAVE))
828 {
829 /** @todo could IEM just grab state partial control in some situations? */
830 Assert( (fExtrn & (CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX | CPUMCTX_EXTRN_OTHER_XSAVE))
831 == (CPUMCTX_EXTRN_X87 | CPUMCTX_EXTRN_SSE_AVX | CPUMCTX_EXTRN_OTHER_XSAVE)); /* no partial states */
832
833 AssertCompile(sizeof(pCtx->XState) >= sizeof(struct kvm_xsave));
834 int rc = ioctl(pVCpu->nem.s.fdVCpu, KVM_SET_XSAVE, &pCtx->XState);
835 AssertMsgReturn(rc == 0, ("rc=%d errno=%d\n", rc, errno), VERR_NEM_IPE_3);
836 }
837
838 if (fExtrn & CPUMCTX_EXTRN_XCRx)
839 {
840 struct kvm_xcrs Xcrs =
841 { /*.nr_xcrs = */ 2,
842 /*.flags = */ 0,
843 /*.xcrs= */ {
844 { /*.xcr =*/ 0, /*.reserved=*/ 0, /*.value=*/ pCtx->aXcr[0] },
845 { /*.xcr =*/ 1, /*.reserved=*/ 0, /*.value=*/ pCtx->aXcr[1] },
846 }
847 };
848
849 int rc = ioctl(pVCpu->nem.s.fdVCpu, KVM_SET_XCRS, &Xcrs);
850 AssertMsgReturn(rc == 0, ("rc=%d errno=%d\n", rc, errno), VERR_NEM_IPE_3);
851 }
852 }
853
854 /*
855 * MSRs.
856 */
857 if (fExtrn & ( CPUMCTX_EXTRN_KERNEL_GS_BASE | CPUMCTX_EXTRN_SYSCALL_MSRS | CPUMCTX_EXTRN_SYSENTER_MSRS
858 | CPUMCTX_EXTRN_TSC_AUX | CPUMCTX_EXTRN_OTHER_MSRS))
859 {
860 union
861 {
862 struct kvm_msrs Core;
863 uint64_t padding[2 + sizeof(struct kvm_msr_entry) * 32];
864 } uBuf;
865 uint32_t iMsr = 0;
866 PCPUMCTXMSRS const pCtxMsrs = CPUMQueryGuestCtxMsrsPtr(pVCpu);
867
868#define ADD_MSR(a_Msr, a_uValue) do { \
869 Assert(iMsr < 32); \
870 uBuf.Core.entries[iMsr].index = (a_Msr); \
871 uBuf.Core.entries[iMsr].reserved = 0; \
872 uBuf.Core.entries[iMsr].data = (a_uValue); \
873 iMsr += 1; \
874 } while (0)
875
876 if (fExtrn & CPUMCTX_EXTRN_KERNEL_GS_BASE)
877 ADD_MSR(MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE);
878 if (fExtrn & CPUMCTX_EXTRN_SYSCALL_MSRS)
879 {
880 ADD_MSR(MSR_K6_STAR, pCtx->msrSTAR);
881 ADD_MSR(MSR_K8_LSTAR, pCtx->msrLSTAR);
882 ADD_MSR(MSR_K8_CSTAR, pCtx->msrCSTAR);
883 ADD_MSR(MSR_K8_SF_MASK, pCtx->msrSFMASK);
884 }
885 if (fExtrn & CPUMCTX_EXTRN_SYSENTER_MSRS)
886 {
887 ADD_MSR(MSR_IA32_SYSENTER_CS, pCtx->SysEnter.cs);
888 ADD_MSR(MSR_IA32_SYSENTER_EIP, pCtx->SysEnter.eip);
889 ADD_MSR(MSR_IA32_SYSENTER_ESP, pCtx->SysEnter.esp);
890 }
891 if (fExtrn & CPUMCTX_EXTRN_TSC_AUX)
892 ADD_MSR(MSR_K8_TSC_AUX, pCtxMsrs->msr.TscAux);
893 if (fExtrn & CPUMCTX_EXTRN_OTHER_MSRS)
894 {
895 ADD_MSR(MSR_IA32_CR_PAT, pCtx->msrPAT);
896 /** @todo What do we _have_ to add here?
897 * We also have: Mttr*, MiscEnable, FeatureControl. */
898 }
899
900 uBuf.Core.pad = 0;
901 uBuf.Core.nmsrs = iMsr;
902 int rc = ioctl(pVCpu->nem.s.fdVCpu, KVM_SET_MSRS, &uBuf);
903 AssertMsgReturn(rc == (int)iMsr,
904 ("rc=%d iMsr=%d (->%#x) errno=%d\n",
905 rc, iMsr, (uint32_t)rc < iMsr ? uBuf.Core.entries[rc].index : 0, errno),
906 VERR_NEM_IPE_3);
907 }
908
909 /*
910 * Interruptibility state.
911 *
912 * Note! This I/O control function sets most fields passed in, so when
913 * raising an interrupt, NMI, SMI or exception, this must be done
914 * by the code doing the rasing or we'll overwrite it here.
915 */
916 if (fExtrn & (CPUMCTX_EXTRN_INHIBIT_INT | CPUMCTX_EXTRN_INHIBIT_NMI))
917 {
918 Assert( (fExtrn & (CPUMCTX_EXTRN_INHIBIT_INT | CPUMCTX_EXTRN_INHIBIT_NMI))
919 == (CPUMCTX_EXTRN_INHIBIT_INT | CPUMCTX_EXTRN_INHIBIT_NMI));
920
921 struct kvm_vcpu_events KvmEvents = {0};
922
923 KvmEvents.flags = KVM_VCPUEVENT_VALID_SHADOW;
924 if (!CPUMIsInInterruptShadowWithUpdate(&pVCpu->cpum.GstCtx))
925 { /* probably likely */ }
926 else
927 KvmEvents.interrupt.shadow = (CPUMIsInInterruptShadowAfterSs(&pVCpu->cpum.GstCtx) ? KVM_X86_SHADOW_INT_MOV_SS : 0)
928 | (CPUMIsInInterruptShadowAfterSti(&pVCpu->cpum.GstCtx) ? KVM_X86_SHADOW_INT_STI : 0);
929
930 /* No flag - this is updated unconditionally. */
931 KvmEvents.nmi.masked = CPUMAreInterruptsInhibitedByNmi(&pVCpu->cpum.GstCtx);
932
933 if (TRPMHasTrap(pVCpu))
934 {
935 TRPMEVENT enmType = TRPM_32BIT_HACK;
936 uint8_t bTrapNo = 0;
937 TRPMQueryTrap(pVCpu, &bTrapNo, &enmType);
938 Log(("nemHCLnxExportState: Pending trap: bTrapNo=%#x enmType=%d\n", bTrapNo, enmType));
939 if ( enmType == TRPM_HARDWARE_INT
940 || enmType == TRPM_SOFTWARE_INT)
941 {
942 KvmEvents.interrupt.soft = enmType == TRPM_SOFTWARE_INT;
943 KvmEvents.interrupt.nr = bTrapNo;
944 KvmEvents.interrupt.injected = 1;
945 STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExportPendingInterrupt);
946 TRPMResetTrap(pVCpu);
947 }
948 else
949 AssertFailed();
950 }
951
952 int rcLnx = ioctl(pVCpu->nem.s.fdVCpu, KVM_SET_VCPU_EVENTS, &KvmEvents);
953 AssertLogRelMsgReturn(rcLnx == 0, ("rcLnx=%d errno=%d\n", rcLnx, errno), VERR_NEM_IPE_3);
954 }
955
956 /*
957 * KVM now owns all the state.
958 */
959 pCtx->fExtrn = CPUMCTX_EXTRN_KEEPER_NEM | CPUMCTX_EXTRN_ALL;
960
961 RT_NOREF(pVM);
962 return VINF_SUCCESS;
963}
964
965
966/**
967 * Query the CPU tick counter and optionally the TSC_AUX MSR value.
968 *
969 * @returns VBox status code.
970 * @param pVCpu The cross context CPU structure.
971 * @param pcTicks Where to return the CPU tick count.
972 * @param puAux Where to return the TSC_AUX register value.
973 */
974VMM_INT_DECL(int) NEMHCQueryCpuTick(PVMCPUCC pVCpu, uint64_t *pcTicks, uint32_t *puAux)
975{
976 STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatQueryCpuTick);
977 // KVM_GET_CLOCK?
978 RT_NOREF(pVCpu, pcTicks, puAux);
979 return VINF_SUCCESS;
980}
981
982
983/**
984 * Resumes CPU clock (TSC) on all virtual CPUs.
985 *
986 * This is called by TM when the VM is started, restored, resumed or similar.
987 *
988 * @returns VBox status code.
989 * @param pVM The cross context VM structure.
990 * @param pVCpu The cross context CPU structure of the calling EMT.
991 * @param uPausedTscValue The TSC value at the time of pausing.
992 */
993VMM_INT_DECL(int) NEMHCResumeCpuTickOnAll(PVMCC pVM, PVMCPUCC pVCpu, uint64_t uPausedTscValue)
994{
995 // KVM_SET_CLOCK?
996 RT_NOREF(pVM, pVCpu, uPausedTscValue);
997 return VINF_SUCCESS;
998}
999
1000
1001VMM_INT_DECL(uint32_t) NEMHCGetFeatures(PVMCC pVM)
1002{
1003 RT_NOREF(pVM);
1004 return NEM_FEAT_F_NESTED_PAGING
1005 | NEM_FEAT_F_FULL_GST_EXEC
1006 | NEM_FEAT_F_XSAVE_XRSTOR;
1007}
1008
1009
1010
1011/*********************************************************************************************************************************
1012* Execution *
1013*********************************************************************************************************************************/
1014
1015
1016VMMR3_INT_DECL(bool) NEMR3CanExecuteGuest(PVM pVM, PVMCPU pVCpu)
1017{
1018 /*
1019 * Only execute when the A20 gate is enabled as I cannot immediately
1020 * spot any A20 support in KVM.
1021 */
1022 RT_NOREF(pVM);
1023 Assert(VM_IS_NEM_ENABLED(pVM));
1024 return PGMPhysIsA20Enabled(pVCpu);
1025}
1026
1027
1028bool nemR3NativeSetSingleInstruction(PVM pVM, PVMCPU pVCpu, bool fEnable)
1029{
1030 NOREF(pVM); NOREF(pVCpu); NOREF(fEnable);
1031 return false;
1032}
1033
1034
1035void nemR3NativeNotifyFF(PVM pVM, PVMCPU pVCpu, uint32_t fFlags)
1036{
1037 int rc = RTThreadPoke(pVCpu->hThread);
1038 LogFlow(("nemR3NativeNotifyFF: #%u -> %Rrc\n", pVCpu->idCpu, rc));
1039 AssertRC(rc);
1040 RT_NOREF(pVM, fFlags);
1041}
1042
1043
1044DECLHIDDEN(bool) nemR3NativeNotifyDebugEventChanged(PVM pVM, bool fUseDebugLoop)
1045{
1046 RT_NOREF(pVM, fUseDebugLoop);
1047 return false;
1048}
1049
1050
1051DECLHIDDEN(bool) nemR3NativeNotifyDebugEventChangedPerCpu(PVM pVM, PVMCPU pVCpu, bool fUseDebugLoop)
1052{
1053 RT_NOREF(pVM, pVCpu, fUseDebugLoop);
1054 return false;
1055}
1056
1057
1058/**
1059 * Deals with pending interrupt FFs prior to executing guest code.
1060 */
1061static VBOXSTRICTRC nemHCLnxHandleInterruptFF(PVM pVM, PVMCPU pVCpu, struct kvm_run *pRun)
1062{
1063 RT_NOREF_PV(pVM);
1064
1065 /*
1066 * Do not doing anything if TRPM has something pending already as we can
1067 * only inject one event per KVM_RUN call. This can only happend if we
1068 * can directly from the loop in EM, so the inhibit bits must be internal.
1069 */
1070 if (!TRPMHasTrap(pVCpu))
1071 { /* semi likely */ }
1072 else
1073 {
1074 Assert(!(pVCpu->cpum.GstCtx.fExtrn & (CPUMCTX_EXTRN_INHIBIT_INT | CPUMCTX_EXTRN_INHIBIT_NMI)));
1075 Log8(("nemHCLnxHandleInterruptFF: TRPM has an pending event already\n"));
1076 return VINF_SUCCESS;
1077 }
1078
1079 /*
1080 * First update APIC. We ASSUME this won't need TPR/CR8.
1081 */
1082 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
1083 {
1084 APICUpdatePendingInterrupts(pVCpu);
1085 if (!VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC
1086 | VMCPU_FF_INTERRUPT_NMI | VMCPU_FF_INTERRUPT_SMI))
1087 return VINF_SUCCESS;
1088 }
1089
1090 /*
1091 * We don't currently implement SMIs.
1092 */
1093 AssertReturn(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_SMI), VERR_NEM_IPE_0);
1094
1095 /*
1096 * In KVM the CPUMCTX_EXTRN_INHIBIT_INT and CPUMCTX_EXTRN_INHIBIT_NMI states
1097 * are tied together with interrupt and NMI delivery, so we must get and
1098 * synchronize these all in one go and set both CPUMCTX_EXTRN_INHIBIT_XXX flags.
1099 * If we don't we may lose the interrupt/NMI we marked pending here when the
1100 * state is exported again before execution.
1101 */
1102 struct kvm_vcpu_events KvmEvents = {0};
1103 int rcLnx = ioctl(pVCpu->nem.s.fdVCpu, KVM_GET_VCPU_EVENTS, &KvmEvents);
1104 AssertLogRelMsgReturn(rcLnx == 0, ("rcLnx=%d errno=%d\n", rcLnx, errno), VERR_NEM_IPE_5);
1105
1106 if (!(pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_RIP))
1107 pRun->s.regs.regs.rip = pVCpu->cpum.GstCtx.rip;
1108
1109 KvmEvents.flags |= KVM_VCPUEVENT_VALID_SHADOW;
1110 if (!(pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_INHIBIT_INT))
1111 KvmEvents.interrupt.shadow = !CPUMIsInInterruptShadowWithUpdate(&pVCpu->cpum.GstCtx) ? 0
1112 : (CPUMIsInInterruptShadowAfterSs(&pVCpu->cpum.GstCtx) ? KVM_X86_SHADOW_INT_MOV_SS : 0)
1113 | (CPUMIsInInterruptShadowAfterSti(&pVCpu->cpum.GstCtx) ? KVM_X86_SHADOW_INT_STI : 0);
1114 else
1115 CPUMUpdateInterruptShadowSsStiEx(&pVCpu->cpum.GstCtx,
1116 RT_BOOL(KvmEvents.interrupt.shadow & KVM_X86_SHADOW_INT_MOV_SS),
1117 RT_BOOL(KvmEvents.interrupt.shadow & KVM_X86_SHADOW_INT_STI),
1118 pRun->s.regs.regs.rip);
1119
1120 if (!(pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_INHIBIT_NMI))
1121 KvmEvents.nmi.masked = CPUMAreInterruptsInhibitedByNmi(&pVCpu->cpum.GstCtx);
1122 else
1123 CPUMUpdateInterruptInhibitingByNmi(&pVCpu->cpum.GstCtx, KvmEvents.nmi.masked != 0);
1124
1125 /* KVM will own the INT + NMI inhibit state soon: */
1126 pVCpu->cpum.GstCtx.fExtrn = (pVCpu->cpum.GstCtx.fExtrn & ~CPUMCTX_EXTRN_KEEPER_MASK)
1127 | CPUMCTX_EXTRN_KEEPER_NEM | CPUMCTX_EXTRN_INHIBIT_INT | CPUMCTX_EXTRN_INHIBIT_NMI;
1128
1129 /*
1130 * NMI? Try deliver it first.
1131 */
1132 if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI))
1133 {
1134#if 0
1135 int rcLnx = ioctl(pVCpu->nem.s.fdVm, KVM_NMI, 0UL);
1136 AssertLogRelMsgReturn(rcLnx == 0, ("rcLnx=%d errno=%d\n", rcLnx, errno), VERR_NEM_IPE_5);
1137#else
1138 KvmEvents.flags |= KVM_VCPUEVENT_VALID_NMI_PENDING;
1139 KvmEvents.nmi.pending = 1;
1140#endif
1141 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
1142 Log8(("Queuing NMI on %u\n", pVCpu->idCpu));
1143 }
1144
1145 /*
1146 * APIC or PIC interrupt?
1147 */
1148 if (VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
1149 {
1150 if (pRun->s.regs.regs.rflags & X86_EFL_IF)
1151 {
1152 if (KvmEvents.interrupt.shadow == 0)
1153 {
1154 /*
1155 * If CR8 is in KVM, update the VBox copy so PDMGetInterrupt will
1156 * work correctly.
1157 */
1158 if (pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_APIC_TPR)
1159 APICSetTpr(pVCpu, (uint8_t)pRun->cr8 << 4);
1160
1161 uint8_t bInterrupt;
1162 int rc = PDMGetInterrupt(pVCpu, &bInterrupt);
1163 if (RT_SUCCESS(rc))
1164 {
1165 Assert(KvmEvents.interrupt.injected == false);
1166#if 0
1167 int rcLnx = ioctl(pVCpu->nem.s.fdVm, KVM_INTERRUPT, (unsigned long)bInterrupt);
1168 AssertLogRelMsgReturn(rcLnx == 0, ("rcLnx=%d errno=%d\n", rcLnx, errno), VERR_NEM_IPE_5);
1169#else
1170 KvmEvents.interrupt.nr = bInterrupt;
1171 KvmEvents.interrupt.soft = false;
1172 KvmEvents.interrupt.injected = true;
1173#endif
1174 Log8(("Queuing interrupt %#x on %u: %04x:%08RX64 efl=%#x\n", bInterrupt, pVCpu->idCpu,
1175 pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.eflags.u));
1176 }
1177 else if (rc == VERR_APIC_INTR_MASKED_BY_TPR) /** @todo this isn't extremely efficient if we get a lot of exits... */
1178 Log8(("VERR_APIC_INTR_MASKED_BY_TPR\n")); /* We'll get a TRP exit - no interrupt window needed. */
1179 else
1180 Log8(("PDMGetInterrupt failed -> %Rrc\n", rc));
1181 }
1182 else
1183 {
1184 pRun->request_interrupt_window = 1;
1185 Log8(("Interrupt window pending on %u (#2)\n", pVCpu->idCpu));
1186 }
1187 }
1188 else
1189 {
1190 pRun->request_interrupt_window = 1;
1191 Log8(("Interrupt window pending on %u (#1)\n", pVCpu->idCpu));
1192 }
1193 }
1194
1195 /*
1196 * Now, update the state.
1197 */
1198 /** @todo skip when possible... */
1199 rcLnx = ioctl(pVCpu->nem.s.fdVCpu, KVM_SET_VCPU_EVENTS, &KvmEvents);
1200 AssertLogRelMsgReturn(rcLnx == 0, ("rcLnx=%d errno=%d\n", rcLnx, errno), VERR_NEM_IPE_5);
1201
1202 return VINF_SUCCESS;
1203}
1204
1205
1206/**
1207 * Handles KVM_EXIT_INTERNAL_ERROR.
1208 */
1209static VBOXSTRICTRC nemR3LnxHandleInternalError(PVMCPU pVCpu, struct kvm_run *pRun)
1210{
1211 Log(("NEM: KVM_EXIT_INTERNAL_ERROR! suberror=%#x (%d) ndata=%u data=%.*Rhxs\n", pRun->internal.suberror,
1212 pRun->internal.suberror, pRun->internal.ndata, sizeof(pRun->internal.data), &pRun->internal.data[0]));
1213
1214 /*
1215 * Deal with each suberror, returning if we don't want IEM to handle it.
1216 */
1217 switch (pRun->internal.suberror)
1218 {
1219 case KVM_INTERNAL_ERROR_EMULATION:
1220 {
1221 EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_INTERNAL_ERROR_EMULATION),
1222 pRun->s.regs.regs.rip + pRun->s.regs.sregs.cs.base, ASMReadTSC());
1223 STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitInternalErrorEmulation);
1224 break;
1225 }
1226
1227 case KVM_INTERNAL_ERROR_SIMUL_EX:
1228 case KVM_INTERNAL_ERROR_DELIVERY_EV:
1229 case KVM_INTERNAL_ERROR_UNEXPECTED_EXIT_REASON:
1230 default:
1231 {
1232 EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_INTERNAL_ERROR_FATAL),
1233 pRun->s.regs.regs.rip + pRun->s.regs.sregs.cs.base, ASMReadTSC());
1234 STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitInternalErrorFatal);
1235 const char *pszName;
1236 switch (pRun->internal.suberror)
1237 {
1238 case KVM_INTERNAL_ERROR_EMULATION: pszName = "KVM_INTERNAL_ERROR_EMULATION"; break;
1239 case KVM_INTERNAL_ERROR_SIMUL_EX: pszName = "KVM_INTERNAL_ERROR_SIMUL_EX"; break;
1240 case KVM_INTERNAL_ERROR_DELIVERY_EV: pszName = "KVM_INTERNAL_ERROR_DELIVERY_EV"; break;
1241 case KVM_INTERNAL_ERROR_UNEXPECTED_EXIT_REASON: pszName = "KVM_INTERNAL_ERROR_UNEXPECTED_EXIT_REASON"; break;
1242 default: pszName = "unknown"; break;
1243 }
1244 LogRel(("NEM: KVM_EXIT_INTERNAL_ERROR! suberror=%#x (%s) ndata=%u data=%.*Rhxs\n", pRun->internal.suberror, pszName,
1245 pRun->internal.ndata, sizeof(pRun->internal.data), &pRun->internal.data[0]));
1246 return VERR_NEM_IPE_0;
1247 }
1248 }
1249
1250 /*
1251 * Execute instruction in IEM and try get on with it.
1252 */
1253 Log2(("nemR3LnxHandleInternalError: Executing instruction at %04x:%08RX64 in IEM\n",
1254 pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip));
1255 VBOXSTRICTRC rcStrict = nemHCLnxImportState(pVCpu,
1256 IEM_CPUMCTX_EXTRN_MUST_MASK | CPUMCTX_EXTRN_INHIBIT_INT
1257 | CPUMCTX_EXTRN_INHIBIT_NMI,
1258 &pVCpu->cpum.GstCtx, pRun);
1259 if (RT_SUCCESS(rcStrict))
1260 rcStrict = IEMExecOne(pVCpu);
1261 return rcStrict;
1262}
1263
1264
1265/**
1266 * Handles KVM_EXIT_IO.
1267 */
1268static VBOXSTRICTRC nemHCLnxHandleExitIo(PVMCC pVM, PVMCPUCC pVCpu, struct kvm_run *pRun)
1269{
1270 /*
1271 * Input validation.
1272 */
1273 Assert(pRun->io.count > 0);
1274 Assert(pRun->io.size == 1 || pRun->io.size == 2 || pRun->io.size == 4);
1275 Assert(pRun->io.direction == KVM_EXIT_IO_IN || pRun->io.direction == KVM_EXIT_IO_OUT);
1276 Assert(pRun->io.data_offset < pVM->nem.s.cbVCpuMmap);
1277 Assert(pRun->io.data_offset + pRun->io.size * pRun->io.count <= pVM->nem.s.cbVCpuMmap);
1278
1279 /*
1280 * We cannot easily act on the exit history here, because the I/O port
1281 * exit is stateful and the instruction will be completed in the next
1282 * KVM_RUN call. There seems no way to avoid this.
1283 */
1284 EMHistoryAddExit(pVCpu,
1285 pRun->io.count == 1
1286 ? ( pRun->io.direction == KVM_EXIT_IO_IN
1287 ? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_IO_PORT_READ)
1288 : EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_IO_PORT_WRITE))
1289 : ( pRun->io.direction == KVM_EXIT_IO_IN
1290 ? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_IO_PORT_STR_READ)
1291 : EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_IO_PORT_STR_WRITE)),
1292 pRun->s.regs.regs.rip + pRun->s.regs.sregs.cs.base, ASMReadTSC());
1293
1294 /*
1295 * Do the requested job.
1296 */
1297 VBOXSTRICTRC rcStrict;
1298 RTPTRUNION uPtrData;
1299 uPtrData.pu8 = (uint8_t *)pRun + pRun->io.data_offset;
1300 if (pRun->io.count == 1)
1301 {
1302 if (pRun->io.direction == KVM_EXIT_IO_IN)
1303 {
1304 uint32_t uValue = 0;
1305 rcStrict = IOMIOPortRead(pVM, pVCpu, pRun->io.port, &uValue, pRun->io.size);
1306 Log4(("IOExit/%u: %04x:%08RX64: IN %#x LB %u -> %#x, rcStrict=%Rrc\n",
1307 pVCpu->idCpu, pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip,
1308 pRun->io.port, pRun->io.size, uValue, VBOXSTRICTRC_VAL(rcStrict) ));
1309 if (IOM_SUCCESS(rcStrict))
1310 {
1311 if (pRun->io.size == 4)
1312 *uPtrData.pu32 = uValue;
1313 else if (pRun->io.size == 2)
1314 *uPtrData.pu16 = (uint16_t)uValue;
1315 else
1316 *uPtrData.pu8 = (uint8_t)uValue;
1317 }
1318 }
1319 else
1320 {
1321 uint32_t const uValue = pRun->io.size == 4 ? *uPtrData.pu32
1322 : pRun->io.size == 2 ? *uPtrData.pu16
1323 : *uPtrData.pu8;
1324 rcStrict = IOMIOPortWrite(pVM, pVCpu, pRun->io.port, uValue, pRun->io.size);
1325 Log4(("IOExit/%u: %04x:%08RX64: OUT %#x, %#x LB %u rcStrict=%Rrc\n",
1326 pVCpu->idCpu, pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip,
1327 pRun->io.port, uValue, pRun->io.size, VBOXSTRICTRC_VAL(rcStrict) ));
1328 }
1329 }
1330 else
1331 {
1332 uint32_t cTransfers = pRun->io.count;
1333 if (pRun->io.direction == KVM_EXIT_IO_IN)
1334 {
1335 rcStrict = IOMIOPortReadString(pVM, pVCpu, pRun->io.port, uPtrData.pv, &cTransfers, pRun->io.size);
1336 Log4(("IOExit/%u: %04x:%08RX64: REP INS %#x LB %u * %#x times -> rcStrict=%Rrc cTransfers=%d\n",
1337 pVCpu->idCpu, pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip,
1338 pRun->io.port, pRun->io.size, pRun->io.count, VBOXSTRICTRC_VAL(rcStrict), cTransfers ));
1339 }
1340 else
1341 {
1342 rcStrict = IOMIOPortWriteString(pVM, pVCpu, pRun->io.port, uPtrData.pv, &cTransfers, pRun->io.size);
1343 Log4(("IOExit/%u: %04x:%08RX64: REP OUTS %#x LB %u * %#x times -> rcStrict=%Rrc cTransfers=%d\n",
1344 pVCpu->idCpu, pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip,
1345 pRun->io.port, pRun->io.size, pRun->io.count, VBOXSTRICTRC_VAL(rcStrict), cTransfers ));
1346 }
1347 Assert(cTransfers == 0);
1348 }
1349 return rcStrict;
1350}
1351
1352
1353/**
1354 * Handles KVM_EXIT_MMIO.
1355 */
1356static VBOXSTRICTRC nemHCLnxHandleExitMmio(PVMCC pVM, PVMCPUCC pVCpu, struct kvm_run *pRun)
1357{
1358 /*
1359 * Input validation.
1360 */
1361 Assert(pRun->mmio.len <= sizeof(pRun->mmio.data));
1362 Assert(pRun->mmio.is_write <= 1);
1363
1364 /*
1365 * We cannot easily act on the exit history here, because the MMIO port
1366 * exit is stateful and the instruction will be completed in the next
1367 * KVM_RUN call. There seems no way to circumvent this.
1368 */
1369 EMHistoryAddExit(pVCpu,
1370 pRun->mmio.is_write
1371 ? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_MMIO_WRITE)
1372 : EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_MMIO_READ),
1373 pRun->s.regs.regs.rip + pRun->s.regs.sregs.cs.base, ASMReadTSC());
1374
1375 /*
1376 * Do the requested job.
1377 */
1378 VBOXSTRICTRC rcStrict;
1379 if (pRun->mmio.is_write)
1380 {
1381 rcStrict = PGMPhysWrite(pVM, pRun->mmio.phys_addr, pRun->mmio.data, pRun->mmio.len, PGMACCESSORIGIN_HM);
1382 Log4(("MmioExit/%u: %04x:%08RX64: WRITE %#x LB %u, %.*Rhxs -> rcStrict=%Rrc\n",
1383 pVCpu->idCpu, pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip,
1384 pRun->mmio.phys_addr, pRun->mmio.len, pRun->mmio.len, pRun->mmio.data, VBOXSTRICTRC_VAL(rcStrict) ));
1385 }
1386 else
1387 {
1388 rcStrict = PGMPhysRead(pVM, pRun->mmio.phys_addr, pRun->mmio.data, pRun->mmio.len, PGMACCESSORIGIN_HM);
1389 Log4(("MmioExit/%u: %04x:%08RX64: READ %#x LB %u -> %.*Rhxs rcStrict=%Rrc\n",
1390 pVCpu->idCpu, pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip,
1391 pRun->mmio.phys_addr, pRun->mmio.len, pRun->mmio.len, pRun->mmio.data, VBOXSTRICTRC_VAL(rcStrict) ));
1392 }
1393 return rcStrict;
1394}
1395
1396
1397/**
1398 * Handles KVM_EXIT_RDMSR
1399 */
1400static VBOXSTRICTRC nemHCLnxHandleExitRdMsr(PVMCPUCC pVCpu, struct kvm_run *pRun)
1401{
1402 /*
1403 * Input validation.
1404 */
1405 Assert( pRun->msr.reason == KVM_MSR_EXIT_REASON_INVAL
1406 || pRun->msr.reason == KVM_MSR_EXIT_REASON_UNKNOWN
1407 || pRun->msr.reason == KVM_MSR_EXIT_REASON_FILTER);
1408
1409 /*
1410 * We cannot easily act on the exit history here, because the MSR exit is
1411 * stateful and the instruction will be completed in the next KVM_RUN call.
1412 * There seems no way to circumvent this.
1413 */
1414 EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_MSR_READ),
1415 pRun->s.regs.regs.rip + pRun->s.regs.sregs.cs.base, ASMReadTSC());
1416
1417 /*
1418 * Do the requested job.
1419 */
1420 uint64_t uValue = 0;
1421 VBOXSTRICTRC rcStrict = CPUMQueryGuestMsr(pVCpu, pRun->msr.index, &uValue);
1422 pRun->msr.data = uValue;
1423 if (rcStrict != VERR_CPUM_RAISE_GP_0)
1424 {
1425 Log3(("MsrRead/%u: %04x:%08RX64: msr=%#010x (reason=%#x) -> %#RX64 rcStrict=%Rrc\n", pVCpu->idCpu,
1426 pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip, pRun->msr.index, pRun->msr.reason, uValue, VBOXSTRICTRC_VAL(rcStrict) ));
1427 pRun->msr.error = 0;
1428 }
1429 else
1430 {
1431 Log3(("MsrRead/%u: %04x:%08RX64: msr=%#010x (reason%#x)-> %#RX64 rcStrict=#GP!\n", pVCpu->idCpu,
1432 pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip, pRun->msr.index, pRun->msr.reason, uValue));
1433 pRun->msr.error = 1;
1434 rcStrict = VINF_SUCCESS;
1435 }
1436 return rcStrict;
1437}
1438
1439
1440/**
1441 * Handles KVM_EXIT_WRMSR
1442 */
1443static VBOXSTRICTRC nemHCLnxHandleExitWrMsr(PVMCPUCC pVCpu, struct kvm_run *pRun)
1444{
1445 /*
1446 * Input validation.
1447 */
1448 Assert( pRun->msr.reason == KVM_MSR_EXIT_REASON_INVAL
1449 || pRun->msr.reason == KVM_MSR_EXIT_REASON_UNKNOWN
1450 || pRun->msr.reason == KVM_MSR_EXIT_REASON_FILTER);
1451
1452 /*
1453 * We cannot easily act on the exit history here, because the MSR exit is
1454 * stateful and the instruction will be completed in the next KVM_RUN call.
1455 * There seems no way to circumvent this.
1456 */
1457 EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM, EMEXITTYPE_MSR_WRITE),
1458 pRun->s.regs.regs.rip + pRun->s.regs.sregs.cs.base, ASMReadTSC());
1459
1460 /*
1461 * Do the requested job.
1462 */
1463 VBOXSTRICTRC rcStrict = CPUMSetGuestMsr(pVCpu, pRun->msr.index, pRun->msr.data);
1464 if (rcStrict != VERR_CPUM_RAISE_GP_0)
1465 {
1466 Log3(("MsrWrite/%u: %04x:%08RX64: msr=%#010x := %#RX64 (reason=%#x) -> rcStrict=%Rrc\n", pVCpu->idCpu,
1467 pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip, pRun->msr.index, pRun->msr.data, pRun->msr.reason, VBOXSTRICTRC_VAL(rcStrict) ));
1468 pRun->msr.error = 0;
1469 }
1470 else
1471 {
1472 Log3(("MsrWrite/%u: %04x:%08RX64: msr=%#010x := %#RX64 (reason%#x)-> rcStrict=#GP!\n", pVCpu->idCpu,
1473 pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip, pRun->msr.index, pRun->msr.data, pRun->msr.reason));
1474 pRun->msr.error = 1;
1475 rcStrict = VINF_SUCCESS;
1476 }
1477 return rcStrict;
1478}
1479
1480
1481
1482static VBOXSTRICTRC nemHCLnxHandleExit(PVMCC pVM, PVMCPUCC pVCpu, struct kvm_run *pRun, bool *pfStatefulExit)
1483{
1484 STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitTotal);
1485 switch (pRun->exit_reason)
1486 {
1487 case KVM_EXIT_EXCEPTION:
1488 AssertFailed();
1489 break;
1490
1491 case KVM_EXIT_IO:
1492 STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitIo);
1493 *pfStatefulExit = true;
1494 return nemHCLnxHandleExitIo(pVM, pVCpu, pRun);
1495
1496 case KVM_EXIT_MMIO:
1497 STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitMmio);
1498 *pfStatefulExit = true;
1499 return nemHCLnxHandleExitMmio(pVM, pVCpu, pRun);
1500
1501 case KVM_EXIT_IRQ_WINDOW_OPEN:
1502 EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_INTTERRUPT_WINDOW),
1503 pRun->s.regs.regs.rip + pRun->s.regs.sregs.cs.base, ASMReadTSC());
1504 STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitIrqWindowOpen);
1505 Log5(("IrqWinOpen/%u: %d\n", pVCpu->idCpu, pRun->request_interrupt_window));
1506 pRun->request_interrupt_window = 0;
1507 return VINF_SUCCESS;
1508
1509 case KVM_EXIT_SET_TPR:
1510 STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitSetTpr);
1511 AssertFailed();
1512 break;
1513
1514 case KVM_EXIT_TPR_ACCESS:
1515 STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitTprAccess);
1516 AssertFailed();
1517 break;
1518
1519 case KVM_EXIT_X86_RDMSR:
1520 STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitRdMsr);
1521 *pfStatefulExit = true;
1522 return nemHCLnxHandleExitRdMsr(pVCpu, pRun);
1523
1524 case KVM_EXIT_X86_WRMSR:
1525 STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitWrMsr);
1526 *pfStatefulExit = true;
1527 return nemHCLnxHandleExitWrMsr(pVCpu, pRun);
1528
1529 case KVM_EXIT_HLT:
1530 EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_HALT),
1531 pRun->s.regs.regs.rip + pRun->s.regs.sregs.cs.base, ASMReadTSC());
1532 STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitHalt);
1533 Log5(("Halt/%u\n", pVCpu->idCpu));
1534 return VINF_EM_HALT;
1535
1536 case KVM_EXIT_INTR: /* EINTR */
1537 EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_INTERRUPTED),
1538 pRun->s.regs.regs.rip + pRun->s.regs.sregs.cs.base, ASMReadTSC());
1539 STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitIntr);
1540 Log5(("Intr/%u\n", pVCpu->idCpu));
1541 return VINF_SUCCESS;
1542
1543 case KVM_EXIT_HYPERCALL:
1544 STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitHypercall);
1545 AssertFailed();
1546 break;
1547
1548 case KVM_EXIT_DEBUG:
1549 STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitDebug);
1550 AssertFailed();
1551 break;
1552
1553 case KVM_EXIT_SYSTEM_EVENT:
1554 AssertFailed();
1555 break;
1556 case KVM_EXIT_IOAPIC_EOI:
1557 AssertFailed();
1558 break;
1559 case KVM_EXIT_HYPERV:
1560 AssertFailed();
1561 break;
1562
1563 case KVM_EXIT_DIRTY_RING_FULL:
1564 AssertFailed();
1565 break;
1566 case KVM_EXIT_AP_RESET_HOLD:
1567 AssertFailed();
1568 break;
1569 case KVM_EXIT_X86_BUS_LOCK:
1570 STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatExitBusLock);
1571 AssertFailed();
1572 break;
1573
1574
1575 case KVM_EXIT_SHUTDOWN:
1576 AssertFailed();
1577 break;
1578
1579 case KVM_EXIT_FAIL_ENTRY:
1580 LogRel(("NEM: KVM_EXIT_FAIL_ENTRY! hardware_entry_failure_reason=%#x cpu=%#x\n",
1581 pRun->fail_entry.hardware_entry_failure_reason, pRun->fail_entry.cpu));
1582 EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_NEM, NEMEXITTYPE_FAILED_ENTRY),
1583 pRun->s.regs.regs.rip + pRun->s.regs.sregs.cs.base, ASMReadTSC());
1584 return VERR_NEM_IPE_1;
1585
1586 case KVM_EXIT_INTERNAL_ERROR:
1587 /* we're counting sub-reasons inside the function. */
1588 return nemR3LnxHandleInternalError(pVCpu, pRun);
1589
1590 /*
1591 * Foreign and unknowns.
1592 */
1593 case KVM_EXIT_NMI:
1594 AssertLogRelMsgFailedReturn(("KVM_EXIT_NMI on VCpu #%u at %04x:%RX64!\n", pVCpu->idCpu, pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip), VERR_NEM_IPE_1);
1595 case KVM_EXIT_EPR:
1596 AssertLogRelMsgFailedReturn(("KVM_EXIT_EPR on VCpu #%u at %04x:%RX64!\n", pVCpu->idCpu, pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip), VERR_NEM_IPE_1);
1597 case KVM_EXIT_WATCHDOG:
1598 AssertLogRelMsgFailedReturn(("KVM_EXIT_WATCHDOG on VCpu #%u at %04x:%RX64!\n", pVCpu->idCpu, pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip), VERR_NEM_IPE_1);
1599 case KVM_EXIT_ARM_NISV:
1600 AssertLogRelMsgFailedReturn(("KVM_EXIT_ARM_NISV on VCpu #%u at %04x:%RX64!\n", pVCpu->idCpu, pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip), VERR_NEM_IPE_1);
1601 case KVM_EXIT_S390_STSI:
1602 AssertLogRelMsgFailedReturn(("KVM_EXIT_S390_STSI on VCpu #%u at %04x:%RX64!\n", pVCpu->idCpu, pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip), VERR_NEM_IPE_1);
1603 case KVM_EXIT_S390_TSCH:
1604 AssertLogRelMsgFailedReturn(("KVM_EXIT_S390_TSCH on VCpu #%u at %04x:%RX64!\n", pVCpu->idCpu, pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip), VERR_NEM_IPE_1);
1605 case KVM_EXIT_OSI:
1606 AssertLogRelMsgFailedReturn(("KVM_EXIT_OSI on VCpu #%u at %04x:%RX64!\n", pVCpu->idCpu, pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip), VERR_NEM_IPE_1);
1607 case KVM_EXIT_PAPR_HCALL:
1608 AssertLogRelMsgFailedReturn(("KVM_EXIT_PAPR_HCALL on VCpu #%u at %04x:%RX64!\n", pVCpu->idCpu, pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip), VERR_NEM_IPE_1);
1609 case KVM_EXIT_S390_UCONTROL:
1610 AssertLogRelMsgFailedReturn(("KVM_EXIT_S390_UCONTROL on VCpu #%u at %04x:%RX64!\n", pVCpu->idCpu, pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip), VERR_NEM_IPE_1);
1611 case KVM_EXIT_DCR:
1612 AssertLogRelMsgFailedReturn(("KVM_EXIT_DCR on VCpu #%u at %04x:%RX64!\n", pVCpu->idCpu, pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip), VERR_NEM_IPE_1);
1613 case KVM_EXIT_S390_SIEIC:
1614 AssertLogRelMsgFailedReturn(("KVM_EXIT_S390_SIEIC on VCpu #%u at %04x:%RX64!\n", pVCpu->idCpu, pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip), VERR_NEM_IPE_1);
1615 case KVM_EXIT_S390_RESET:
1616 AssertLogRelMsgFailedReturn(("KVM_EXIT_S390_RESET on VCpu #%u at %04x:%RX64!\n", pVCpu->idCpu, pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip), VERR_NEM_IPE_1);
1617 case KVM_EXIT_UNKNOWN:
1618 AssertLogRelMsgFailedReturn(("KVM_EXIT_UNKNOWN on VCpu #%u at %04x:%RX64!\n", pVCpu->idCpu, pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip), VERR_NEM_IPE_1);
1619 case KVM_EXIT_XEN:
1620 AssertLogRelMsgFailedReturn(("KVM_EXIT_XEN on VCpu #%u at %04x:%RX64!\n", pVCpu->idCpu, pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip), VERR_NEM_IPE_1);
1621 default:
1622 AssertLogRelMsgFailedReturn(("Unknown exit reason %u on VCpu #%u at %04x:%RX64!\n", pRun->exit_reason, pVCpu->idCpu, pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip), VERR_NEM_IPE_1);
1623 }
1624
1625 RT_NOREF(pVM, pVCpu, pRun);
1626 return VERR_NOT_IMPLEMENTED;
1627}
1628
1629
1630VBOXSTRICTRC nemR3NativeRunGC(PVM pVM, PVMCPU pVCpu)
1631{
1632 /*
1633 * Try switch to NEM runloop state.
1634 */
1635 if (VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC_NEM, VMCPUSTATE_STARTED))
1636 { /* likely */ }
1637 else
1638 {
1639 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC_NEM, VMCPUSTATE_STARTED_EXEC_NEM_CANCELED);
1640 LogFlow(("NEM/%u: returning immediately because canceled\n", pVCpu->idCpu));
1641 return VINF_SUCCESS;
1642 }
1643
1644 /*
1645 * The run loop.
1646 */
1647 struct kvm_run * const pRun = pVCpu->nem.s.pRun;
1648 const bool fSingleStepping = DBGFIsStepping(pVCpu);
1649 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
1650 bool fStatefulExit = false; /* For MMIO and IO exits. */
1651 for (unsigned iLoop = 0;; iLoop++)
1652 {
1653 /*
1654 * Pending interrupts or such? Need to check and deal with this prior
1655 * to the state syncing.
1656 */
1657 if (VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_UPDATE_APIC | VMCPU_FF_INTERRUPT_PIC
1658 | VMCPU_FF_INTERRUPT_NMI | VMCPU_FF_INTERRUPT_SMI))
1659 {
1660 /* Try inject interrupt. */
1661 rcStrict = nemHCLnxHandleInterruptFF(pVM, pVCpu, pRun);
1662 if (rcStrict == VINF_SUCCESS)
1663 { /* likely */ }
1664 else
1665 {
1666 LogFlow(("NEM/%u: breaking: nemHCLnxHandleInterruptFF -> %Rrc\n", pVCpu->idCpu, VBOXSTRICTRC_VAL(rcStrict) ));
1667 STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatBreakOnStatus);
1668 break;
1669 }
1670 }
1671
1672 /*
1673 * Do not execute in KVM if the A20 isn't enabled.
1674 */
1675 if (PGMPhysIsA20Enabled(pVCpu))
1676 { /* likely */ }
1677 else
1678 {
1679 rcStrict = VINF_EM_RESCHEDULE_REM;
1680 LogFlow(("NEM/%u: breaking: A20 disabled\n", pVCpu->idCpu));
1681 break;
1682 }
1683
1684 /*
1685 * Ensure KVM has the whole state.
1686 */
1687 if ((pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_ALL) != CPUMCTX_EXTRN_ALL)
1688 {
1689 int rc2 = nemHCLnxExportState(pVM, pVCpu, &pVCpu->cpum.GstCtx, pRun);
1690 AssertRCReturn(rc2, rc2);
1691 }
1692
1693 /*
1694 * Poll timers and run for a bit.
1695 *
1696 * With the VID approach (ring-0 or ring-3) we can specify a timeout here,
1697 * so we take the time of the next timer event and uses that as a deadline.
1698 * The rounding heuristics are "tuned" so that rhel5 (1K timer) will boot fine.
1699 */
1700 /** @todo See if we cannot optimize this TMTimerPollGIP by only redoing
1701 * the whole polling job when timers have changed... */
1702 uint64_t offDeltaIgnored;
1703 uint64_t const nsNextTimerEvt = TMTimerPollGIP(pVM, pVCpu, &offDeltaIgnored); NOREF(nsNextTimerEvt);
1704 if ( !VM_FF_IS_ANY_SET(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
1705 && !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
1706 {
1707 if (VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC_NEM_WAIT, VMCPUSTATE_STARTED_EXEC_NEM))
1708 {
1709 LogFlow(("NEM/%u: Entry @ %04x:%08RX64 IF=%d EFL=%#RX64 SS:RSP=%04x:%08RX64 cr0=%RX64\n",
1710 pVCpu->idCpu, pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip,
1711 !!(pRun->s.regs.regs.rflags & X86_EFL_IF), pRun->s.regs.regs.rflags,
1712 pRun->s.regs.sregs.ss.selector, pRun->s.regs.regs.rsp, pRun->s.regs.sregs.cr0));
1713 TMNotifyStartOfExecution(pVM, pVCpu);
1714
1715 int rcLnx = ioctl(pVCpu->nem.s.fdVCpu, KVM_RUN, 0UL);
1716
1717 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC_NEM, VMCPUSTATE_STARTED_EXEC_NEM_WAIT);
1718 TMNotifyEndOfExecution(pVM, pVCpu, ASMReadTSC());
1719
1720#ifdef LOG_ENABLED
1721 if (LogIsFlowEnabled())
1722 {
1723 struct kvm_mp_state MpState = {UINT32_MAX};
1724 ioctl(pVCpu->nem.s.fdVCpu, KVM_GET_MP_STATE, &MpState);
1725 LogFlow(("NEM/%u: Exit @ %04x:%08RX64 IF=%d EFL=%#RX64 CR8=%#x Reason=%#x IrqReady=%d Flags=%#x %#lx\n", pVCpu->idCpu,
1726 pRun->s.regs.sregs.cs.selector, pRun->s.regs.regs.rip, pRun->if_flag,
1727 pRun->s.regs.regs.rflags, pRun->s.regs.sregs.cr8, pRun->exit_reason,
1728 pRun->ready_for_interrupt_injection, pRun->flags, MpState.mp_state));
1729 }
1730#endif
1731 fStatefulExit = false;
1732 if (RT_LIKELY(rcLnx == 0 || errno == EINTR))
1733 {
1734 /*
1735 * Deal with the exit.
1736 */
1737 rcStrict = nemHCLnxHandleExit(pVM, pVCpu, pRun, &fStatefulExit);
1738 if (rcStrict == VINF_SUCCESS)
1739 { /* hopefully likely */ }
1740 else
1741 {
1742 LogFlow(("NEM/%u: breaking: nemHCLnxHandleExit -> %Rrc\n", pVCpu->idCpu, VBOXSTRICTRC_VAL(rcStrict) ));
1743 STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatBreakOnStatus);
1744 break;
1745 }
1746 }
1747 else
1748 {
1749 int rc2 = RTErrConvertFromErrno(errno);
1750 AssertLogRelMsgFailedReturn(("KVM_RUN failed: rcLnx=%d errno=%u rc=%Rrc\n", rcLnx, errno, rc2), rc2);
1751 }
1752
1753 /*
1754 * If no relevant FFs are pending, loop.
1755 */
1756 if ( !VM_FF_IS_ANY_SET( pVM, !fSingleStepping ? VM_FF_HP_R0_PRE_HM_MASK : VM_FF_HP_R0_PRE_HM_STEP_MASK)
1757 && !VMCPU_FF_IS_ANY_SET(pVCpu, !fSingleStepping ? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
1758 { /* likely */ }
1759 else
1760 {
1761
1762 /** @todo Try handle pending flags, not just return to EM loops. Take care
1763 * not to set important RCs here unless we've handled an exit. */
1764 LogFlow(("NEM/%u: breaking: pending FF (%#x / %#RX64)\n",
1765 pVCpu->idCpu, pVM->fGlobalForcedActions, (uint64_t)pVCpu->fLocalForcedActions));
1766 STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatBreakOnFFPost);
1767 break;
1768 }
1769 }
1770 else
1771 {
1772 LogFlow(("NEM/%u: breaking: canceled %d (pre exec)\n", pVCpu->idCpu, VMCPU_GET_STATE(pVCpu) ));
1773 STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatBreakOnCancel);
1774 break;
1775 }
1776 }
1777 else
1778 {
1779 LogFlow(("NEM/%u: breaking: pending FF (pre exec)\n", pVCpu->idCpu));
1780 STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatBreakOnFFPre);
1781 break;
1782 }
1783 } /* the run loop */
1784
1785
1786 /*
1787 * If the last exit was stateful, commit the state we provided before
1788 * returning to the EM loop so we have a consistent state and can safely
1789 * be rescheduled and whatnot. This may require us to make multiple runs
1790 * for larger MMIO and I/O operations. Sigh^3.
1791 *
1792 * Note! There is no 'ing way to reset the kernel side completion callback
1793 * for these stateful i/o exits. Very annoying interface.
1794 */
1795 /** @todo check how this works with string I/O and string MMIO. */
1796 if (fStatefulExit && RT_SUCCESS(rcStrict))
1797 {
1798 STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatFlushExitOnReturn);
1799 uint32_t const uOrgExit = pRun->exit_reason;
1800 for (uint32_t i = 0; ; i++)
1801 {
1802 pRun->immediate_exit = 1;
1803 int rcLnx = ioctl(pVCpu->nem.s.fdVCpu, KVM_RUN, 0UL);
1804 Log(("NEM/%u: Flushed stateful exit -> %d/%d exit_reason=%d\n", pVCpu->idCpu, rcLnx, errno, pRun->exit_reason));
1805 if (rcLnx == -1 && errno == EINTR)
1806 {
1807 switch (i)
1808 {
1809 case 0: STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatFlushExitOnReturn1Loop); break;
1810 case 1: STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatFlushExitOnReturn2Loops); break;
1811 case 2: STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatFlushExitOnReturn3Loops); break;
1812 default: STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatFlushExitOnReturn4PlusLoops); break;
1813 }
1814 break;
1815 }
1816 AssertLogRelMsgBreakStmt(rcLnx == 0 && pRun->exit_reason == uOrgExit,
1817 ("rcLnx=%d errno=%d exit_reason=%d uOrgExit=%d\n", rcLnx, errno, pRun->exit_reason, uOrgExit),
1818 rcStrict = VERR_NEM_IPE_6);
1819 VBOXSTRICTRC rcStrict2 = nemHCLnxHandleExit(pVM, pVCpu, pRun, &fStatefulExit);
1820 if (rcStrict2 == VINF_SUCCESS || rcStrict2 == rcStrict)
1821 { /* likely */ }
1822 else if (RT_FAILURE(rcStrict2))
1823 {
1824 rcStrict = rcStrict2;
1825 break;
1826 }
1827 else
1828 {
1829 AssertLogRelMsgBreakStmt(rcStrict == VINF_SUCCESS,
1830 ("rcStrict=%Rrc rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict), VBOXSTRICTRC_VAL(rcStrict2)),
1831 rcStrict = VERR_NEM_IPE_7);
1832 rcStrict = rcStrict2;
1833 }
1834 }
1835 pRun->immediate_exit = 0;
1836 }
1837
1838 /*
1839 * If the CPU is running, make sure to stop it before we try sync back the
1840 * state and return to EM. We don't sync back the whole state if we can help it.
1841 */
1842 if (!VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED, VMCPUSTATE_STARTED_EXEC_NEM))
1843 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED, VMCPUSTATE_STARTED_EXEC_NEM_CANCELED);
1844
1845 if (pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_ALL)
1846 {
1847 /* Try anticipate what we might need. */
1848 uint64_t fImport = CPUMCTX_EXTRN_INHIBIT_INT | CPUMCTX_EXTRN_INHIBIT_NMI /* Required for processing APIC,PIC,NMI & SMI FFs. */
1849 | IEM_CPUMCTX_EXTRN_MUST_MASK /*?*/;
1850 if ( (rcStrict >= VINF_EM_FIRST && rcStrict <= VINF_EM_LAST)
1851 || RT_FAILURE(rcStrict))
1852 fImport = CPUMCTX_EXTRN_ALL;
1853# ifdef IN_RING0 /* Ring-3 I/O port access optimizations: */
1854 else if ( rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE
1855 || rcStrict == VINF_EM_PENDING_R3_IOPORT_WRITE)
1856 fImport = CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_RFLAGS;
1857 else if (rcStrict == VINF_EM_PENDING_R3_IOPORT_READ)
1858 fImport = CPUMCTX_EXTRN_RAX | CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_RFLAGS;
1859# endif
1860 else if (VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_PIC | VMCPU_FF_INTERRUPT_APIC
1861 | VMCPU_FF_INTERRUPT_NMI | VMCPU_FF_INTERRUPT_SMI))
1862 fImport |= IEM_CPUMCTX_EXTRN_XCPT_MASK;
1863
1864 if (pVCpu->cpum.GstCtx.fExtrn & fImport)
1865 {
1866 int rc2 = nemHCLnxImportState(pVCpu, fImport, &pVCpu->cpum.GstCtx, pRun);
1867 if (RT_SUCCESS(rc2))
1868 pVCpu->cpum.GstCtx.fExtrn &= ~fImport;
1869 else if (RT_SUCCESS(rcStrict))
1870 rcStrict = rc2;
1871 if (!(pVCpu->cpum.GstCtx.fExtrn & CPUMCTX_EXTRN_ALL))
1872 pVCpu->cpum.GstCtx.fExtrn = 0;
1873 STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatImportOnReturn);
1874 }
1875 else
1876 STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatImportOnReturnSkipped);
1877 }
1878 else
1879 {
1880 pVCpu->cpum.GstCtx.fExtrn = 0;
1881 STAM_REL_COUNTER_INC(&pVCpu->nem.s.StatImportOnReturnSkipped);
1882 }
1883
1884 LogFlow(("NEM/%u: %04x:%08RX64 efl=%#08RX64 => %Rrc\n", pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
1885 pVCpu->cpum.GstCtx.rflags.u, VBOXSTRICTRC_VAL(rcStrict) ));
1886 return rcStrict;
1887}
1888
1889
1890/** @page pg_nem_linux NEM/linux - Native Execution Manager, Linux.
1891 *
1892 * This is using KVM.
1893 *
1894 */
1895
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette