VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HWVMXR0.cpp@ 46474

Last change on this file since 46474 was 46420, checked in by vboxsync, 12 years ago

VMM, recompiler: Purge deprecated macros.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Id Revision
File size: 228.2 KB
Line 
1/* $Id: HWVMXR0.cpp 46420 2013-06-06 16:27:25Z vboxsync $ */
2/** @file
3 * HM VMX (VT-x) - Host Context Ring-0.
4 */
5
6/*
7 * Copyright (C) 2006-2013 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18
19/*******************************************************************************
20* Header Files *
21*******************************************************************************/
22#define LOG_GROUP LOG_GROUP_HM
23#include <iprt/asm-amd64-x86.h>
24#include <VBox/vmm/hm.h>
25#include <VBox/vmm/pgm.h>
26#include <VBox/vmm/dbgf.h>
27#include <VBox/vmm/dbgftrace.h>
28#include <VBox/vmm/selm.h>
29#include <VBox/vmm/iom.h>
30#ifdef VBOX_WITH_REM
31# include <VBox/vmm/rem.h>
32#endif
33#include <VBox/vmm/tm.h>
34#include "HMInternal.h"
35#include <VBox/vmm/vm.h>
36#include <VBox/vmm/pdmapi.h>
37#include <VBox/err.h>
38#include <VBox/log.h>
39#include <iprt/assert.h>
40#include <iprt/param.h>
41#include <iprt/string.h>
42#include <iprt/time.h>
43#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
44# include <iprt/thread.h>
45#endif
46#include <iprt/x86.h>
47#include "HWVMXR0.h"
48
49#include "dtrace/VBoxVMM.h"
50
51
52/*******************************************************************************
53* Defined Constants And Macros *
54*******************************************************************************/
55#if defined(RT_ARCH_AMD64)
56# define VMX_IS_64BIT_HOST_MODE() (true)
57#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
58# define VMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
59#else
60# define VMX_IS_64BIT_HOST_MODE() (false)
61#endif
62
63# define VMX_WRITE_SELREG(REG, reg) \
64 do \
65 { \
66 rc = VMXWriteVmcs(VMX_VMCS16_GUEST_FIELD_##REG, pCtx->reg.Sel); \
67 rc |= VMXWriteVmcs(VMX_VMCS32_GUEST_##REG##_LIMIT, pCtx->reg.u32Limit); \
68 rc |= VMXWriteVmcs64(VMX_VMCS_GUEST_##REG##_BASE, pCtx->reg.u64Base); \
69 if ((pCtx->eflags.u32 & X86_EFL_VM)) \
70 { \
71 /* Must override this or else VT-x will fail with invalid guest state errors. */ \
72 /* DPL=3, present, code/data, r/w/accessed. */ \
73 /** @todo we shouldn't have to do this, if it is not 0xf3 it means we screwed up elsewhere (recompiler). */ \
74 /** @todo VT-x docs explicitly mentions 0xF3. Why not just val = 0xf3 ??. */ \
75 val = (pCtx->reg.Attr.u & ~0xFF) | 0xF3; \
76 } \
77 else \
78 if ( CPUMIsGuestInRealModeEx(pCtx) \
79 && !pVM->hm.s.vmx.fUnrestrictedGuest) \
80 { \
81 /** @todo shouldn't the 'if' condition above check for 'pRealModeTSS' ? */ \
82 /* Must override this or else VT-x will fail with invalid guest state errors. */ \
83 /* DPL=3, present, code/data, r/w/accessed. */ \
84 val = 0xf3; \
85 } \
86 else \
87 if ( ( pCtx->reg.Sel \
88 || !CPUMIsGuestInPagedProtectedModeEx(pCtx) \
89 || (!pCtx->cs.Attr.n.u1DefBig && !CPUMIsGuestIn64BitCodeEx(pCtx)) \
90 ) \
91 && pCtx->reg.Attr.n.u1Present == 1) \
92 { \
93 val = pCtx->reg.Attr.u | X86_SEL_TYPE_ACCESSED; \
94 } \
95 else \
96 val = 0x10000; /* Invalid guest state error otherwise. (BIT(16) = Unusable) */ \
97 \
98 rc |= VMXWriteVmcs(VMX_VMCS32_GUEST_##REG##_ACCESS_RIGHTS, val); \
99 } while (0)
100
101# define VMX_READ_SELREG(REG, reg) \
102 do \
103 { \
104 VMXReadCachedVmcs(VMX_VMCS16_GUEST_FIELD_##REG, &val); \
105 pCtx->reg.Sel = val; \
106 pCtx->reg.ValidSel = val; \
107 pCtx->reg.fFlags = CPUMSELREG_FLAGS_VALID; \
108 VMXReadCachedVmcs(VMX_VMCS32_GUEST_##REG##_LIMIT, &val); \
109 pCtx->reg.u32Limit = val; \
110 VMXReadCachedVmcs(VMX_VMCS_GUEST_##REG##_BASE, &val); \
111 pCtx->reg.u64Base = val; \
112 VMXReadCachedVmcs(VMX_VMCS32_GUEST_##REG##_ACCESS_RIGHTS, &val); \
113 pCtx->reg.Attr.u = val; \
114 } while (0)
115
116/* Don't read from the cache in this macro; used only in case of failure where the cache is out of sync. */
117# define VMX_LOG_SELREG(REG, szSelReg, val) \
118 do \
119 { \
120 VMXReadVmcs(VMX_VMCS16_GUEST_FIELD_##REG, &(val)); \
121 Log(("%s Selector %x\n", szSelReg, (val))); \
122 VMXReadVmcs(VMX_VMCS32_GUEST_##REG##_LIMIT, &(val)); \
123 Log(("%s Limit %x\n", szSelReg, (val))); \
124 VMXReadVmcs(VMX_VMCS_GUEST_##REG##_BASE, &(val)); \
125 Log(("%s Base %RX64\n", szSelReg, (uint64_t)(val))); \
126 VMXReadVmcs(VMX_VMCS32_GUEST_##REG##_ACCESS_RIGHTS, &(val)); \
127 Log(("%s Attributes %x\n", szSelReg, (val))); \
128 } while (0)
129
130#define VMXSetupCachedReadVmcs(pCache, idxField) \
131{ \
132 Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
133 pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
134 pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
135}
136#define VMX_SETUP_SELREG(REG, pCache) \
137{ \
138 VMXSetupCachedReadVmcs(pCache, VMX_VMCS16_GUEST_FIELD_##REG); \
139 VMXSetupCachedReadVmcs(pCache, VMX_VMCS32_GUEST_##REG##_LIMIT); \
140 VMXSetupCachedReadVmcs(pCache, VMX_VMCS_GUEST_##REG##_BASE); \
141 VMXSetupCachedReadVmcs(pCache, VMX_VMCS32_GUEST_##REG##_ACCESS_RIGHTS); \
142}
143
144
145/*******************************************************************************
146* Global Variables *
147*******************************************************************************/
148/* IO operation lookup arrays. */
149static uint32_t const g_aIOSize[4] = {1, 2, 0, 4};
150static uint32_t const g_aIOOpAnd[4] = {0xff, 0xffff, 0, 0xffffffff};
151
152#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
153/** See HMR0A.asm. */
154extern "C" uint32_t g_fVMXIs64bitHost;
155#endif
156
157
158/*******************************************************************************
159* Local Functions *
160*******************************************************************************/
161static DECLCALLBACK(void) hmR0VmxSetupTLBEPT(PVM pVM, PVMCPU pVCpu);
162static DECLCALLBACK(void) hmR0VmxSetupTLBVPID(PVM pVM, PVMCPU pVCpu);
163static DECLCALLBACK(void) hmR0VmxSetupTLBBoth(PVM pVM, PVMCPU pVCpu);
164static DECLCALLBACK(void) hmR0VmxSetupTLBDummy(PVM pVM, PVMCPU pVCpu);
165static void hmR0VmxFlushEPT(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush);
166static void hmR0VmxFlushVPID(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr);
167static void hmR0VmxUpdateExceptionBitmap(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
168static void hmR0VmxSetMSRPermission(PVMCPU pVCpu, unsigned ulMSR, bool fRead, bool fWrite);
169static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, VBOXSTRICTRC rc, PCPUMCTX pCtx);
170
171
172/**
173 * Updates error from VMCS to HMCPU's lasterror record.
174 *
175 * @param pVM Pointer to the VM.
176 * @param pVCpu Pointer to the VMCPU.
177 * @param rc The error code.
178 */
179static void hmR0VmxCheckError(PVM pVM, PVMCPU pVCpu, int rc)
180{
181 if ( rc == VERR_VMX_UNABLE_TO_START_VM
182 || rc == VERR_VMX_INVALID_VMCS_FIELD)
183 {
184 RTCCUINTREG instrError;
185
186 VMXReadVmcs(VMX_VMCS32_RO_VM_INSTR_ERROR, &instrError);
187 pVCpu->hm.s.vmx.lasterror.u32InstrError = instrError;
188 }
189 pVM->hm.s.lLastError = rc;
190}
191
192
193/**
194 * Sets up and activates VT-x on the current CPU.
195 *
196 * @returns VBox status code.
197 * @param pCpu Pointer to the CPU info struct.
198 * @param pVM Pointer to the VM. (can be NULL after a resume!!)
199 * @param pvCpuPage Pointer to the global CPU page.
200 * @param HCPhysCpuPage Physical address of the global CPU page.
201 * @param fEnabledByHost Set if SUPR0EnableVTx or similar was used to enable
202 * VT-x/AMD-V on the host.
203 */
204VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBLCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost)
205{
206 if (!fEnabledByHost)
207 {
208 AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
209 AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
210
211 if (pVM)
212 {
213 /* Set revision dword at the beginning of the VMXON structure. */
214 *(uint32_t *)pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.msr.vmx_basic_info);
215 }
216
217 /** @todo we should unmap the two pages from the virtual address space in order to prevent accidental corruption.
218 * (which can have very bad consequences!!!)
219 */
220
221 /** @todo r=bird: Why is this code different than the probing code earlier
222 * on? It just sets VMXE if needed and doesn't check that it isn't
223 * set. Mac OS X host_vmxoff may leave this set and we'll fail here
224 * and debug-assert in the calling code. This is what caused the
225 * "regression" after backing out the SUPR0EnableVTx code hours before
226 * 4.2.0GA (reboot fixed the issue). I've changed here to do the same
227 * as the init code. */
228 uint64_t uCr4 = ASMGetCR4();
229 if (!(uCr4 & X86_CR4_VMXE))
230 ASMSetCR4(ASMGetCR4() | X86_CR4_VMXE); /* Make sure the VMX instructions don't cause #UD faults. */
231
232 /*
233 * Enter VM root mode.
234 */
235 int rc = VMXEnable(HCPhysCpuPage);
236 if (RT_FAILURE(rc))
237 {
238 ASMSetCR4(uCr4);
239 return VERR_VMX_VMXON_FAILED;
240 }
241 }
242
243 /*
244 * Flush all VPIDs (in case we or any other hypervisor have been using VPIDs) so that
245 * we can avoid an explicit flush while using new VPIDs. We would still need to flush
246 * each time while reusing a VPID after hitting the MaxASID limit once.
247 */
248 if ( pVM
249 && pVM->hm.s.vmx.fVpid
250 && (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS))
251 {
252 hmR0VmxFlushVPID(pVM, NULL /* pvCpu */, VMX_FLUSH_VPID_ALL_CONTEXTS, 0 /* GCPtr */);
253 pCpu->fFlushAsidBeforeUse = false;
254 }
255 else
256 pCpu->fFlushAsidBeforeUse = true;
257
258 /*
259 * Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}.
260 */
261 ++pCpu->cTlbFlushes;
262
263 return VINF_SUCCESS;
264}
265
266
267/**
268 * Deactivates VT-x on the current CPU.
269 *
270 * @returns VBox status code.
271 * @param pCpu Pointer to the CPU info struct.
272 * @param pvCpuPage Pointer to the global CPU page.
273 * @param HCPhysCpuPage Physical address of the global CPU page.
274 */
275VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBLCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
276{
277 AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
278 AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
279 NOREF(pCpu);
280
281 /* If we're somehow not in VMX root mode, then we shouldn't dare leaving it. */
282 if (!(ASMGetCR4() & X86_CR4_VMXE))
283 return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
284
285 /* Leave VMX Root Mode. */
286 VMXDisable();
287
288 /* And clear the X86_CR4_VMXE bit. */
289 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
290 return VINF_SUCCESS;
291}
292
293VMMR0DECL(int) VMXR0GlobalInit(void)
294{
295 /* Nothing to do. */
296 return VINF_SUCCESS;
297}
298
299VMMR0DECL(void) VMXR0GlobalTerm(void)
300{
301 /* Nothing to do. */
302}
303
304/**
305 * Does Ring-0 per VM VT-x initialization.
306 *
307 * @returns VBox status code.
308 * @param pVM Pointer to the VM.
309 */
310VMMR0DECL(int) VMXR0InitVM(PVM pVM)
311{
312 int rc;
313
314#ifdef LOG_ENABLED
315 SUPR0Printf("VMXR0InitVM %p\n", pVM);
316#endif
317
318 pVM->hm.s.vmx.hMemObjApicAccess = NIL_RTR0MEMOBJ;
319
320 if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
321 {
322 /* Allocate one page for the APIC physical page (serves for filtering accesses). */
323 rc = RTR0MemObjAllocCont(&pVM->hm.s.vmx.hMemObjApicAccess, PAGE_SIZE, false /* fExecutable */);
324 AssertRC(rc);
325 if (RT_FAILURE(rc))
326 return rc;
327
328 pVM->hm.s.vmx.pbApicAccess = (uint8_t *)RTR0MemObjAddress(pVM->hm.s.vmx.hMemObjApicAccess);
329 pVM->hm.s.vmx.HCPhysApicAccess = RTR0MemObjGetPagePhysAddr(pVM->hm.s.vmx.hMemObjApicAccess, 0);
330 ASMMemZero32(pVM->hm.s.vmx.pbApicAccess, PAGE_SIZE);
331 }
332 else
333 {
334 pVM->hm.s.vmx.hMemObjApicAccess = 0;
335 pVM->hm.s.vmx.pbApicAccess = 0;
336 pVM->hm.s.vmx.HCPhysApicAccess = 0;
337 }
338
339#ifdef VBOX_WITH_CRASHDUMP_MAGIC
340 {
341 rc = RTR0MemObjAllocCont(&pVM->hm.s.vmx.hMemObjScratch, PAGE_SIZE, false /* fExecutable */);
342 AssertRC(rc);
343 if (RT_FAILURE(rc))
344 return rc;
345
346 pVM->hm.s.vmx.pScratch = (uint8_t *)RTR0MemObjAddress(pVM->hm.s.vmx.hMemObjScratch);
347 pVM->hm.s.vmx.pScratchPhys = RTR0MemObjGetPagePhysAddr(pVM->hm.s.vmx.hMemObjScratch, 0);
348
349 ASMMemZero32(pVM->hm.s.vmx.pbScratch, PAGE_SIZE);
350 strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
351 *(uint64_t *)(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xDEADBEEFDEADBEEF);
352 }
353#endif
354
355 /* Allocate VMCSs for all guest CPUs. */
356 for (VMCPUID i = 0; i < pVM->cCpus; i++)
357 {
358 PVMCPU pVCpu = &pVM->aCpus[i];
359
360 pVCpu->hm.s.vmx.hMemObjVmcs = NIL_RTR0MEMOBJ;
361
362 /* Allocate one page for the VM control structure (VMCS). */
363 rc = RTR0MemObjAllocCont(&pVCpu->hm.s.vmx.hMemObjVmcs, PAGE_SIZE, false /* fExecutable */);
364 AssertRC(rc);
365 if (RT_FAILURE(rc))
366 return rc;
367
368 pVCpu->hm.s.vmx.pvVmcs = RTR0MemObjAddress(pVCpu->hm.s.vmx.hMemObjVmcs);
369 pVCpu->hm.s.vmx.HCPhysVmcs = RTR0MemObjGetPagePhysAddr(pVCpu->hm.s.vmx.hMemObjVmcs, 0);
370 ASMMemZeroPage(pVCpu->hm.s.vmx.pvVmcs);
371
372 pVCpu->hm.s.vmx.u32CR0Mask = 0;
373 pVCpu->hm.s.vmx.u32CR4Mask = 0;
374
375 /* Allocate one page for the virtual APIC page for TPR caching. */
376 rc = RTR0MemObjAllocCont(&pVCpu->hm.s.vmx.hMemObjVirtApic, PAGE_SIZE, false /* fExecutable */);
377 AssertRC(rc);
378 if (RT_FAILURE(rc))
379 return rc;
380
381 pVCpu->hm.s.vmx.pbVirtApic = (uint8_t *)RTR0MemObjAddress(pVCpu->hm.s.vmx.hMemObjVirtApic);
382 pVCpu->hm.s.vmx.HCPhysVirtApic = RTR0MemObjGetPagePhysAddr(pVCpu->hm.s.vmx.hMemObjVirtApic, 0);
383 ASMMemZeroPage(pVCpu->hm.s.vmx.pbVirtApic);
384
385 /* Allocate the MSR bitmap if this feature is supported. */
386 if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
387 {
388 rc = RTR0MemObjAllocCont(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, PAGE_SIZE, false /* fExecutable */);
389 AssertRC(rc);
390 if (RT_FAILURE(rc))
391 return rc;
392
393 pVCpu->hm.s.vmx.pvMsrBitmap = (uint8_t *)RTR0MemObjAddress(pVCpu->hm.s.vmx.hMemObjMsrBitmap);
394 pVCpu->hm.s.vmx.HCPhysMsrBitmap = RTR0MemObjGetPagePhysAddr(pVCpu->hm.s.vmx.hMemObjMsrBitmap, 0);
395 memset(pVCpu->hm.s.vmx.pvMsrBitmap, 0xff, PAGE_SIZE);
396 }
397
398#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
399 /* Allocate one page for the guest MSR load area (for preloading guest MSRs during the world switch). */
400 rc = RTR0MemObjAllocCont(&pVCpu->hm.s.vmx.hMemObjGuestMsr, PAGE_SIZE, false /* fExecutable */);
401 AssertRC(rc);
402 if (RT_FAILURE(rc))
403 return rc;
404
405 pVCpu->hm.s.vmx.pvGuestMsr = (uint8_t *)RTR0MemObjAddress(pVCpu->hm.s.vmx.hMemObjGuestMsr);
406 pVCpu->hm.s.vmx.HCPhysGuestMsr = RTR0MemObjGetPagePhysAddr(pVCpu->hm.s.vmx.hMemObjGuestMsr, 0);
407 Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf));
408 memset(pVCpu->hm.s.vmx.pvGuestMsr, 0, PAGE_SIZE);
409
410 /* Allocate one page for the host MSR load area (for restoring host MSRs after the world switch back). */
411 rc = RTR0MemObjAllocCont(&pVCpu->hm.s.vmx.hMemObjHostMsr, PAGE_SIZE, false /* fExecutable */);
412 AssertRC(rc);
413 if (RT_FAILURE(rc))
414 return rc;
415
416 pVCpu->hm.s.vmx.pvHostMsr = (uint8_t *)RTR0MemObjAddress(pVCpu->hm.s.vmx.hMemObjHostMsr);
417 pVCpu->hm.s.vmx.HCPhysHostMsr = RTR0MemObjGetPagePhysAddr(pVCpu->hm.s.vmx.hMemObjHostMsr, 0);
418 Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf));
419 memset(pVCpu->hm.s.vmx.pvHostMsr, 0, PAGE_SIZE);
420#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
421
422 /* Current guest paging mode. */
423 pVCpu->hm.s.vmx.enmLastSeenGuestMode = PGMMODE_REAL;
424
425#ifdef LOG_ENABLED
426 SUPR0Printf("VMXR0InitVM %x VMCS=%x (%x)\n", pVM, pVCpu->hm.s.vmx.pvVmcs, (uint32_t)pVCpu->hm.s.vmx.HCPhysVmcs);
427#endif
428 }
429
430 return VINF_SUCCESS;
431}
432
433
434/**
435 * Does Ring-0 per VM VT-x termination.
436 *
437 * @returns VBox status code.
438 * @param pVM Pointer to the VM.
439 */
440VMMR0DECL(int) VMXR0TermVM(PVM pVM)
441{
442 for (VMCPUID i = 0; i < pVM->cCpus; i++)
443 {
444 PVMCPU pVCpu = &pVM->aCpus[i];
445
446 if (pVCpu->hm.s.vmx.hMemObjVmcs != NIL_RTR0MEMOBJ)
447 {
448 RTR0MemObjFree(pVCpu->hm.s.vmx.hMemObjVmcs, false);
449 pVCpu->hm.s.vmx.hMemObjVmcs = NIL_RTR0MEMOBJ;
450 pVCpu->hm.s.vmx.pvVmcs = 0;
451 pVCpu->hm.s.vmx.HCPhysVmcs = 0;
452 }
453 if (pVCpu->hm.s.vmx.hMemObjVirtApic != NIL_RTR0MEMOBJ)
454 {
455 RTR0MemObjFree(pVCpu->hm.s.vmx.hMemObjVirtApic, false);
456 pVCpu->hm.s.vmx.hMemObjVirtApic = NIL_RTR0MEMOBJ;
457 pVCpu->hm.s.vmx.pbVirtApic = 0;
458 pVCpu->hm.s.vmx.HCPhysVirtApic = 0;
459 }
460 if (pVCpu->hm.s.vmx.hMemObjMsrBitmap != NIL_RTR0MEMOBJ)
461 {
462 RTR0MemObjFree(pVCpu->hm.s.vmx.hMemObjMsrBitmap, false);
463 pVCpu->hm.s.vmx.hMemObjMsrBitmap = NIL_RTR0MEMOBJ;
464 pVCpu->hm.s.vmx.pvMsrBitmap = 0;
465 pVCpu->hm.s.vmx.HCPhysMsrBitmap = 0;
466 }
467#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
468 if (pVCpu->hm.s.vmx.hMemObjHostMsr != NIL_RTR0MEMOBJ)
469 {
470 RTR0MemObjFree(pVCpu->hm.s.vmx.hMemObjHostMsr, false);
471 pVCpu->hm.s.vmx.hMemObjHostMsr = NIL_RTR0MEMOBJ;
472 pVCpu->hm.s.vmx.pvHostMsr = 0;
473 pVCpu->hm.s.vmx.HCPhysHostMsr = 0;
474 }
475 if (pVCpu->hm.s.vmx.hMemObjGuestMsr != NIL_RTR0MEMOBJ)
476 {
477 RTR0MemObjFree(pVCpu->hm.s.vmx.hMemObjGuestMsr, false);
478 pVCpu->hm.s.vmx.hMemObjGuestMsr = NIL_RTR0MEMOBJ;
479 pVCpu->hm.s.vmx.pvGuestMsr = 0;
480 pVCpu->hm.s.vmx.HCPhysGuestMsr = 0;
481 }
482#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
483 }
484 if (pVM->hm.s.vmx.hMemObjApicAccess != NIL_RTR0MEMOBJ)
485 {
486 RTR0MemObjFree(pVM->hm.s.vmx.hMemObjApicAccess, false);
487 pVM->hm.s.vmx.hMemObjApicAccess = NIL_RTR0MEMOBJ;
488 pVM->hm.s.vmx.pbApicAccess = 0;
489 pVM->hm.s.vmx.HCPhysApicAccess = 0;
490 }
491#ifdef VBOX_WITH_CRASHDUMP_MAGIC
492 if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
493 {
494 ASMMemZero32(pVM->hm.s.vmx.pScratch, PAGE_SIZE);
495 RTR0MemObjFree(pVM->hm.s.vmx.hMemObjScratch, false);
496 pVM->hm.s.vmx.hMemObjScratch = NIL_RTR0MEMOBJ;
497 pVM->hm.s.vmx.pScratch = 0;
498 pVM->hm.s.vmx.pScratchPhys = 0;
499 }
500#endif
501 return VINF_SUCCESS;
502}
503
504
505/**
506 * Sets up VT-x for the specified VM.
507 *
508 * @returns VBox status code.
509 * @param pVM Pointer to the VM.
510 */
511VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
512{
513 int rc = VINF_SUCCESS;
514 uint32_t val;
515
516 AssertReturn(pVM, VERR_INVALID_PARAMETER);
517
518 /* Initialize these always, see hmR3InitFinalizeR0().*/
519 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NONE;
520 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NONE;
521
522 /* Determine optimal flush type for EPT. */
523 if (pVM->hm.s.fNestedPaging)
524 {
525 if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
526 {
527 if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
528 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_SINGLE_CONTEXT;
529 else if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
530 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_ALL_CONTEXTS;
531 else
532 {
533 /*
534 * Should never really happen. EPT is supported but no suitable flush types supported.
535 * We cannot ignore EPT at this point as we've already setup Unrestricted Guest execution.
536 */
537 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
538 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
539 }
540 }
541 else
542 {
543 /*
544 * Should never really happen. EPT is supported but INVEPT instruction is not supported.
545 */
546 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
547 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
548 }
549 }
550
551 /* Determine optimal flush type for VPID. */
552 if (pVM->hm.s.vmx.fVpid)
553 {
554 if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
555 {
556 if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
557 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_SINGLE_CONTEXT;
558 else if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
559 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_ALL_CONTEXTS;
560 else
561 {
562 /*
563 * Neither SINGLE nor ALL context flush types for VPID supported by the CPU.
564 * We do not handle other flush type combinations, ignore VPID capabilities.
565 */
566 if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
567 Log(("VMXR0SetupVM: Only VMX_FLUSH_VPID_INDIV_ADDR supported. Ignoring VPID.\n"));
568 if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
569 Log(("VMXR0SetupVM: Only VMX_FLUSH_VPID_SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
570 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
571 pVM->hm.s.vmx.fVpid = false;
572 }
573 }
574 else
575 {
576 /*
577 * Should not really happen. EPT is supported but INVEPT is not supported.
578 * Ignore VPID capabilities as our code relies on using INVEPT for selective flushing.
579 */
580 Log(("VMXR0SetupVM: VPID supported without INVEPT support. Ignoring VPID.\n"));
581 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
582 pVM->hm.s.vmx.fVpid = false;
583 }
584 }
585
586 for (VMCPUID i = 0; i < pVM->cCpus; i++)
587 {
588 PVMCPU pVCpu = &pVM->aCpus[i];
589
590 AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
591
592 /* Set revision dword at the beginning of the VMCS structure. */
593 *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.msr.vmx_basic_info);
594
595 /*
596 * Clear and activate the VMCS.
597 */
598 Log(("HCPhysVmcs = %RHp\n", pVCpu->hm.s.vmx.HCPhysVmcs));
599 rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
600 if (RT_FAILURE(rc))
601 goto vmx_end;
602
603 rc = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
604 if (RT_FAILURE(rc))
605 goto vmx_end;
606
607 /*
608 * VMX_VMCS_CTRL_PIN_EXEC
609 * Set required bits to one and zero according to the MSR capabilities.
610 */
611 val = pVM->hm.s.vmx.msr.vmx_pin_ctls.n.disallowed0;
612 val |= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts */
613 | VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts */
614
615 /*
616 * Enable the VMX preemption timer.
617 */
618 if (pVM->hm.s.vmx.fUsePreemptTimer)
619 val |= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
620 val &= pVM->hm.s.vmx.msr.vmx_pin_ctls.n.allowed1;
621
622 rc = VMXWriteVmcs(VMX_VMCS32_CTRL_PIN_EXEC, val);
623 AssertRC(rc);
624 pVCpu->hm.s.vmx.u32PinCtls = val;
625
626 /*
627 * VMX_VMCS_CTRL_PROC_EXEC
628 * Set required bits to one and zero according to the MSR capabilities.
629 */
630 val = pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0;
631 /* Program which event cause VM-exits and which features we want to use. */
632 val |= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT
633 | VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING
634 | VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT
635 | VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT
636 | VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT
637 | VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT
638 | VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* don't execute mwait or else we'll idle inside
639 the guest (host thinks the cpu load is high) */
640
641 /* Without nested paging we should intercept invlpg and cr3 mov instructions. */
642 if (!pVM->hm.s.fNestedPaging)
643 {
644 val |= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
645 | VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
646 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
647 }
648
649 /*
650 * VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT might cause a vmlaunch
651 * failure with an invalid control fields error. (combined with some other exit reasons)
652 */
653 if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
654 {
655 /* CR8 reads from the APIC shadow page; writes cause an exit is they lower the TPR below the threshold */
656 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW;
657 Assert(pVM->hm.s.vmx.pbApicAccess);
658 }
659 else
660 /* Exit on CR8 reads & writes in case the TPR shadow feature isn't present. */
661 val |= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT;
662
663 if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
664 {
665 Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
666 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
667 }
668
669 /* We will use the secondary control if it's present. */
670 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
671
672 /* Mask away the bits that the CPU doesn't support */
673 /** @todo make sure they don't conflict with the above requirements. */
674 val &= pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1;
675 pVCpu->hm.s.vmx.u32ProcCtls = val;
676
677 rc = VMXWriteVmcs(VMX_VMCS32_CTRL_PROC_EXEC, val);
678 AssertRC(rc);
679
680 if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
681 {
682 /*
683 * VMX_VMCS_CTRL_PROC_EXEC2
684 * Set required bits to one and zero according to the MSR capabilities.
685 */
686 val = pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0;
687 val |= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT;
688
689 if (pVM->hm.s.fNestedPaging)
690 val |= VMX_VMCS_CTRL_PROC_EXEC2_EPT;
691
692 if (pVM->hm.s.vmx.fVpid)
693 val |= VMX_VMCS_CTRL_PROC_EXEC2_VPID;
694
695 if (pVM->hm.s.fHasIoApic)
696 val |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC;
697
698 if (pVM->hm.s.vmx.fUnrestrictedGuest)
699 val |= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST;
700
701 if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
702 val |= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP;
703
704 /* Mask away the bits that the CPU doesn't support */
705 /** @todo make sure they don't conflict with the above requirements. */
706 val &= pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1;
707 pVCpu->hm.s.vmx.u32ProcCtls2 = val;
708 rc = VMXWriteVmcs(VMX_VMCS32_CTRL_PROC_EXEC2, val);
709 AssertRC(rc);
710 }
711
712 /*
713 * VMX_VMCS_CTRL_CR3_TARGET_COUNT
714 * Set required bits to one and zero according to the MSR capabilities.
715 */
716 rc = VMXWriteVmcs(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0);
717 AssertRC(rc);
718
719 /*
720 * Forward all exception except #NM & #PF to the guest.
721 * We always need to check pagefaults since our shadow page table can be out of sync.
722 * And we always lazily sync the FPU & XMM state. .
723 */
724
725 /** @todo Possible optimization:
726 * Keep the FPU and XMM state current in the EM thread. That way there's no need to
727 * lazily sync anything, but the downside is that we can't use the FPU stack or XMM
728 * registers ourselves of course.
729 *
730 * Note: only possible if the current state is actually ours (X86_CR0_TS flag)
731 */
732
733 /*
734 * Don't filter page faults, all of them should cause a world switch.
735 */
736 rc = VMXWriteVmcs(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0);
737 AssertRC(rc);
738 rc = VMXWriteVmcs(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0);
739 AssertRC(rc);
740
741 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0);
742 AssertRC(rc);
743 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0);
744 AssertRC(rc);
745 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0);
746 AssertRC(rc);
747
748 /*
749 * Set the MSR bitmap address.
750 */
751 if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
752 {
753 Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
754
755 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
756 AssertRC(rc);
757
758 /*
759 * Allow the guest to directly modify these MSRs; they are loaded/stored automatically
760 * using MSR-load/store areas in the VMCS.
761 */
762 hmR0VmxSetMSRPermission(pVCpu, MSR_IA32_SYSENTER_CS, true, true);
763 hmR0VmxSetMSRPermission(pVCpu, MSR_IA32_SYSENTER_ESP, true, true);
764 hmR0VmxSetMSRPermission(pVCpu, MSR_IA32_SYSENTER_EIP, true, true);
765 hmR0VmxSetMSRPermission(pVCpu, MSR_K8_LSTAR, true, true);
766 hmR0VmxSetMSRPermission(pVCpu, MSR_K6_STAR, true, true);
767 hmR0VmxSetMSRPermission(pVCpu, MSR_K8_SF_MASK, true, true);
768 hmR0VmxSetMSRPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, true, true);
769 hmR0VmxSetMSRPermission(pVCpu, MSR_K8_GS_BASE, true, true);
770 hmR0VmxSetMSRPermission(pVCpu, MSR_K8_FS_BASE, true, true);
771 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
772 hmR0VmxSetMSRPermission(pVCpu, MSR_K8_TSC_AUX, true, true);
773 }
774
775#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
776 /*
777 * Set the guest & host MSR load/store physical addresses.
778 */
779 Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
780 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
781 AssertRC(rc);
782 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
783 AssertRC(rc);
784 Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
785 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
786 AssertRC(rc);
787#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
788
789 rc = VMXWriteVmcs(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
790 AssertRC(rc);
791 rc = VMXWriteVmcs(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0);
792 AssertRC(rc);
793 rc = VMXWriteVmcs(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0);
794 AssertRC(rc);
795
796 if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
797 {
798 Assert(pVM->hm.s.vmx.hMemObjApicAccess);
799 /* Optional */
800 rc = VMXWriteVmcs(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
801 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
802
803 if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
804 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
805
806 AssertRC(rc);
807 }
808
809 /* Set link pointer to -1. Not currently used. */
810 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, 0xFFFFFFFFFFFFFFFFULL);
811 AssertRC(rc);
812
813 /*
814 * Clear VMCS, marking it inactive. Clear implementation specific data and writing back
815 * VMCS data back to memory.
816 */
817 rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
818 AssertRC(rc);
819
820 /*
821 * Configure the VMCS read cache.
822 */
823 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
824
825 VMXSetupCachedReadVmcs(pCache, VMX_VMCS_GUEST_RIP);
826 VMXSetupCachedReadVmcs(pCache, VMX_VMCS_GUEST_RSP);
827 VMXSetupCachedReadVmcs(pCache, VMX_VMCS_GUEST_RFLAGS);
828 VMXSetupCachedReadVmcs(pCache, VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE);
829 VMXSetupCachedReadVmcs(pCache, VMX_VMCS_CTRL_CR0_READ_SHADOW);
830 VMXSetupCachedReadVmcs(pCache, VMX_VMCS_GUEST_CR0);
831 VMXSetupCachedReadVmcs(pCache, VMX_VMCS_CTRL_CR4_READ_SHADOW);
832 VMXSetupCachedReadVmcs(pCache, VMX_VMCS_GUEST_CR4);
833 VMXSetupCachedReadVmcs(pCache, VMX_VMCS_GUEST_DR7);
834 VMXSetupCachedReadVmcs(pCache, VMX_VMCS32_GUEST_SYSENTER_CS);
835 VMXSetupCachedReadVmcs(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
836 VMXSetupCachedReadVmcs(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
837 VMXSetupCachedReadVmcs(pCache, VMX_VMCS32_GUEST_GDTR_LIMIT);
838 VMXSetupCachedReadVmcs(pCache, VMX_VMCS_GUEST_GDTR_BASE);
839 VMXSetupCachedReadVmcs(pCache, VMX_VMCS32_GUEST_IDTR_LIMIT);
840 VMXSetupCachedReadVmcs(pCache, VMX_VMCS_GUEST_IDTR_BASE);
841
842 VMX_SETUP_SELREG(ES, pCache);
843 VMX_SETUP_SELREG(SS, pCache);
844 VMX_SETUP_SELREG(CS, pCache);
845 VMX_SETUP_SELREG(DS, pCache);
846 VMX_SETUP_SELREG(FS, pCache);
847 VMX_SETUP_SELREG(GS, pCache);
848 VMX_SETUP_SELREG(LDTR, pCache);
849 VMX_SETUP_SELREG(TR, pCache);
850
851 /*
852 * Status code VMCS reads.
853 */
854 VMXSetupCachedReadVmcs(pCache, VMX_VMCS32_RO_EXIT_REASON);
855 VMXSetupCachedReadVmcs(pCache, VMX_VMCS32_RO_VM_INSTR_ERROR);
856 VMXSetupCachedReadVmcs(pCache, VMX_VMCS32_RO_EXIT_INSTR_LENGTH);
857 VMXSetupCachedReadVmcs(pCache, VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE);
858 VMXSetupCachedReadVmcs(pCache, VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO);
859 VMXSetupCachedReadVmcs(pCache, VMX_VMCS32_RO_EXIT_INSTR_INFO);
860 VMXSetupCachedReadVmcs(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
861 VMXSetupCachedReadVmcs(pCache, VMX_VMCS32_RO_IDT_INFO);
862 VMXSetupCachedReadVmcs(pCache, VMX_VMCS32_RO_IDT_ERROR_CODE);
863
864 if (pVM->hm.s.fNestedPaging)
865 {
866 VMXSetupCachedReadVmcs(pCache, VMX_VMCS_GUEST_CR3);
867 VMXSetupCachedReadVmcs(pCache, VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL);
868 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
869 }
870 else
871 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
872 } /* for each VMCPU */
873
874 /*
875 * Setup the right TLB function based on CPU capabilities.
876 */
877 if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
878 pVM->hm.s.vmx.pfnFlushTaggedTlb = hmR0VmxSetupTLBBoth;
879 else if (pVM->hm.s.fNestedPaging)
880 pVM->hm.s.vmx.pfnFlushTaggedTlb = hmR0VmxSetupTLBEPT;
881 else if (pVM->hm.s.vmx.fVpid)
882 pVM->hm.s.vmx.pfnFlushTaggedTlb = hmR0VmxSetupTLBVPID;
883 else
884 pVM->hm.s.vmx.pfnFlushTaggedTlb = hmR0VmxSetupTLBDummy;
885
886vmx_end:
887 hmR0VmxCheckError(pVM, &pVM->aCpus[0], rc);
888 return rc;
889}
890
891
892/**
893 * Sets the permission bits for the specified MSR.
894 *
895 * @param pVCpu Pointer to the VMCPU.
896 * @param ulMSR The MSR value.
897 * @param fRead Whether reading is allowed.
898 * @param fWrite Whether writing is allowed.
899 */
900static void hmR0VmxSetMSRPermission(PVMCPU pVCpu, unsigned ulMSR, bool fRead, bool fWrite)
901{
902 unsigned ulBit;
903 uint8_t *pvMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
904
905 /*
906 * Layout:
907 * 0x000 - 0x3ff - Low MSR read bits
908 * 0x400 - 0x7ff - High MSR read bits
909 * 0x800 - 0xbff - Low MSR write bits
910 * 0xc00 - 0xfff - High MSR write bits
911 */
912 if (ulMSR <= 0x00001FFF)
913 {
914 /* Pentium-compatible MSRs */
915 ulBit = ulMSR;
916 }
917 else if ( ulMSR >= 0xC0000000
918 && ulMSR <= 0xC0001FFF)
919 {
920 /* AMD Sixth Generation x86 Processor MSRs */
921 ulBit = (ulMSR - 0xC0000000);
922 pvMsrBitmap += 0x400;
923 }
924 else
925 {
926 AssertFailed();
927 return;
928 }
929
930 Assert(ulBit <= 0x1fff);
931 if (fRead)
932 ASMBitClear(pvMsrBitmap, ulBit);
933 else
934 ASMBitSet(pvMsrBitmap, ulBit);
935
936 if (fWrite)
937 ASMBitClear(pvMsrBitmap + 0x800, ulBit);
938 else
939 ASMBitSet(pvMsrBitmap + 0x800, ulBit);
940}
941
942
943/**
944 * Injects an event (trap or external interrupt).
945 *
946 * @returns VBox status code. Note that it may return VINF_EM_RESET to
947 * indicate a triple fault when injecting X86_XCPT_DF.
948 *
949 * @param pVM Pointer to the VM.
950 * @param pVCpu Pointer to the VMCPU.
951 * @param pCtx Pointer to the guest CPU Context.
952 * @param intInfo VMX interrupt info.
953 * @param cbInstr Opcode length of faulting instruction.
954 * @param errCode Error code (optional).
955 */
956static int hmR0VmxInjectEvent(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, uint32_t intInfo, uint32_t cbInstr, uint32_t errCode)
957{
958 int rc;
959 uint32_t iGate = VMX_EXIT_INTERRUPTION_INFO_VECTOR(intInfo);
960
961#ifdef VBOX_WITH_STATISTICS
962 STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[iGate & MASK_INJECT_IRQ_STAT]);
963#endif
964
965#ifdef VBOX_STRICT
966 if (iGate == 0xE)
967 {
968 LogFlow(("hmR0VmxInjectEvent: Injecting interrupt %d at %RGv error code=%08x CR2=%RGv intInfo=%08x\n", iGate,
969 (RTGCPTR)pCtx->rip, errCode, pCtx->cr2, intInfo));
970 }
971 else if (iGate < 0x20)
972 {
973 LogFlow(("hmR0VmxInjectEvent: Injecting interrupt %d at %RGv error code=%08x\n", iGate, (RTGCPTR)pCtx->rip,
974 errCode));
975 }
976 else
977 {
978 LogFlow(("INJ-EI: %x at %RGv\n", iGate, (RTGCPTR)pCtx->rip));
979 Assert( VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT
980 || !VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
981 Assert( VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT
982 || pCtx->eflags.u32 & X86_EFL_IF);
983 }
984#endif
985
986 if ( CPUMIsGuestInRealModeEx(pCtx)
987 && pVM->hm.s.vmx.pRealModeTSS)
988 {
989 RTGCPHYS GCPhysHandler;
990 uint16_t offset, ip;
991 RTSEL sel;
992
993 /*
994 * Injecting events doesn't work right with real mode emulation.
995 * (#GP if we try to inject external hardware interrupts)
996 * Inject the interrupt or trap directly instead.
997 *
998 * ASSUMES no access handlers for the bits we read or write below (should be safe).
999 */
1000 Log(("Manual interrupt/trap '%x' inject (real mode)\n", iGate));
1001
1002 /*
1003 * Check if the interrupt handler is present.
1004 */
1005 if (iGate * 4 + 3 > pCtx->idtr.cbIdt)
1006 {
1007 Log(("IDT cbIdt violation\n"));
1008 if (iGate != X86_XCPT_DF)
1009 {
1010 uint32_t intInfo2;
1011
1012 intInfo2 = (iGate == X86_XCPT_GP) ? (uint32_t)X86_XCPT_DF : (uint32_t)X86_XCPT_GP;
1013 intInfo2 |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
1014 intInfo2 |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
1015 intInfo2 |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
1016
1017 return hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo2, 0, 0 /* no error code according to the Intel docs */);
1018 }
1019 Log(("Triple fault -> reset the VM!\n"));
1020 return VINF_EM_RESET;
1021 }
1022 if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT
1023 || iGate == 3 /* Both #BP and #OF point to the instruction after. */
1024 || iGate == 4)
1025 {
1026 ip = pCtx->ip + cbInstr;
1027 }
1028 else
1029 ip = pCtx->ip;
1030
1031 /*
1032 * Read the selector:offset pair of the interrupt handler.
1033 */
1034 GCPhysHandler = (RTGCPHYS)pCtx->idtr.pIdt + iGate * 4;
1035 rc = PGMPhysSimpleReadGCPhys(pVM, &offset, GCPhysHandler, sizeof(offset)); AssertRC(rc);
1036 rc = PGMPhysSimpleReadGCPhys(pVM, &sel, GCPhysHandler + 2, sizeof(sel)); AssertRC(rc);
1037
1038 LogFlow(("IDT handler %04X:%04X\n", sel, offset));
1039
1040 /*
1041 * Construct the stack frame.
1042 */
1043 /** @todo Check stack limit. */
1044 pCtx->sp -= 2;
1045 LogFlow(("ss:sp %04X:%04X eflags=%x\n", pCtx->ss.Sel, pCtx->sp, pCtx->eflags.u));
1046 rc = PGMPhysSimpleWriteGCPhys(pVM, pCtx->ss.u64Base + pCtx->sp, &pCtx->eflags, sizeof(uint16_t)); AssertRC(rc);
1047 pCtx->sp -= 2;
1048 LogFlow(("ss:sp %04X:%04X cs=%x\n", pCtx->ss.Sel, pCtx->sp, pCtx->cs.Sel));
1049 rc = PGMPhysSimpleWriteGCPhys(pVM, pCtx->ss.u64Base + pCtx->sp, &pCtx->cs, sizeof(uint16_t)); AssertRC(rc);
1050 pCtx->sp -= 2;
1051 LogFlow(("ss:sp %04X:%04X ip=%x\n", pCtx->ss.Sel, pCtx->sp, ip));
1052 rc = PGMPhysSimpleWriteGCPhys(pVM, pCtx->ss.u64Base + pCtx->sp, &ip, sizeof(ip)); AssertRC(rc);
1053
1054 /*
1055 * Update the CPU state for executing the handler.
1056 */
1057 pCtx->rip = offset;
1058 pCtx->cs.Sel = sel;
1059 pCtx->cs.u64Base = sel << 4;
1060 pCtx->eflags.u &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_RF | X86_EFL_AC);
1061
1062 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_SEGMENT_REGS;
1063 return VINF_SUCCESS;
1064 }
1065
1066 /*
1067 * Set event injection state.
1068 */
1069 rc = VMXWriteVmcs(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, intInfo | (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT));
1070 rc |= VMXWriteVmcs(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
1071 rc |= VMXWriteVmcs(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, errCode);
1072
1073 AssertRC(rc);
1074 return rc;
1075}
1076
1077
1078/**
1079 * Checks for pending guest interrupts and injects them.
1080 *
1081 * @returns VBox status code.
1082 * @param pVM Pointer to the VM.
1083 * @param pVCpu Pointer to the VMCPU.
1084 * @param pCtx Pointer to the guest CPU context.
1085 */
1086static int hmR0VmxCheckPendingInterrupt(PVM pVM, PVMCPU pVCpu, CPUMCTX *pCtx)
1087{
1088 int rc;
1089
1090 /*
1091 * Dispatch any pending interrupts (injected before, but a VM exit occurred prematurely).
1092 */
1093 if (pVCpu->hm.s.Event.fPending)
1094 {
1095 Log(("CPU%d: Reinjecting event %RX64 %08x at %RGv cr2=%RX64\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntrInfo,
1096 pVCpu->hm.s.Event.u32ErrCode, (RTGCPTR)pCtx->rip, pCtx->cr2));
1097 STAM_COUNTER_INC(&pVCpu->hm.s.StatIntReinject);
1098 rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, pVCpu->hm.s.Event.u64IntrInfo, 0, pVCpu->hm.s.Event.u32ErrCode);
1099 AssertRC(rc);
1100
1101 pVCpu->hm.s.Event.fPending = false;
1102 return VINF_SUCCESS;
1103 }
1104
1105 /*
1106 * If an active trap is already pending, we must forward it first!
1107 */
1108 if (!TRPMHasTrap(pVCpu))
1109 {
1110 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI))
1111 {
1112 RTGCUINTPTR intInfo;
1113
1114 Log(("CPU%d: injecting #NMI\n", pVCpu->idCpu));
1115
1116 intInfo = X86_XCPT_NMI;
1117 intInfo |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
1118 intInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
1119
1120 rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo, 0, 0);
1121 AssertRC(rc);
1122
1123 return VINF_SUCCESS;
1124 }
1125
1126 /** @todo SMI interrupts. */
1127
1128 /*
1129 * When external interrupts are pending, we should exit the VM when IF is set.
1130 */
1131 if (VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC|VMCPU_FF_INTERRUPT_PIC)))
1132 {
1133 if (!(pCtx->eflags.u32 & X86_EFL_IF))
1134 {
1135 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
1136 {
1137 LogFlow(("Enable irq window exit!\n"));
1138 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
1139 rc = VMXWriteVmcs(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
1140 AssertRC(rc);
1141 }
1142 /* else nothing to do but wait */
1143 }
1144 else if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
1145 {
1146 uint8_t u8Interrupt;
1147
1148 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
1149 Log(("CPU%d: Dispatch interrupt: u8Interrupt=%x (%d) rc=%Rrc cs:rip=%04X:%RGv\n", pVCpu->idCpu,
1150 u8Interrupt, u8Interrupt, rc, pCtx->cs.Sel, (RTGCPTR)pCtx->rip));
1151 if (RT_SUCCESS(rc))
1152 {
1153 rc = TRPMAssertTrap(pVCpu, u8Interrupt, TRPM_HARDWARE_INT);
1154 AssertRC(rc);
1155 }
1156 else
1157 {
1158 /* Can only happen in rare cases where a pending interrupt is cleared behind our back */
1159 Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC|VMCPU_FF_INTERRUPT_PIC)));
1160 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
1161 /* Just continue */
1162 }
1163 }
1164 else
1165 Log(("Pending interrupt blocked at %RGv by VM_FF_INHIBIT_INTERRUPTS!!\n", (RTGCPTR)pCtx->rip));
1166 }
1167 }
1168
1169#ifdef VBOX_STRICT
1170 if (TRPMHasTrap(pVCpu))
1171 {
1172 uint8_t u8Vector;
1173 rc = TRPMQueryTrapAll(pVCpu, &u8Vector, 0, NULL, NULL, NULL);
1174 AssertRC(rc);
1175 }
1176#endif
1177
1178 if ( (pCtx->eflags.u32 & X86_EFL_IF)
1179 && (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
1180 && TRPMHasTrap(pVCpu)
1181 )
1182 {
1183 uint8_t u8Vector;
1184 TRPMEVENT enmType;
1185 RTGCUINTPTR intInfo;
1186 RTGCUINT errCode;
1187
1188 /*
1189 * If a new event is pending, dispatch it now.
1190 */
1191 rc = TRPMQueryTrapAll(pVCpu, &u8Vector, &enmType, &errCode, NULL, NULL);
1192 AssertRC(rc);
1193 Assert(pCtx->eflags.Bits.u1IF == 1 || enmType == TRPM_TRAP);
1194 Assert(enmType != TRPM_SOFTWARE_INT);
1195
1196 /*
1197 * Clear the pending trap.
1198 */
1199 rc = TRPMResetTrap(pVCpu);
1200 AssertRC(rc);
1201
1202 intInfo = u8Vector;
1203 intInfo |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
1204
1205 if (enmType == TRPM_TRAP)
1206 {
1207 switch (u8Vector)
1208 {
1209 case X86_XCPT_DF:
1210 case X86_XCPT_TS:
1211 case X86_XCPT_NP:
1212 case X86_XCPT_SS:
1213 case X86_XCPT_GP:
1214 case X86_XCPT_PF:
1215 case X86_XCPT_AC:
1216 {
1217 /** @todo r=ramshankar: setting this bit would blow up for real-mode guests with
1218 * unrestricted guest execution. */
1219 /* Valid error codes. */
1220 intInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
1221 break;
1222 }
1223
1224 default:
1225 break;
1226 }
1227
1228 if ( u8Vector == X86_XCPT_BP
1229 || u8Vector == X86_XCPT_OF)
1230 {
1231 intInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
1232 }
1233 else
1234 intInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
1235 }
1236 else
1237 intInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
1238
1239 STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
1240 rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo, 0, errCode);
1241 AssertRC(rc);
1242 } /* if (interrupts can be dispatched) */
1243
1244 return VINF_SUCCESS;
1245}
1246
1247/**
1248 * Checks for pending VMX events and converts them to TRPM. Before we execute any instruction
1249 * outside of VMX, any pending VMX event must be converted so that it can be delivered properly.
1250 *
1251 * @returns VBox status code.
1252 * @param pVCpu Pointer to the VMCPU.
1253 */
1254static int hmR0VmxCheckPendingEvent(PVMCPU pVCpu)
1255{
1256 if (pVCpu->hm.s.Event.fPending)
1257 {
1258 TRPMEVENT enmTrapType;
1259
1260 /* If a trap was already pending, we did something wrong! */
1261 Assert((TRPMQueryTrap(pVCpu, NULL, NULL) == VERR_TRPM_NO_ACTIVE_TRAP));
1262
1263 /*
1264 * Clear the pending event and move it over to TRPM for the rest
1265 * of the world to see.
1266 */
1267 pVCpu->hm.s.Event.fPending = false;
1268 switch (VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo))
1269 {
1270 case VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT:
1271 case VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI:
1272 enmTrapType = TRPM_HARDWARE_INT;
1273 break;
1274 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT:
1275 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /** @todo Is classifying #BP, #OF as TRPM_SOFTWARE_INT correct? */
1276 case VMX_EXIT_INTERRUPTION_INFO_TYPE_DB_XCPT:
1277 enmTrapType = TRPM_SOFTWARE_INT;
1278 break;
1279 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
1280 enmTrapType = TRPM_TRAP;
1281 break;
1282 default:
1283 enmTrapType = TRPM_32BIT_HACK; /* Can't get here. */
1284 AssertFailed();
1285 }
1286 TRPMAssertTrap(pVCpu, VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVCpu->hm.s.Event.u64IntrInfo), enmTrapType);
1287 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntrInfo))
1288 TRPMSetErrorCode(pVCpu, pVCpu->hm.s.Event.u32ErrCode);
1289 //@todo: Is there any situation where we need to call TRPMSetFaultAddress()?
1290 }
1291 return VINF_SUCCESS;
1292}
1293
1294/**
1295 * Save the host state into the VMCS.
1296 *
1297 * @returns VBox status code.
1298 * @param pVM Pointer to the VM.
1299 * @param pVCpu Pointer to the VMCPU.
1300 */
1301VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
1302{
1303 int rc = VINF_SUCCESS;
1304 NOREF(pVM);
1305
1306 /*
1307 * Host CPU Context.
1308 */
1309 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT)
1310 {
1311 RTIDTR idtr;
1312 RTGDTR gdtr;
1313 RTSEL SelTR;
1314 PCX86DESCHC pDesc;
1315 uintptr_t trBase;
1316 RTSEL cs;
1317 RTSEL ss;
1318 uint64_t cr3;
1319
1320 /*
1321 * Control registers.
1322 */
1323 rc = VMXWriteVmcs(VMX_VMCS_HOST_CR0, ASMGetCR0());
1324 Log2(("VMX_VMCS_HOST_CR0 %08x\n", ASMGetCR0()));
1325#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1326 if (VMX_IS_64BIT_HOST_MODE())
1327 {
1328 cr3 = HMR0Get64bitCR3();
1329 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_CR3, cr3);
1330 }
1331 else
1332#endif
1333 {
1334 cr3 = ASMGetCR3();
1335 rc |= VMXWriteVmcs(VMX_VMCS_HOST_CR3, cr3);
1336 }
1337 Log2(("VMX_VMCS_HOST_CR3 %08RX64\n", cr3));
1338 rc |= VMXWriteVmcs(VMX_VMCS_HOST_CR4, ASMGetCR4());
1339 Log2(("VMX_VMCS_HOST_CR4 %08x\n", ASMGetCR4()));
1340 AssertRC(rc);
1341
1342 /*
1343 * Selector registers.
1344 */
1345#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1346 if (VMX_IS_64BIT_HOST_MODE())
1347 {
1348 cs = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
1349 ss = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
1350 }
1351 else
1352 {
1353 /* sysenter loads LDT cs & ss, VMX doesn't like this. Load the GDT ones (safe). */
1354 cs = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
1355 ss = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
1356 }
1357#else
1358 cs = ASMGetCS();
1359 ss = ASMGetSS();
1360#endif
1361 Assert(!(cs & X86_SEL_LDT)); Assert((cs & X86_SEL_RPL) == 0);
1362 Assert(!(ss & X86_SEL_LDT)); Assert((ss & X86_SEL_RPL) == 0);
1363 rc = VMXWriteVmcs(VMX_VMCS16_HOST_FIELD_CS, cs);
1364 /* Note: VMX is (again) very picky about the RPL of the selectors here; we'll restore them manually. */
1365 rc |= VMXWriteVmcs(VMX_VMCS16_HOST_FIELD_DS, 0);
1366 rc |= VMXWriteVmcs(VMX_VMCS16_HOST_FIELD_ES, 0);
1367#if HC_ARCH_BITS == 32
1368 if (!VMX_IS_64BIT_HOST_MODE())
1369 {
1370 rc |= VMXWriteVmcs(VMX_VMCS16_HOST_FIELD_FS, 0);
1371 rc |= VMXWriteVmcs(VMX_VMCS16_HOST_FIELD_GS, 0);
1372 }
1373#endif
1374 rc |= VMXWriteVmcs(VMX_VMCS16_HOST_FIELD_SS, ss);
1375 SelTR = ASMGetTR();
1376 rc |= VMXWriteVmcs(VMX_VMCS16_HOST_FIELD_TR, SelTR);
1377 AssertRC(rc);
1378 Log2(("VMX_VMCS_HOST_FIELD_CS %08x (%08x)\n", cs, ASMGetSS()));
1379 Log2(("VMX_VMCS_HOST_FIELD_DS 00000000 (%08x)\n", ASMGetDS()));
1380 Log2(("VMX_VMCS_HOST_FIELD_ES 00000000 (%08x)\n", ASMGetES()));
1381 Log2(("VMX_VMCS_HOST_FIELD_FS 00000000 (%08x)\n", ASMGetFS()));
1382 Log2(("VMX_VMCS_HOST_FIELD_GS 00000000 (%08x)\n", ASMGetGS()));
1383 Log2(("VMX_VMCS_HOST_FIELD_SS %08x (%08x)\n", ss, ASMGetSS()));
1384 Log2(("VMX_VMCS_HOST_FIELD_TR %08x\n", ASMGetTR()));
1385
1386 /*
1387 * GDTR & IDTR.
1388 */
1389#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1390 if (VMX_IS_64BIT_HOST_MODE())
1391 {
1392 X86XDTR64 gdtr64, idtr64;
1393 HMR0Get64bitGdtrAndIdtr(&gdtr64, &idtr64);
1394 rc = VMXWriteVmcs64(VMX_VMCS_HOST_GDTR_BASE, gdtr64.uAddr);
1395 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_IDTR_BASE, idtr64.uAddr);
1396 AssertRC(rc);
1397 Log2(("VMX_VMCS_HOST_GDTR_BASE %RX64\n", gdtr64.uAddr));
1398 Log2(("VMX_VMCS_HOST_IDTR_BASE %RX64\n", idtr64.uAddr));
1399 gdtr.cbGdt = gdtr64.cb;
1400 gdtr.pGdt = (uintptr_t)gdtr64.uAddr;
1401 }
1402 else
1403#endif
1404 {
1405 ASMGetGDTR(&gdtr);
1406 rc = VMXWriteVmcs(VMX_VMCS_HOST_GDTR_BASE, gdtr.pGdt);
1407 ASMGetIDTR(&idtr);
1408 rc |= VMXWriteVmcs(VMX_VMCS_HOST_IDTR_BASE, idtr.pIdt);
1409 AssertRC(rc);
1410 Log2(("VMX_VMCS_HOST_GDTR_BASE %RHv\n", gdtr.pGdt));
1411 Log2(("VMX_VMCS_HOST_IDTR_BASE %RHv\n", idtr.pIdt));
1412 }
1413
1414 /*
1415 * Save the base address of the TR selector.
1416 */
1417 if (SelTR > gdtr.cbGdt)
1418 {
1419 AssertMsgFailed(("Invalid TR selector %x. GDTR.cbGdt=%x\n", SelTR, gdtr.cbGdt));
1420 return VERR_VMX_INVALID_HOST_STATE;
1421 }
1422
1423 pDesc = (PCX86DESCHC)(gdtr.pGdt + (SelTR & X86_SEL_MASK));
1424#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1425 if (VMX_IS_64BIT_HOST_MODE())
1426 {
1427 uint64_t trBase64 = X86DESC64_BASE((PX86DESC64)pDesc);
1428 rc = VMXWriteVmcs64(VMX_VMCS_HOST_TR_BASE, trBase64);
1429 Log2(("VMX_VMCS_HOST_TR_BASE %RX64\n", trBase64));
1430 AssertRC(rc);
1431 }
1432 else
1433#endif
1434 {
1435#if HC_ARCH_BITS == 64
1436 trBase = X86DESC64_BASE(pDesc);
1437#else
1438 trBase = X86DESC_BASE(pDesc);
1439#endif
1440 rc = VMXWriteVmcs(VMX_VMCS_HOST_TR_BASE, trBase);
1441 AssertRC(rc);
1442 Log2(("VMX_VMCS_HOST_TR_BASE %RHv\n", trBase));
1443 }
1444
1445 /*
1446 * FS base and GS base.
1447 */
1448#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1449 if (VMX_IS_64BIT_HOST_MODE())
1450 {
1451 Log2(("MSR_K8_FS_BASE = %RX64\n", ASMRdMsr(MSR_K8_FS_BASE)));
1452 Log2(("MSR_K8_GS_BASE = %RX64\n", ASMRdMsr(MSR_K8_GS_BASE)));
1453 rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, ASMRdMsr(MSR_K8_FS_BASE));
1454 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, ASMRdMsr(MSR_K8_GS_BASE));
1455 }
1456#endif
1457 AssertRC(rc);
1458
1459 /*
1460 * Sysenter MSRs.
1461 */
1462 /** @todo expensive!! */
1463 rc = VMXWriteVmcs(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
1464 Log2(("VMX_VMCS_HOST_SYSENTER_CS %08x\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_CS)));
1465#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1466 if (VMX_IS_64BIT_HOST_MODE())
1467 {
1468 Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_EIP)));
1469 Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_ESP)));
1470 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
1471 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
1472 }
1473 else
1474 {
1475 rc |= VMXWriteVmcs(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
1476 rc |= VMXWriteVmcs(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
1477 Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP)));
1478 Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP)));
1479 }
1480#elif HC_ARCH_BITS == 32
1481 rc |= VMXWriteVmcs(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
1482 rc |= VMXWriteVmcs(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
1483 Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP)));
1484 Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP)));
1485#else
1486 Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_EIP)));
1487 Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_ESP)));
1488 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
1489 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
1490#endif
1491 AssertRC(rc);
1492
1493
1494#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
1495 /*
1496 * Store all host MSRs in the VM-Exit load area, so they will be reloaded after
1497 * the world switch back to the host.
1498 */
1499 PVMXMSR pMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvHostMsr;
1500 unsigned idxMsr = 0;
1501
1502 uint32_t u32HostExtFeatures = ASMCpuId_EDX(0x80000001);
1503 if (u32HostExtFeatures & (X86_CPUID_EXT_FEATURE_EDX_NX | X86_CPUID_EXT_FEATURE_EDX_LONG_MODE))
1504 {
1505 pMsr->u32IndexMSR = MSR_K6_EFER;
1506 pMsr->u32Reserved = 0;
1507# if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1508 if (CPUMIsGuestInLongMode(pVCpu))
1509 {
1510 /* Must match the EFER value in our 64 bits switcher. */
1511 pMsr->u64Value = ASMRdMsr(MSR_K6_EFER) | MSR_K6_EFER_LME | MSR_K6_EFER_SCE | MSR_K6_EFER_NXE;
1512 }
1513 else
1514# endif
1515 pMsr->u64Value = ASMRdMsr(MSR_K6_EFER);
1516 pMsr++; idxMsr++;
1517 }
1518
1519# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1520 if (VMX_IS_64BIT_HOST_MODE())
1521 {
1522 pMsr->u32IndexMSR = MSR_K6_STAR;
1523 pMsr->u32Reserved = 0;
1524 pMsr->u64Value = ASMRdMsr(MSR_K6_STAR); /* legacy syscall eip, cs & ss */
1525 pMsr++; idxMsr++;
1526 pMsr->u32IndexMSR = MSR_K8_LSTAR;
1527 pMsr->u32Reserved = 0;
1528 pMsr->u64Value = ASMRdMsr(MSR_K8_LSTAR); /* 64 bits mode syscall rip */
1529 pMsr++; idxMsr++;
1530 pMsr->u32IndexMSR = MSR_K8_SF_MASK;
1531 pMsr->u32Reserved = 0;
1532 pMsr->u64Value = ASMRdMsr(MSR_K8_SF_MASK); /* syscall flag mask */
1533 pMsr++; idxMsr++;
1534# if 0
1535 /* The KERNEL_GS_BASE MSR does not work reliably with auto load/store. See @bugref{6208} */
1536 pMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
1537 pMsr->u32Reserved = 0;
1538 pMsr->u64Value = ASMRdMsr(MSR_K8_KERNEL_GS_BASE); /* swapgs exchange value */
1539 pMsr++; idxMsr++;
1540# endif
1541 }
1542# endif
1543
1544 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
1545 {
1546 pMsr->u32IndexMSR = MSR_K8_TSC_AUX;
1547 pMsr->u32Reserved = 0;
1548 pMsr->u64Value = ASMRdMsr(MSR_K8_TSC_AUX);
1549 pMsr++; idxMsr++;
1550 }
1551
1552 /** @todo r=ramshankar: check IA32_VMX_MISC bits 27:25 for valid idxMsr
1553 * range. */
1554 rc = VMXWriteVmcs(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, idxMsr);
1555 AssertRC(rc);
1556#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
1557
1558 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_HOST_CONTEXT;
1559 }
1560 return rc;
1561}
1562
1563
1564/**
1565 * Loads the 4 PDPEs into the guest state when nested paging is used and the
1566 * guest operates in PAE mode.
1567 *
1568 * @returns VBox status code.
1569 * @param pVCpu Pointer to the VMCPU.
1570 * @param pCtx Pointer to the guest CPU context.
1571 */
1572static int hmR0VmxLoadPaePdpes(PVMCPU pVCpu, PCPUMCTX pCtx)
1573{
1574 if (CPUMIsGuestInPAEModeEx(pCtx))
1575 {
1576 X86PDPE aPdpes[4];
1577 int rc = PGMGstGetPaePdpes(pVCpu, &aPdpes[0]);
1578 AssertRCReturn(rc, rc);
1579
1580 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, aPdpes[0].u); AssertRCReturn(rc, rc);
1581 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, aPdpes[1].u); AssertRCReturn(rc, rc);
1582 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, aPdpes[2].u); AssertRCReturn(rc, rc);
1583 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, aPdpes[3].u); AssertRCReturn(rc, rc);
1584 }
1585 return VINF_SUCCESS;
1586}
1587
1588
1589/**
1590 * Saves the 4 PDPEs into the guest state when nested paging is used and the
1591 * guest operates in PAE mode.
1592 *
1593 * @returns VBox status code.
1594 * @param pVCpu Pointer to the VM CPU.
1595 * @param pCtx Pointer to the guest CPU context.
1596 *
1597 * @remarks Tell PGM about CR3 changes before calling this helper.
1598 */
1599static int hmR0VmxSavePaePdpes(PVMCPU pVCpu, PCPUMCTX pCtx)
1600{
1601 if (CPUMIsGuestInPAEModeEx(pCtx))
1602 {
1603 int rc;
1604 X86PDPE aPdpes[4];
1605 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &aPdpes[0].u); AssertRCReturn(rc, rc);
1606 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &aPdpes[1].u); AssertRCReturn(rc, rc);
1607 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &aPdpes[2].u); AssertRCReturn(rc, rc);
1608 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &aPdpes[3].u); AssertRCReturn(rc, rc);
1609
1610 rc = PGMGstUpdatePaePdpes(pVCpu, &aPdpes[0]);
1611 AssertRCReturn(rc, rc);
1612 }
1613 return VINF_SUCCESS;
1614}
1615
1616
1617/**
1618 * Update the exception bitmap according to the current CPU state.
1619 *
1620 * @param pVM Pointer to the VM.
1621 * @param pVCpu Pointer to the VMCPU.
1622 * @param pCtx Pointer to the guest CPU context.
1623 */
1624static void hmR0VmxUpdateExceptionBitmap(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1625{
1626 uint32_t u32TrapMask;
1627 Assert(pCtx);
1628
1629 /*
1630 * Set up a mask for intercepting traps.
1631 */
1632 /** @todo Do we really need to always intercept #DB? */
1633 u32TrapMask = RT_BIT(X86_XCPT_DB)
1634 | RT_BIT(X86_XCPT_NM)
1635#ifdef VBOX_ALWAYS_TRAP_PF
1636 | RT_BIT(X86_XCPT_PF)
1637#endif
1638#ifdef VBOX_STRICT
1639 | RT_BIT(X86_XCPT_BP)
1640 | RT_BIT(X86_XCPT_DB)
1641 | RT_BIT(X86_XCPT_DE)
1642 | RT_BIT(X86_XCPT_NM)
1643 | RT_BIT(X86_XCPT_UD)
1644 | RT_BIT(X86_XCPT_NP)
1645 | RT_BIT(X86_XCPT_SS)
1646 | RT_BIT(X86_XCPT_GP)
1647 | RT_BIT(X86_XCPT_MF)
1648#endif
1649 ;
1650
1651 /*
1652 * Without nested paging, #PF must be intercepted to implement shadow paging.
1653 */
1654 /** @todo NP state won't change so maybe we should build the initial trap mask up front? */
1655 if (!pVM->hm.s.fNestedPaging)
1656 u32TrapMask |= RT_BIT(X86_XCPT_PF);
1657
1658 /* Catch floating point exceptions if we need to report them to the guest in a different way. */
1659 if (!(pCtx->cr0 & X86_CR0_NE))
1660 u32TrapMask |= RT_BIT(X86_XCPT_MF);
1661
1662#ifdef VBOX_STRICT
1663 Assert(u32TrapMask & RT_BIT(X86_XCPT_GP));
1664#endif
1665
1666 /*
1667 * Intercept all exceptions in real mode as none of them can be injected directly (#GP otherwise).
1668 */
1669 /** @todo Despite the claim to intercept everything, with NP we do not intercept #PF. Should we? */
1670 if ( CPUMIsGuestInRealModeEx(pCtx)
1671 && pVM->hm.s.vmx.pRealModeTSS)
1672 {
1673 u32TrapMask |= RT_BIT(X86_XCPT_DE)
1674 | RT_BIT(X86_XCPT_DB)
1675 | RT_BIT(X86_XCPT_NMI)
1676 | RT_BIT(X86_XCPT_BP)
1677 | RT_BIT(X86_XCPT_OF)
1678 | RT_BIT(X86_XCPT_BR)
1679 | RT_BIT(X86_XCPT_UD)
1680 | RT_BIT(X86_XCPT_DF)
1681 | RT_BIT(X86_XCPT_CO_SEG_OVERRUN)
1682 | RT_BIT(X86_XCPT_TS)
1683 | RT_BIT(X86_XCPT_NP)
1684 | RT_BIT(X86_XCPT_SS)
1685 | RT_BIT(X86_XCPT_GP)
1686 | RT_BIT(X86_XCPT_MF)
1687 | RT_BIT(X86_XCPT_AC)
1688 | RT_BIT(X86_XCPT_MC)
1689 | RT_BIT(X86_XCPT_XF)
1690 ;
1691 }
1692
1693 int rc = VMXWriteVmcs(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32TrapMask);
1694 AssertRC(rc);
1695}
1696
1697
1698/**
1699 * Loads a minimal guest state.
1700 *
1701 * NOTE: Don't do anything here that can cause a jump back to ring 3!!!!!
1702 *
1703 * @param pVM Pointer to the VM.
1704 * @param pVCpu Pointer to the VMCPU.
1705 * @param pCtx Pointer to the guest CPU context.
1706 */
1707VMMR0DECL(void) VMXR0LoadMinimalGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1708{
1709 int rc;
1710 X86EFLAGS eflags;
1711
1712 Assert(!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_ALL_GUEST));
1713
1714 /*
1715 * Load EIP, ESP and EFLAGS.
1716 */
1717 rc = VMXWriteVmcs64(VMX_VMCS_GUEST_RIP, pCtx->rip);
1718 rc |= VMXWriteVmcs64(VMX_VMCS_GUEST_RSP, pCtx->rsp);
1719 AssertRC(rc);
1720
1721 /*
1722 * Bits 22-31, 15, 5 & 3 must be zero. Bit 1 must be 1.
1723 */
1724 eflags = pCtx->eflags;
1725 eflags.u32 &= VMX_EFLAGS_RESERVED_0;
1726 eflags.u32 |= VMX_EFLAGS_RESERVED_1;
1727
1728 /*
1729 * Check if real mode emulation using v86 mode.
1730 */
1731 if ( CPUMIsGuestInRealModeEx(pCtx)
1732 && pVM->hm.s.vmx.pRealModeTSS)
1733 {
1734 pVCpu->hm.s.vmx.RealMode.eflags = eflags;
1735
1736 eflags.Bits.u1VM = 1;
1737 eflags.Bits.u2IOPL = 0; /* must always be 0 or else certain instructions won't cause faults. */
1738 }
1739 rc = VMXWriteVmcs(VMX_VMCS_GUEST_RFLAGS, eflags.u32);
1740 AssertRC(rc);
1741}
1742
1743
1744/**
1745 * Sets up TSC offsetting and VMX preemption, if supported/configured/available.
1746 *
1747 * NOTE: This function reads the host TSC value. Therefore it must be executed very
1748 * shortly before a VM entry and execution MUST NOT be rescheduled between a call to
1749 * this function and a VM entry without calling this function again.
1750 *
1751 * @returns VBox status code.
1752 * @param pVM Pointer to the VM.
1753 * @param pVCpu Pointer to the VMCPU.
1754 */
1755VMMR0DECL(int) VMXR0SetupTscOffsetAndPreemption(PVM pVM, PVMCPU pVCpu)
1756{
1757 int rc = VINF_SUCCESS;
1758 bool fOffsettedTsc;
1759
1760 if (pVM->hm.s.vmx.fUsePreemptTimer)
1761 {
1762 uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &pVCpu->hm.s.vmx.u64TSCOffset);
1763
1764 /* Make sure the returned values have sane upper and lower boundaries. */
1765 uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
1766
1767 cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64 of a second */
1768 cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
1769
1770 cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
1771 uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
1772 rc = VMXWriteVmcs(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount);
1773 AssertRC(rc);
1774 }
1775 else
1776 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset);
1777
1778 if (fOffsettedTsc)
1779 {
1780 uint64_t u64CurTSC = ASMReadTSC();
1781 if (u64CurTSC + pVCpu->hm.s.vmx.u64TSCOffset > TMCpuTickGetLastSeen(pVCpu))
1782 {
1783 /* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
1784 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset);
1785 AssertRC(rc);
1786
1787 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
1788 rc = VMXWriteVmcs(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
1789 AssertRC(rc);
1790 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
1791 }
1792 else
1793 {
1794 /* Fall back to rdtsc, rdtscp emulation as we would otherwise pass decreasing tsc values to the guest. */
1795 LogFlow(("TSC %RX64 offset %RX64 time=%RX64 last=%RX64 (diff=%RX64, virt_tsc=%RX64)\n", u64CurTSC,
1796 pVCpu->hm.s.vmx.u64TSCOffset, u64CurTSC + pVCpu->hm.s.vmx.u64TSCOffset,
1797 TMCpuTickGetLastSeen(pVCpu), TMCpuTickGetLastSeen(pVCpu) - u64CurTSC - pVCpu->hm.s.vmx.u64TSCOffset,
1798 TMCpuTickGet(pVCpu)));
1799 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
1800 rc = VMXWriteVmcs(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
1801 AssertRC(rc);
1802 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscInterceptOverFlow);
1803 }
1804 }
1805 else
1806 {
1807 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
1808 rc = VMXWriteVmcs(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
1809 AssertRC(rc);
1810 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
1811 }
1812 return rc;
1813}
1814
1815/**
1816 * Loads the guest state.
1817 *
1818 * NOTE: Don't do anything here that can cause a jump back to ring 3!!!!!
1819 *
1820 * @returns VBox status code.
1821 * @param pVM Pointer to the VM.
1822 * @param pVCpu Pointer to the VMCPU.
1823 * @param pCtx Pointer to the guest CPU context.
1824 */
1825VMMR0DECL(int) VMXR0LoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1826{
1827 int rc = VINF_SUCCESS;
1828 RTGCUINTPTR val;
1829
1830 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
1831
1832 /*
1833 * VMX_VMCS_CTRL_ENTRY
1834 * Set required bits to one and zero according to the MSR capabilities.
1835 */
1836 val = pVM->hm.s.vmx.msr.vmx_entry.n.disallowed0;
1837
1838 /*
1839 * Load guest debug controls (DR7 & IA32_DEBUGCTL_MSR).
1840 * Forced to 1 on the 'first' VT-x capable CPUs; this actually includes the newest Nehalem CPUs
1841 */
1842 val |= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
1843
1844 if (CPUMIsGuestInLongModeEx(pCtx))
1845 val |= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
1846 /* else Must be zero when AMD64 is not available. */
1847
1848 /*
1849 * Mask away the bits that the CPU doesn't support.
1850 */
1851 val &= pVM->hm.s.vmx.msr.vmx_entry.n.allowed1;
1852 rc = VMXWriteVmcs(VMX_VMCS32_CTRL_ENTRY, val);
1853 AssertRC(rc);
1854
1855 /*
1856 * VMX_VMCS_CTRL_EXIT
1857 * Set required bits to one and zero according to the MSR capabilities.
1858 */
1859 val = pVM->hm.s.vmx.msr.vmx_exit.n.disallowed0;
1860
1861 /*
1862 * Save debug controls (DR7 & IA32_DEBUGCTL_MSR)
1863 * Forced to 1 on the 'first' VT-x capable CPUs; this actually includes the newest Nehalem CPUs
1864 */
1865 val |= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
1866
1867#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1868 if (VMX_IS_64BIT_HOST_MODE())
1869 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
1870 /* else Must be zero when AMD64 is not available. */
1871#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
1872 if (CPUMIsGuestInLongModeEx(pCtx))
1873 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE; /* our switcher goes to long mode */
1874 else
1875 Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
1876#endif
1877 val &= pVM->hm.s.vmx.msr.vmx_exit.n.allowed1;
1878
1879 /*
1880 * Don't acknowledge external interrupts on VM-exit.
1881 */
1882 rc = VMXWriteVmcs(VMX_VMCS32_CTRL_EXIT, val);
1883 AssertRC(rc);
1884
1885 /*
1886 * Guest CPU context: ES, CS, SS, DS, FS, GS.
1887 */
1888 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SEGMENT_REGS)
1889 {
1890 if (pVM->hm.s.vmx.pRealModeTSS)
1891 {
1892 PGMMODE enmGuestMode = PGMGetGuestMode(pVCpu);
1893 if (pVCpu->hm.s.vmx.enmLastSeenGuestMode != enmGuestMode)
1894 {
1895 /*
1896 * Correct weird requirements for switching to protected mode.
1897 */
1898 if ( pVCpu->hm.s.vmx.enmLastSeenGuestMode == PGMMODE_REAL
1899 && enmGuestMode >= PGMMODE_PROTECTED)
1900 {
1901#ifdef VBOX_WITH_REM
1902 /*
1903 * Flush the recompiler code cache as it's not unlikely the guest will rewrite code
1904 * it will later execute in real mode (OpenBSD 4.0 is one such example)
1905 */
1906 REMFlushTBs(pVM);
1907#endif
1908
1909 /*
1910 * DPL of all hidden selector registers must match the current CPL (0).
1911 */
1912 pCtx->cs.Attr.n.u2Dpl = 0;
1913 pCtx->cs.Attr.n.u4Type = X86_SEL_TYPE_CODE | X86_SEL_TYPE_RW_ACC;
1914
1915 pCtx->ds.Attr.n.u2Dpl = 0;
1916 pCtx->es.Attr.n.u2Dpl = 0;
1917 pCtx->fs.Attr.n.u2Dpl = 0;
1918 pCtx->gs.Attr.n.u2Dpl = 0;
1919 pCtx->ss.Attr.n.u2Dpl = 0;
1920 }
1921 pVCpu->hm.s.vmx.enmLastSeenGuestMode = enmGuestMode;
1922 }
1923 }
1924
1925 VMX_WRITE_SELREG(ES, es);
1926 AssertRC(rc);
1927
1928 VMX_WRITE_SELREG(CS, cs);
1929 AssertRC(rc);
1930
1931 VMX_WRITE_SELREG(SS, ss);
1932 AssertRC(rc);
1933
1934 VMX_WRITE_SELREG(DS, ds);
1935 AssertRC(rc);
1936
1937 VMX_WRITE_SELREG(FS, fs);
1938 AssertRC(rc);
1939
1940 VMX_WRITE_SELREG(GS, gs);
1941 AssertRC(rc);
1942 }
1943
1944 /*
1945 * Guest CPU context: LDTR.
1946 */
1947 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_LDTR)
1948 {
1949 if (pCtx->ldtr.Sel == 0)
1950 {
1951 rc = VMXWriteVmcs(VMX_VMCS16_GUEST_FIELD_LDTR, 0);
1952 rc |= VMXWriteVmcs(VMX_VMCS32_GUEST_LDTR_LIMIT, 0);
1953 rc |= VMXWriteVmcs64(VMX_VMCS_GUEST_LDTR_BASE, 0); /* @todo removing "64" in the function should be the same. */
1954 /* Note: vmlaunch will fail with 0 or just 0x02. No idea why. */
1955 rc |= VMXWriteVmcs(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, 0x82 /* present, LDT */);
1956 }
1957 else
1958 {
1959 rc = VMXWriteVmcs(VMX_VMCS16_GUEST_FIELD_LDTR, pCtx->ldtr.Sel);
1960 rc |= VMXWriteVmcs(VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtr.u32Limit);
1961 rc |= VMXWriteVmcs64(VMX_VMCS_GUEST_LDTR_BASE, pCtx->ldtr.u64Base); /* @todo removing "64" and it should be the same */
1962 rc |= VMXWriteVmcs(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, pCtx->ldtr.Attr.u);
1963 }
1964 AssertRC(rc);
1965 }
1966
1967 /*
1968 * Guest CPU context: TR.
1969 */
1970 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_TR)
1971 {
1972 /*
1973 * Real mode emulation using v86 mode with CR4.VME (interrupt redirection
1974 * using the int bitmap in the TSS).
1975 */
1976 if ( CPUMIsGuestInRealModeEx(pCtx)
1977 && pVM->hm.s.vmx.pRealModeTSS)
1978 {
1979 RTGCPHYS GCPhys;
1980
1981 /* We convert it here every time as PCI regions could be reconfigured. */
1982 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
1983 AssertRC(rc);
1984
1985 rc = VMXWriteVmcs(VMX_VMCS16_GUEST_FIELD_TR, 0);
1986 rc |= VMXWriteVmcs(VMX_VMCS32_GUEST_TR_LIMIT, HM_VTX_TSS_SIZE);
1987 rc |= VMXWriteVmcs64(VMX_VMCS_GUEST_TR_BASE, GCPhys /* phys = virt in this mode */);
1988
1989 X86DESCATTR attr;
1990
1991 attr.u = 0;
1992 attr.n.u1Present = 1;
1993 attr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
1994 val = attr.u;
1995 }
1996 else
1997 {
1998 rc = VMXWriteVmcs(VMX_VMCS16_GUEST_FIELD_TR, pCtx->tr.Sel);
1999 rc |= VMXWriteVmcs(VMX_VMCS32_GUEST_TR_LIMIT, pCtx->tr.u32Limit);
2000 rc |= VMXWriteVmcs64(VMX_VMCS_GUEST_TR_BASE, pCtx->tr.u64Base);
2001
2002 val = pCtx->tr.Attr.u;
2003
2004 /* The TSS selector must be busy (REM bugs? see defect #XXXX). */
2005 if (!(val & X86_SEL_TYPE_SYS_TSS_BUSY_MASK))
2006 {
2007 if (val & 0xf)
2008 val |= X86_SEL_TYPE_SYS_TSS_BUSY_MASK;
2009 else
2010 /* Default if no TR selector has been set (otherwise vmlaunch will fail!) */
2011 val = (val & ~0xF) | X86_SEL_TYPE_SYS_386_TSS_BUSY;
2012 }
2013 AssertMsg((val & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY || (val & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY,
2014 ("%#x\n", val));
2015 }
2016 rc |= VMXWriteVmcs(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, val);
2017 AssertRC(rc);
2018 }
2019
2020 /*
2021 * Guest CPU context: GDTR.
2022 */
2023 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_GDTR)
2024 {
2025 rc = VMXWriteVmcs(VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt);
2026 rc |= VMXWriteVmcs64(VMX_VMCS_GUEST_GDTR_BASE, pCtx->gdtr.pGdt);
2027 AssertRC(rc);
2028 }
2029
2030 /*
2031 * Guest CPU context: IDTR.
2032 */
2033 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_IDTR)
2034 {
2035 rc = VMXWriteVmcs(VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt);
2036 rc |= VMXWriteVmcs64(VMX_VMCS_GUEST_IDTR_BASE, pCtx->idtr.pIdt);
2037 AssertRC(rc);
2038 }
2039
2040 /*
2041 * Sysenter MSRs.
2042 */
2043 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_MSR)
2044 {
2045 rc = VMXWriteVmcs(VMX_VMCS32_GUEST_SYSENTER_CS, pCtx->SysEnter.cs);
2046 rc |= VMXWriteVmcs64(VMX_VMCS_GUEST_SYSENTER_EIP, pCtx->SysEnter.eip);
2047 rc |= VMXWriteVmcs64(VMX_VMCS_GUEST_SYSENTER_ESP, pCtx->SysEnter.esp);
2048 AssertRC(rc);
2049 }
2050
2051 /*
2052 * Guest CPU context: Control registers.
2053 */
2054 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR0)
2055 {
2056 val = pCtx->cr0;
2057 rc = VMXWriteVmcs(VMX_VMCS_CTRL_CR0_READ_SHADOW, val);
2058 Log2(("Guest CR0-shadow %08x\n", val));
2059 if (CPUMIsGuestFPUStateActive(pVCpu) == false)
2060 {
2061 /* Always use #NM exceptions to load the FPU/XMM state on demand. */
2062 val |= X86_CR0_TS | X86_CR0_ET | X86_CR0_NE | X86_CR0_MP;
2063 }
2064 else
2065 {
2066 /** @todo check if we support the old style mess correctly. */
2067 if (!(val & X86_CR0_NE))
2068 Log(("Forcing X86_CR0_NE!!!\n"));
2069
2070 val |= X86_CR0_NE; /* always turn on the native mechanism to report FPU errors (old style uses interrupts) */
2071 }
2072 /* Protected mode & paging are always enabled; we use them for emulating real and protected mode without paging too. */
2073 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
2074 val |= X86_CR0_PE | X86_CR0_PG;
2075
2076 if (pVM->hm.s.fNestedPaging)
2077 {
2078 if (CPUMIsGuestInPagedProtectedModeEx(pCtx))
2079 {
2080 /* Disable CR3 read/write monitoring as we don't need it for EPT. */
2081 pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2082 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
2083 }
2084 else
2085 {
2086 /* Reenable CR3 read/write monitoring as our identity mapped page table is active. */
2087 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2088 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2089 }
2090 rc = VMXWriteVmcs(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
2091 AssertRC(rc);
2092 }
2093 else
2094 {
2095 /* Note: We must also set this as we rely on protecting various pages for which supervisor writes must be caught. */
2096 val |= X86_CR0_WP;
2097 }
2098
2099 /* Always enable caching. */
2100 val &= ~(X86_CR0_CD|X86_CR0_NW);
2101
2102 rc |= VMXWriteVmcs64(VMX_VMCS_GUEST_CR0, val);
2103 Log2(("Guest CR0 %08x\n", val));
2104
2105 /*
2106 * CR0 flags owned by the host; if the guests attempts to change them, then the VM will exit.
2107 */
2108 val = X86_CR0_PE /* Must monitor this bit (assumptions are made for real mode emulation) */
2109 | X86_CR0_WP /* Must monitor this bit (it must always be enabled). */
2110 | X86_CR0_PG /* Must monitor this bit (assumptions are made for real mode & protected mode without paging emulation) */
2111 | X86_CR0_CD /* Bit not restored during VM-exit! */
2112 | X86_CR0_NW /* Bit not restored during VM-exit! */
2113 | X86_CR0_NE;
2114
2115 /*
2116 * When the guest's FPU state is active, then we no longer care about the FPU related bits.
2117 */
2118 if (CPUMIsGuestFPUStateActive(pVCpu) == false)
2119 val |= X86_CR0_TS | X86_CR0_ET | X86_CR0_MP;
2120
2121 pVCpu->hm.s.vmx.u32CR0Mask = val;
2122
2123 rc |= VMXWriteVmcs(VMX_VMCS_CTRL_CR0_MASK, val);
2124 Log2(("Guest CR0-mask %08x\n", val));
2125 AssertRC(rc);
2126 }
2127
2128 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR4)
2129 {
2130 rc = VMXWriteVmcs(VMX_VMCS_CTRL_CR4_READ_SHADOW, pCtx->cr4);
2131 Log2(("Guest CR4-shadow %08x\n", pCtx->cr4));
2132 /* Set the required bits in cr4 too (currently X86_CR4_VMXE). */
2133 val = pCtx->cr4 | (uint32_t)pVM->hm.s.vmx.msr.vmx_cr4_fixed0;
2134
2135 if (!pVM->hm.s.fNestedPaging)
2136 {
2137 switch (pVCpu->hm.s.enmShadowMode)
2138 {
2139 case PGMMODE_REAL: /* Real mode -> emulated using v86 mode */
2140 case PGMMODE_PROTECTED: /* Protected mode, no paging -> emulated using identity mapping. */
2141 case PGMMODE_32_BIT: /* 32-bit paging. */
2142 val &= ~X86_CR4_PAE;
2143 break;
2144
2145 case PGMMODE_PAE: /* PAE paging. */
2146 case PGMMODE_PAE_NX: /* PAE paging with NX enabled. */
2147 /** Must use PAE paging as we could use physical memory > 4 GB */
2148 val |= X86_CR4_PAE;
2149 break;
2150
2151 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
2152 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
2153#ifdef VBOX_ENABLE_64_BITS_GUESTS
2154 break;
2155#else
2156 AssertFailed();
2157 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
2158#endif
2159 default: /* shut up gcc */
2160 AssertFailed();
2161 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
2162 }
2163 }
2164 else if ( !CPUMIsGuestInPagedProtectedModeEx(pCtx)
2165 && !pVM->hm.s.vmx.fUnrestrictedGuest)
2166 {
2167 /* We use 4 MB pages in our identity mapping page table for real and protected mode without paging. */
2168 val |= X86_CR4_PSE;
2169 /* Our identity mapping is a 32 bits page directory. */
2170 val &= ~X86_CR4_PAE;
2171 }
2172
2173 /*
2174 * Turn off VME if we're in emulated real mode.
2175 */
2176 if ( CPUMIsGuestInRealModeEx(pCtx)
2177 && pVM->hm.s.vmx.pRealModeTSS)
2178 {
2179 val &= ~X86_CR4_VME;
2180 }
2181
2182 rc |= VMXWriteVmcs64(VMX_VMCS_GUEST_CR4, val);
2183 Log2(("Guest CR4 %08x\n", val));
2184
2185 /*
2186 * CR4 flags owned by the host; if the guests attempts to change them, then the VM will exit.
2187 */
2188 val = 0
2189 | X86_CR4_VME
2190 | X86_CR4_PAE
2191 | X86_CR4_PGE
2192 | X86_CR4_PSE
2193 | X86_CR4_VMXE;
2194 pVCpu->hm.s.vmx.u32CR4Mask = val;
2195
2196 rc |= VMXWriteVmcs(VMX_VMCS_CTRL_CR4_MASK, val);
2197 Log2(("Guest CR4-mask %08x\n", val));
2198 AssertRC(rc);
2199 }
2200
2201#if 0
2202 /* Enable single stepping if requested and CPU supports it. */
2203 if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
2204 if (DBGFIsStepping(pVCpu))
2205 {
2206 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
2207 rc = VMXWriteVmcs(VMX_VMCS_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
2208 AssertRC(rc);
2209 }
2210#endif
2211
2212 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR3)
2213 {
2214 if (pVM->hm.s.fNestedPaging)
2215 {
2216 Assert(PGMGetHyperCR3(pVCpu));
2217 pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
2218
2219 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
2220 /** @todo Check the IA32_VMX_EPT_VPID_CAP MSR for other supported memory types. */
2221 pVCpu->hm.s.vmx.HCPhysEPTP |= VMX_EPT_MEMTYPE_WB
2222 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
2223
2224 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
2225 AssertRC(rc);
2226
2227 if ( !CPUMIsGuestInPagedProtectedModeEx(pCtx)
2228 && !pVM->hm.s.vmx.fUnrestrictedGuest)
2229 {
2230 RTGCPHYS GCPhys;
2231
2232 /* We convert it here every time as PCI regions could be reconfigured. */
2233 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
2234 AssertMsgRC(rc, ("pNonPagingModeEPTPageTable = %RGv\n", pVM->hm.s.vmx.pNonPagingModeEPTPageTable));
2235
2236 /*
2237 * We use our identity mapping page table here as we need to map guest virtual to
2238 * guest physical addresses; EPT will take care of the translation to host physical addresses.
2239 */
2240 val = GCPhys;
2241 }
2242 else
2243 {
2244 /* Save the real guest CR3 in VMX_VMCS_GUEST_CR3 */
2245 val = pCtx->cr3;
2246 rc = hmR0VmxLoadPaePdpes(pVCpu, pCtx);
2247 AssertRCReturn(rc, rc);
2248 }
2249 }
2250 else
2251 {
2252 val = PGMGetHyperCR3(pVCpu);
2253 Assert(val || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL));
2254 }
2255
2256 /* Save our shadow CR3 register. */
2257 rc = VMXWriteVmcs64(VMX_VMCS_GUEST_CR3, val);
2258 AssertRC(rc);
2259 }
2260
2261 /*
2262 * Guest CPU context: Debug registers.
2263 */
2264 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_DEBUG)
2265 {
2266 pCtx->dr[6] |= X86_DR6_INIT_VAL; /* set all reserved bits to 1. */
2267 pCtx->dr[6] &= ~RT_BIT(12); /* must be zero. */
2268
2269 pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
2270 pCtx->dr[7] &= ~(RT_BIT(11) | RT_BIT(12) | RT_BIT(14) | RT_BIT(15)); /* must be zero */
2271 pCtx->dr[7] |= 0x400; /* must be one */
2272
2273 /* Resync DR7 */
2274 rc = VMXWriteVmcs64(VMX_VMCS_GUEST_DR7, pCtx->dr[7]);
2275 AssertRC(rc);
2276
2277#ifdef DEBUG
2278 /* Sync the hypervisor debug state now if any breakpoint is armed. */
2279 if ( CPUMGetHyperDR7(pVCpu) & (X86_DR7_ENABLED_MASK|X86_DR7_GD)
2280 && !CPUMIsHyperDebugStateActive(pVCpu)
2281 && !DBGFIsStepping(pVCpu))
2282 {
2283 /* Save the host and load the hypervisor debug state. */
2284 rc = CPUMR0LoadHyperDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
2285 AssertRC(rc);
2286
2287 /* DRx intercepts remain enabled. */
2288
2289 /* Override dr7 with the hypervisor value. */
2290 rc = VMXWriteVmcs64(VMX_VMCS_GUEST_DR7, CPUMGetHyperDR7(pVCpu));
2291 AssertRC(rc);
2292 }
2293 else
2294#endif
2295 /* Sync the debug state now if any breakpoint is armed. */
2296 if ( (pCtx->dr[7] & (X86_DR7_ENABLED_MASK|X86_DR7_GD))
2297 && !CPUMIsGuestDebugStateActive(pVCpu)
2298 && !DBGFIsStepping(pVCpu))
2299 {
2300 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
2301
2302 /* Disable DRx move intercepts. */
2303 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
2304 rc = VMXWriteVmcs(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
2305 AssertRC(rc);
2306
2307 /* Save the host and load the guest debug state. */
2308 rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
2309 AssertRC(rc);
2310 }
2311
2312 /* IA32_DEBUGCTL MSR. */
2313 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0);
2314 AssertRC(rc);
2315
2316 /** @todo do we really ever need this? */
2317 rc |= VMXWriteVmcs(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2318 AssertRC(rc);
2319 }
2320
2321 /*
2322 * 64-bit guest mode.
2323 */
2324 if (CPUMIsGuestInLongModeEx(pCtx))
2325 {
2326#if !defined(VBOX_ENABLE_64_BITS_GUESTS)
2327 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
2328#elif HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2329 pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
2330#else
2331# ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2332 if (!pVM->hm.s.fAllow64BitGuests)
2333 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
2334# endif
2335 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
2336#endif
2337 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_MSR)
2338 {
2339 /* Update these as wrmsr might have changed them. */
2340 rc = VMXWriteVmcs64(VMX_VMCS_GUEST_FS_BASE, pCtx->fs.u64Base);
2341 AssertRC(rc);
2342 rc = VMXWriteVmcs64(VMX_VMCS_GUEST_GS_BASE, pCtx->gs.u64Base);
2343 AssertRC(rc);
2344 }
2345 }
2346 else
2347 {
2348 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
2349 }
2350
2351 hmR0VmxUpdateExceptionBitmap(pVM, pVCpu, pCtx);
2352
2353#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
2354 /*
2355 * Store all guest MSRs in the VM-entry load area, so they will be loaded
2356 * during VM-entry and restored into the VM-exit store area during VM-exit.
2357 */
2358 PVMXMSR pMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvGuestMsr;
2359 unsigned idxMsr = 0;
2360
2361 uint32_t u32GstExtFeatures;
2362 uint32_t u32Temp;
2363 CPUMGetGuestCpuId(pVCpu, 0x80000001, &u32Temp, &u32Temp, &u32Temp, &u32GstExtFeatures);
2364
2365 if (u32GstExtFeatures & (X86_CPUID_EXT_FEATURE_EDX_NX | X86_CPUID_EXT_FEATURE_EDX_LONG_MODE))
2366 {
2367 pMsr->u32IndexMSR = MSR_K6_EFER;
2368 pMsr->u32Reserved = 0;
2369 pMsr->u64Value = pCtx->msrEFER;
2370 /* VT-x will complain if only MSR_K6_EFER_LME is set. */
2371 if (!CPUMIsGuestInLongModeEx(pCtx))
2372 pMsr->u64Value &= ~(MSR_K6_EFER_LMA | MSR_K6_EFER_LME);
2373 pMsr++; idxMsr++;
2374
2375 if (u32GstExtFeatures & X86_CPUID_EXT_FEATURE_EDX_LONG_MODE)
2376 {
2377 pMsr->u32IndexMSR = MSR_K8_LSTAR;
2378 pMsr->u32Reserved = 0;
2379 pMsr->u64Value = pCtx->msrLSTAR; /* 64 bits mode syscall rip */
2380 pMsr++; idxMsr++;
2381 pMsr->u32IndexMSR = MSR_K6_STAR;
2382 pMsr->u32Reserved = 0;
2383 pMsr->u64Value = pCtx->msrSTAR; /* legacy syscall eip, cs & ss */
2384 pMsr++; idxMsr++;
2385 pMsr->u32IndexMSR = MSR_K8_SF_MASK;
2386 pMsr->u32Reserved = 0;
2387 pMsr->u64Value = pCtx->msrSFMASK; /* syscall flag mask */
2388 pMsr++; idxMsr++;
2389#if 0
2390 /* The KERNEL_GS_BASE MSR does not work reliably with auto load/store. See @bugref{6208} */
2391 pMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
2392 pMsr->u32Reserved = 0;
2393 pMsr->u64Value = pCtx->msrKERNELGSBASE; /* swapgs exchange value */
2394 pMsr++; idxMsr++;
2395#endif
2396 }
2397 }
2398
2399 if ( pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP
2400 && (u32GstExtFeatures & X86_CPUID_EXT_FEATURE_EDX_RDTSCP))
2401 {
2402 pMsr->u32IndexMSR = MSR_K8_TSC_AUX;
2403 pMsr->u32Reserved = 0;
2404 rc = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &pMsr->u64Value);
2405 AssertRC(rc);
2406 pMsr++; idxMsr++;
2407 }
2408
2409 pVCpu->hm.s.vmx.cGuestMsrs = idxMsr;
2410
2411 rc = VMXWriteVmcs(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, idxMsr);
2412 AssertRC(rc);
2413
2414 rc = VMXWriteVmcs(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, idxMsr);
2415 AssertRC(rc);
2416#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
2417
2418 /* Done with the major changes */
2419 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_ALL_GUEST;
2420
2421 /* Minimal guest state update (ESP, EIP, EFLAGS mostly) */
2422 VMXR0LoadMinimalGuestState(pVM, pVCpu, pCtx);
2423
2424 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
2425 return rc;
2426}
2427
2428
2429/**
2430 * Syncs back the guest state from VMCS.
2431 *
2432 * @returns VBox status code.
2433 * @param pVM Pointer to the VM.
2434 * @param pVCpu Pointer to the VMCPU.
2435 * @param pCtx Pointer to the guest CPU context.
2436 */
2437DECLINLINE(int) VMXR0SaveGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2438{
2439 RTGCUINTREG val, valShadow;
2440 RTGCUINTPTR uInterruptState;
2441 int rc;
2442
2443 /* First sync back EIP, ESP, and EFLAGS. */
2444 rc = VMXReadCachedVmcs(VMX_VMCS_GUEST_RIP, &val);
2445 AssertRC(rc);
2446 pCtx->rip = val;
2447 rc = VMXReadCachedVmcs(VMX_VMCS_GUEST_RSP, &val);
2448 AssertRC(rc);
2449 pCtx->rsp = val;
2450 rc = VMXReadCachedVmcs(VMX_VMCS_GUEST_RFLAGS, &val);
2451 AssertRC(rc);
2452 pCtx->eflags.u32 = val;
2453
2454 /* Take care of instruction fusing (sti, mov ss) */
2455 rc |= VMXReadCachedVmcs(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &val);
2456 uInterruptState = val;
2457 if (uInterruptState != 0)
2458 {
2459 Assert(uInterruptState <= 2); /* only sti & mov ss */
2460 Log(("uInterruptState %x eip=%RGv\n", (uint32_t)uInterruptState, pCtx->rip));
2461 EMSetInhibitInterruptsPC(pVCpu, pCtx->rip);
2462 }
2463 else
2464 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2465
2466 /* Control registers. */
2467 VMXReadCachedVmcs(VMX_VMCS_CTRL_CR0_READ_SHADOW, &valShadow);
2468 VMXReadCachedVmcs(VMX_VMCS_GUEST_CR0, &val);
2469 val = (valShadow & pVCpu->hm.s.vmx.u32CR0Mask) | (val & ~pVCpu->hm.s.vmx.u32CR0Mask);
2470 CPUMSetGuestCR0(pVCpu, val);
2471
2472 VMXReadCachedVmcs(VMX_VMCS_CTRL_CR4_READ_SHADOW, &valShadow);
2473 VMXReadCachedVmcs(VMX_VMCS_GUEST_CR4, &val);
2474 val = (valShadow & pVCpu->hm.s.vmx.u32CR4Mask) | (val & ~pVCpu->hm.s.vmx.u32CR4Mask);
2475 CPUMSetGuestCR4(pVCpu, val);
2476
2477 /*
2478 * No reason to sync back the CRx registers. They can't be changed by the guest unless in
2479 * the nested paging case where CR3 & CR4 can be changed by the guest.
2480 */
2481 if ( pVM->hm.s.fNestedPaging
2482 && CPUMIsGuestInPagedProtectedModeEx(pCtx)) /** @todo check if we will always catch mode switches and such... */
2483 {
2484 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
2485
2486 /* Can be updated behind our back in the nested paging case. */
2487 CPUMSetGuestCR2(pVCpu, pCache->cr2);
2488
2489 VMXReadCachedVmcs(VMX_VMCS_GUEST_CR3, &val);
2490
2491 if (val != pCtx->cr3)
2492 {
2493 CPUMSetGuestCR3(pVCpu, val);
2494 PGMUpdateCR3(pVCpu, val);
2495 }
2496 rc = hmR0VmxSavePaePdpes(pVCpu, pCtx);
2497 AssertRCReturn(rc, rc);
2498 }
2499
2500 /* Sync back DR7. */
2501 VMXReadCachedVmcs(VMX_VMCS_GUEST_DR7, &val);
2502 pCtx->dr[7] = val;
2503
2504 /* Guest CPU context: ES, CS, SS, DS, FS, GS. */
2505 VMX_READ_SELREG(ES, es);
2506 VMX_READ_SELREG(SS, ss);
2507 VMX_READ_SELREG(CS, cs);
2508 VMX_READ_SELREG(DS, ds);
2509 VMX_READ_SELREG(FS, fs);
2510 VMX_READ_SELREG(GS, gs);
2511
2512 /* System MSRs */
2513 VMXReadCachedVmcs(VMX_VMCS32_GUEST_SYSENTER_CS, &val);
2514 pCtx->SysEnter.cs = val;
2515 VMXReadCachedVmcs(VMX_VMCS_GUEST_SYSENTER_EIP, &val);
2516 pCtx->SysEnter.eip = val;
2517 VMXReadCachedVmcs(VMX_VMCS_GUEST_SYSENTER_ESP, &val);
2518 pCtx->SysEnter.esp = val;
2519
2520 /* Misc. registers; must sync everything otherwise we can get out of sync when jumping to ring 3. */
2521 VMX_READ_SELREG(LDTR, ldtr);
2522
2523 VMXReadCachedVmcs(VMX_VMCS32_GUEST_GDTR_LIMIT, &val);
2524 pCtx->gdtr.cbGdt = val;
2525 VMXReadCachedVmcs(VMX_VMCS_GUEST_GDTR_BASE, &val);
2526 pCtx->gdtr.pGdt = val;
2527
2528 VMXReadCachedVmcs(VMX_VMCS32_GUEST_IDTR_LIMIT, &val);
2529 pCtx->idtr.cbIdt = val;
2530 VMXReadCachedVmcs(VMX_VMCS_GUEST_IDTR_BASE, &val);
2531 pCtx->idtr.pIdt = val;
2532
2533 /* Real mode emulation using v86 mode. */
2534 if ( CPUMIsGuestInRealModeEx(pCtx)
2535 && pVM->hm.s.vmx.pRealModeTSS)
2536 {
2537 /* Hide our emulation flags */
2538 pCtx->eflags.Bits.u1VM = 0;
2539
2540 /* Restore original IOPL setting as we always use 0. */
2541 pCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.eflags.Bits.u2IOPL;
2542
2543 /* Force a TR resync every time in case we switch modes. */
2544 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_TR;
2545 }
2546 else
2547 {
2548 /* In real mode we have a fake TSS, so only sync it back when it's supposed to be valid. */
2549 VMX_READ_SELREG(TR, tr);
2550 }
2551
2552#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
2553 /*
2554 * Save the possibly changed MSRs that we automatically restore and save during a world switch.
2555 */
2556 for (unsigned i = 0; i < pVCpu->hm.s.vmx.cGuestMsrs; i++)
2557 {
2558 PVMXMSR pMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvGuestMsr;
2559 pMsr += i;
2560
2561 switch (pMsr->u32IndexMSR)
2562 {
2563 case MSR_K8_LSTAR:
2564 pCtx->msrLSTAR = pMsr->u64Value;
2565 break;
2566 case MSR_K6_STAR:
2567 pCtx->msrSTAR = pMsr->u64Value;
2568 break;
2569 case MSR_K8_SF_MASK:
2570 pCtx->msrSFMASK = pMsr->u64Value;
2571 break;
2572#if 0
2573 /* The KERNEL_GS_BASE MSR does not work reliably with auto load/store. See @bugref{6208} */
2574 case MSR_K8_KERNEL_GS_BASE:
2575 pCtx->msrKERNELGSBASE = pMsr->u64Value;
2576 break;
2577#endif
2578 case MSR_K8_TSC_AUX:
2579 CPUMSetGuestMsr(pVCpu, MSR_K8_TSC_AUX, pMsr->u64Value);
2580 break;
2581
2582 case MSR_K6_EFER:
2583 /* EFER can't be changed without causing a VM-exit. */
2584 /* Assert(pCtx->msrEFER == pMsr->u64Value); */
2585 break;
2586
2587 default:
2588 AssertFailed();
2589 return VERR_HM_UNEXPECTED_LD_ST_MSR;
2590 }
2591 }
2592#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
2593 return VINF_SUCCESS;
2594}
2595
2596
2597/**
2598 * Dummy placeholder for TLB flush handling before VM-entry. Used in the case
2599 * where neither EPT nor VPID is supported by the CPU.
2600 *
2601 * @param pVM Pointer to the VM.
2602 * @param pVCpu Pointer to the VMCPU.
2603 */
2604static DECLCALLBACK(void) hmR0VmxSetupTLBDummy(PVM pVM, PVMCPU pVCpu)
2605{
2606 NOREF(pVM);
2607 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
2608 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2609 pVCpu->hm.s.TlbShootdown.cPages = 0;
2610}
2611
2612
2613/**
2614 * Setup the tagged TLB for EPT+VPID.
2615 *
2616 * @param pVM Pointer to the VM.
2617 * @param pVCpu Pointer to the VMCPU.
2618 */
2619static DECLCALLBACK(void) hmR0VmxSetupTLBBoth(PVM pVM, PVMCPU pVCpu)
2620{
2621 PHMGLOBLCPUINFO pCpu;
2622
2623 Assert(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid);
2624
2625 pCpu = HMR0GetCurrentCpu();
2626
2627 /*
2628 * Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last
2629 * This can happen both for start & resume due to long jumps back to ring-3.
2630 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
2631 * or the host Cpu is online after a suspend/resume, so we cannot reuse the current ASID anymore.
2632 */
2633 bool fNewAsid = false;
2634 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2635 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2636 {
2637 pVCpu->hm.s.fForceTLBFlush = true;
2638 fNewAsid = true;
2639 }
2640
2641 /*
2642 * Check for explicit TLB shootdowns.
2643 */
2644 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2645 pVCpu->hm.s.fForceTLBFlush = true;
2646
2647 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2648
2649 if (pVCpu->hm.s.fForceTLBFlush)
2650 {
2651 if (fNewAsid)
2652 {
2653 ++pCpu->uCurrentAsid;
2654 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2655 {
2656 pCpu->uCurrentAsid = 1; /* start at 1; host uses 0 */
2657 pCpu->cTlbFlushes++;
2658 pCpu->fFlushAsidBeforeUse = true;
2659 }
2660
2661 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2662 if (pCpu->fFlushAsidBeforeUse)
2663 hmR0VmxFlushVPID(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
2664 }
2665 else
2666 {
2667 if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2668 hmR0VmxFlushVPID(pVM, pVCpu, VMX_FLUSH_VPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2669 else
2670 hmR0VmxFlushEPT(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2671 }
2672
2673 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2674 pVCpu->hm.s.fForceTLBFlush = false;
2675 }
2676 else
2677 {
2678 AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
2679 ("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
2680 pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
2681 pCpu->uCurrentAsid, pCpu->cTlbFlushes));
2682
2683 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
2684 * not be executed. See hmQueueInvlPage() where it is commented
2685 * out. Support individual entry flushing someday. */
2686 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2687 {
2688 STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
2689
2690 /*
2691 * Flush individual guest entries using VPID from the TLB or as little as possible with EPT
2692 * as supported by the CPU.
2693 */
2694 if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2695 {
2696 for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
2697 hmR0VmxFlushVPID(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
2698 }
2699 else
2700 hmR0VmxFlushEPT(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2701 }
2702 else
2703 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
2704 }
2705
2706 pVCpu->hm.s.TlbShootdown.cPages = 0;
2707 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2708
2709 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2710 ("Flush count mismatch for cpu %d (%x vs %x)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2711 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2712 ("cpu%d uCurrentAsid = %x\n", pCpu->idCpu, pCpu->uCurrentAsid));
2713 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2714 ("cpu%d VM uCurrentAsid = %x\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2715
2716 /* Update VMCS with the VPID. */
2717 int rc = VMXWriteVmcs(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
2718 AssertRC(rc);
2719}
2720
2721
2722/**
2723 * Setup the tagged TLB for EPT only.
2724 *
2725 * @returns VBox status code.
2726 * @param pVM Pointer to the VM.
2727 * @param pVCpu Pointer to the VMCPU.
2728 */
2729static DECLCALLBACK(void) hmR0VmxSetupTLBEPT(PVM pVM, PVMCPU pVCpu)
2730{
2731 PHMGLOBLCPUINFO pCpu;
2732
2733 Assert(pVM->hm.s.fNestedPaging);
2734 Assert(!pVM->hm.s.vmx.fVpid);
2735
2736 pCpu = HMR0GetCurrentCpu();
2737
2738 /*
2739 * Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last
2740 * This can happen both for start & resume due to long jumps back to ring-3.
2741 * A change in the TLB flush count implies the host Cpu is online after a suspend/resume.
2742 */
2743 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2744 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2745 {
2746 pVCpu->hm.s.fForceTLBFlush = true;
2747 }
2748
2749 /*
2750 * Check for explicit TLB shootdown flushes.
2751 */
2752 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2753 pVCpu->hm.s.fForceTLBFlush = true;
2754
2755 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2756 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2757
2758 if (pVCpu->hm.s.fForceTLBFlush)
2759 hmR0VmxFlushEPT(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2760 else
2761 {
2762 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
2763 * not be executed. See hmQueueInvlPage() where it is commented
2764 * out. Support individual entry flushing someday. */
2765 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2766 {
2767 /*
2768 * We cannot flush individual entries without VPID support. Flush using EPT.
2769 */
2770 STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
2771 hmR0VmxFlushEPT(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2772 }
2773 }
2774 pVCpu->hm.s.TlbShootdown.cPages = 0;
2775 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2776
2777#ifdef VBOX_WITH_STATISTICS
2778 /** @todo r=ramshankar: this is not accurate anymore with the VPID+EPT
2779 * handling. Should be fixed later. */
2780 if (pVCpu->hm.s.fForceTLBFlush)
2781 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2782 else
2783 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
2784#endif
2785}
2786
2787
2788/**
2789 * Setup the tagged TLB for VPID.
2790 *
2791 * @returns VBox status code.
2792 * @param pVM Pointer to the VM.
2793 * @param pVCpu Pointer to the VMCPU.
2794 */
2795static DECLCALLBACK(void) hmR0VmxSetupTLBVPID(PVM pVM, PVMCPU pVCpu)
2796{
2797 PHMGLOBLCPUINFO pCpu;
2798
2799 Assert(pVM->hm.s.vmx.fVpid);
2800 Assert(!pVM->hm.s.fNestedPaging);
2801
2802 pCpu = HMR0GetCurrentCpu();
2803
2804 /*
2805 * Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last
2806 * This can happen both for start & resume due to long jumps back to ring-3.
2807 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
2808 * or the host Cpu is online after a suspend/resume, so we cannot reuse the current ASID anymore.
2809 */
2810 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2811 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2812 {
2813 /* Force a TLB flush on VM entry. */
2814 pVCpu->hm.s.fForceTLBFlush = true;
2815 }
2816
2817 /*
2818 * Check for explicit TLB shootdown flushes.
2819 */
2820 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2821 pVCpu->hm.s.fForceTLBFlush = true;
2822
2823 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2824
2825 if (pVCpu->hm.s.fForceTLBFlush)
2826 {
2827 ++pCpu->uCurrentAsid;
2828 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2829 {
2830 pCpu->uCurrentAsid = 1; /* start at 1; host uses 0 */
2831 pCpu->cTlbFlushes++;
2832 pCpu->fFlushAsidBeforeUse = true;
2833 }
2834
2835 pVCpu->hm.s.fForceTLBFlush = false;
2836 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2837 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2838 if (pCpu->fFlushAsidBeforeUse)
2839 hmR0VmxFlushVPID(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
2840 }
2841 else
2842 {
2843 AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
2844 ("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
2845 pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
2846 pCpu->uCurrentAsid, pCpu->cTlbFlushes));
2847
2848 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
2849 * not be executed. See hmQueueInvlPage() where it is commented
2850 * out. Support individual entry flushing someday. */
2851 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2852 {
2853 /*
2854 * Flush individual guest entries using VPID from the TLB or as little as possible with EPT
2855 * as supported by the CPU.
2856 */
2857 if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2858 {
2859 for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
2860 hmR0VmxFlushVPID(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
2861 }
2862 else
2863 hmR0VmxFlushVPID(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
2864 }
2865 }
2866 pVCpu->hm.s.TlbShootdown.cPages = 0;
2867 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2868
2869 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2870 ("Flush count mismatch for cpu %d (%x vs %x)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2871 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2872 ("cpu%d uCurrentAsid = %x\n", pCpu->idCpu, pCpu->uCurrentAsid));
2873 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2874 ("cpu%d VM uCurrentAsid = %x\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2875
2876 int rc = VMXWriteVmcs(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
2877 AssertRC(rc);
2878
2879# ifdef VBOX_WITH_STATISTICS
2880 /** @todo r=ramshankar: this is not accurate anymore with EPT+VPID handling.
2881 * Should be fixed later. */
2882 if (pVCpu->hm.s.fForceTLBFlush)
2883 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2884 else
2885 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
2886# endif
2887}
2888
2889
2890/**
2891 * Runs guest code in a VT-x VM.
2892 *
2893 * @returns VBox status code.
2894 * @param pVM Pointer to the VM.
2895 * @param pVCpu Pointer to the VMCPU.
2896 * @param pCtx Pointer to the guest CPU context.
2897 */
2898VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2899{
2900 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
2901 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
2902 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
2903
2904 VBOXSTRICTRC rc = VINF_SUCCESS;
2905 int rc2;
2906 RTGCUINTREG val;
2907 RTGCUINTREG exitReason = (RTGCUINTREG)VMX_EXIT_INVALID;
2908 RTGCUINTREG instrError, cbInstr;
2909 RTGCUINTPTR exitQualification = 0;
2910 RTGCUINTPTR intInfo = 0; /* shut up buggy gcc 4 */
2911 RTGCUINTPTR errCode, instrInfo;
2912 bool fSetupTPRCaching = false;
2913 bool fNeedTscSetup = true;
2914 uint64_t u64OldLSTAR = 0;
2915 uint8_t u8LastTPR = 0;
2916 RTCCUINTREG uOldEFlags = ~(RTCCUINTREG)0;
2917 unsigned cResume = 0;
2918#ifdef VBOX_STRICT
2919 RTCPUID idCpuCheck;
2920 bool fWasInLongMode = false;
2921#endif
2922#ifdef VBOX_HIGH_RES_TIMERS_HACK_IN_RING0
2923 uint64_t u64LastTime = RTTimeMilliTS();
2924#endif
2925
2926 Assert(!(pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2927 || (pVCpu->hm.s.vmx.pbVirtApic && pVM->hm.s.vmx.pbApicAccess));
2928
2929 /*
2930 * Check if we need to use TPR shadowing.
2931 */
2932 if ( CPUMIsGuestInLongModeEx(pCtx)
2933 || ( (( pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2934 || pVM->hm.s.fTRPPatchingAllowed)
2935 && pVM->hm.s.fHasIoApic)
2936 )
2937 {
2938 fSetupTPRCaching = true;
2939 }
2940
2941 Log2(("\n"));
2942
2943 /* This is not ideal, but if we don't clear the event injection in the VMCS right here,
2944 * we may end up injecting some stale event into a VM, including injecting an event that
2945 * originated before a VM reset *after* the VM has been reset. See @bugref{6220}.
2946 */
2947 VMXWriteVmcs(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
2948
2949#ifdef VBOX_STRICT
2950 {
2951 RTCCUINTREG val2;
2952
2953 rc2 = VMXReadVmcs(VMX_VMCS32_CTRL_PIN_EXEC, &val2);
2954 AssertRC(rc2);
2955 Log2(("VMX_VMCS_CTRL_PIN_EXEC = %08x\n", val2));
2956
2957 /* allowed zero */
2958 if ((val2 & pVM->hm.s.vmx.msr.vmx_pin_ctls.n.disallowed0) != pVM->hm.s.vmx.msr.vmx_pin_ctls.n.disallowed0)
2959 Log(("Invalid VMX_VMCS_CTRL_PIN_EXEC: zero\n"));
2960
2961 /* allowed one */
2962 if ((val2 & ~pVM->hm.s.vmx.msr.vmx_pin_ctls.n.allowed1) != 0)
2963 Log(("Invalid VMX_VMCS_CTRL_PIN_EXEC: one\n"));
2964
2965 rc2 = VMXReadVmcs(VMX_VMCS32_CTRL_PROC_EXEC, &val2);
2966 AssertRC(rc2);
2967 Log2(("VMX_VMCS_CTRL_PROC_EXEC = %08x\n", val2));
2968
2969 /*
2970 * Must be set according to the MSR, but can be cleared if nested paging is used.
2971 */
2972 if (pVM->hm.s.fNestedPaging)
2973 {
2974 val2 |= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
2975 | VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2976 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2977 }
2978
2979 /* allowed zero */
2980 if ((val2 & pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0) != pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0)
2981 Log(("Invalid VMX_VMCS_CTRL_PROC_EXEC: zero\n"));
2982
2983 /* allowed one */
2984 if ((val2 & ~pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1) != 0)
2985 Log(("Invalid VMX_VMCS_CTRL_PROC_EXEC: one\n"));
2986
2987 rc2 = VMXReadVmcs(VMX_VMCS32_CTRL_ENTRY, &val2);
2988 AssertRC(rc2);
2989 Log2(("VMX_VMCS_CTRL_ENTRY = %08x\n", val2));
2990
2991 /* allowed zero */
2992 if ((val2 & pVM->hm.s.vmx.msr.vmx_entry.n.disallowed0) != pVM->hm.s.vmx.msr.vmx_entry.n.disallowed0)
2993 Log(("Invalid VMX_VMCS_CTRL_ENTRY: zero\n"));
2994
2995 /* allowed one */
2996 if ((val2 & ~pVM->hm.s.vmx.msr.vmx_entry.n.allowed1) != 0)
2997 Log(("Invalid VMX_VMCS_CTRL_ENTRY: one\n"));
2998
2999 rc2 = VMXReadVmcs(VMX_VMCS32_CTRL_EXIT, &val2);
3000 AssertRC(rc2);
3001 Log2(("VMX_VMCS_CTRL_EXIT = %08x\n", val2));
3002
3003 /* allowed zero */
3004 if ((val2 & pVM->hm.s.vmx.msr.vmx_exit.n.disallowed0) != pVM->hm.s.vmx.msr.vmx_exit.n.disallowed0)
3005 Log(("Invalid VMX_VMCS_CTRL_EXIT: zero\n"));
3006
3007 /* allowed one */
3008 if ((val2 & ~pVM->hm.s.vmx.msr.vmx_exit.n.allowed1) != 0)
3009 Log(("Invalid VMX_VMCS_CTRL_EXIT: one\n"));
3010 }
3011 fWasInLongMode = CPUMIsGuestInLongModeEx(pCtx);
3012#endif /* VBOX_STRICT */
3013
3014#ifdef VBOX_WITH_CRASHDUMP_MAGIC
3015 pVCpu->hm.s.vmx.VMCSCache.u64TimeEntry = RTTimeNanoTS();
3016#endif
3017
3018 /*
3019 * We can jump to this point to resume execution after determining that a VM-exit is innocent.
3020 */
3021ResumeExecution:
3022 if (!STAM_REL_PROFILE_ADV_IS_RUNNING(&pVCpu->hm.s.StatEntry))
3023 STAM_REL_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit2, &pVCpu->hm.s.StatEntry, x);
3024 AssertMsg(pVCpu->hm.s.idEnteredCpu == RTMpCpuId(),
3025 ("Expected %d, I'm %d; cResume=%d exitReason=%RGv exitQualification=%RGv\n",
3026 (int)pVCpu->hm.s.idEnteredCpu, (int)RTMpCpuId(), cResume, exitReason, exitQualification));
3027 Assert(!HMR0SuspendPending());
3028 /* Not allowed to switch modes without reloading the host state (32->64 switcher)!! */
3029 Assert(fWasInLongMode == CPUMIsGuestInLongModeEx(pCtx));
3030
3031 /*
3032 * Safety precaution; looping for too long here can have a very bad effect on the host.
3033 */
3034 if (RT_UNLIKELY(++cResume > pVM->hm.s.cMaxResumeLoops))
3035 {
3036 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
3037 rc = VINF_EM_RAW_INTERRUPT;
3038 goto end;
3039 }
3040
3041 /*
3042 * Check for IRQ inhibition due to instruction fusing (sti, mov ss).
3043 */
3044 if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3045 {
3046 Log(("VM_FF_INHIBIT_INTERRUPTS at %RGv successor %RGv\n", (RTGCPTR)pCtx->rip, EMGetInhibitInterruptsPC(pVCpu)));
3047 if (pCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
3048 {
3049 /*
3050 * Note: we intentionally don't clear VM_FF_INHIBIT_INTERRUPTS here.
3051 * Before we are able to execute this instruction in raw mode (iret to guest code) an external interrupt might
3052 * force a world switch again. Possibly allowing a guest interrupt to be dispatched in the process. This could
3053 * break the guest. Sounds very unlikely, but such timing sensitive problems are not as rare as you might think.
3054 */
3055 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3056 /* Irq inhibition is no longer active; clear the corresponding VMX state. */
3057 rc2 = VMXWriteVmcs(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, 0);
3058 AssertRC(rc2);
3059 }
3060 }
3061 else
3062 {
3063 /* Irq inhibition is no longer active; clear the corresponding VMX state. */
3064 rc2 = VMXWriteVmcs(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, 0);
3065 AssertRC(rc2);
3066 }
3067
3068#ifdef VBOX_HIGH_RES_TIMERS_HACK_IN_RING0
3069 if (RT_UNLIKELY((cResume & 0xf) == 0))
3070 {
3071 uint64_t u64CurTime = RTTimeMilliTS();
3072
3073 if (RT_UNLIKELY(u64CurTime > u64LastTime))
3074 {
3075 u64LastTime = u64CurTime;
3076 TMTimerPollVoid(pVM, pVCpu);
3077 }
3078 }
3079#endif
3080
3081 /*
3082 * Check for pending actions that force us to go back to ring-3.
3083 */
3084 if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK | VM_FF_REQUEST | VM_FF_PGM_POOL_FLUSH_PENDING | VM_FF_PDM_DMA)
3085 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK | VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL | VMCPU_FF_REQUEST))
3086 {
3087 /* Check if a sync operation is pending. */
3088 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
3089 {
3090 rc = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4, VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
3091 if (rc != VINF_SUCCESS)
3092 {
3093 AssertRC(VBOXSTRICTRC_VAL(rc));
3094 Log(("Pending pool sync is forcing us back to ring 3; rc=%d\n", VBOXSTRICTRC_VAL(rc)));
3095 goto end;
3096 }
3097 }
3098
3099#ifdef DEBUG
3100 /* Intercept X86_XCPT_DB if stepping is enabled */
3101 if (!DBGFIsStepping(pVCpu))
3102#endif
3103 {
3104 if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
3105 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
3106 {
3107 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
3108 rc = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
3109 goto end;
3110 }
3111 }
3112
3113 /* Pending request packets might contain actions that need immediate attention, such as pending hardware interrupts. */
3114 if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
3115 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
3116 {
3117 rc = VINF_EM_PENDING_REQUEST;
3118 goto end;
3119 }
3120
3121 /* Check if a pgm pool flush is in progress. */
3122 if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
3123 {
3124 rc = VINF_PGM_POOL_FLUSH_PENDING;
3125 goto end;
3126 }
3127
3128 /* Check if DMA work is pending (2nd+ run). */
3129 if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA) && cResume > 1)
3130 {
3131 rc = VINF_EM_RAW_TO_R3;
3132 goto end;
3133 }
3134 }
3135
3136#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
3137 /*
3138 * Exit to ring-3 preemption/work is pending.
3139 *
3140 * Interrupts are disabled before the call to make sure we don't miss any interrupt
3141 * that would flag preemption (IPI, timer tick, ++). (Would've been nice to do this
3142 * further down, but hmR0VmxCheckPendingInterrupt makes that impossible.)
3143 *
3144 * Note! Interrupts must be disabled done *before* we check for TLB flushes; TLB
3145 * shootdowns rely on this.
3146 */
3147 uOldEFlags = ASMIntDisableFlags();
3148 if (RTThreadPreemptIsPending(NIL_RTTHREAD))
3149 {
3150 STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
3151 rc = VINF_EM_RAW_INTERRUPT;
3152 goto end;
3153 }
3154 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
3155#endif
3156
3157 /*
3158 * When external interrupts are pending, we should exit the VM when IF is set.
3159 * Note: *After* VM_FF_INHIBIT_INTERRUPTS check!
3160 */
3161 rc = hmR0VmxCheckPendingInterrupt(pVM, pVCpu, pCtx);
3162 if (RT_FAILURE(rc))
3163 goto end;
3164
3165 /** @todo check timers?? */
3166
3167 /*
3168 * TPR caching using CR8 is only available in 64-bit mode.
3169 * Note: The 32-bit exception for AMD (X86_CPUID_AMD_FEATURE_ECX_CR8L), but this appears missing in Intel CPUs.
3170 * Note: We can't do this in LoadGuestState() as PDMApicGetTPR can jump back to ring-3 (lock)!! (no longer true) .
3171 */
3172 /** @todo query and update the TPR only when it could have been changed (mmio
3173 * access & wrsmr (x2apic) */
3174 if (fSetupTPRCaching)
3175 {
3176 /* TPR caching in CR8 */
3177 bool fPending;
3178
3179 rc2 = PDMApicGetTPR(pVCpu, &u8LastTPR, &fPending, NULL /* pu8PendingIrq */);
3180 AssertRC(rc2);
3181 /* The TPR can be found at offset 0x80 in the APIC mmio page. */
3182 pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8LastTPR;
3183
3184 /*
3185 * Two options here:
3186 * - external interrupt pending, but masked by the TPR value.
3187 * -> a CR8 update that lower the current TPR value should cause an exit
3188 * - no pending interrupts
3189 * -> We don't need to be explicitely notified. There are enough world switches for detecting pending interrupts.
3190 */
3191
3192 /* cr8 bits 3-0 correspond to bits 7-4 of the task priority mmio register. */
3193 rc = VMXWriteVmcs(VMX_VMCS32_CTRL_TPR_THRESHOLD, (fPending) ? (u8LastTPR >> 4) : 0);
3194 AssertRC(VBOXSTRICTRC_VAL(rc));
3195
3196 if (pVM->hm.s.fTPRPatchingActive)
3197 {
3198 Assert(!CPUMIsGuestInLongModeEx(pCtx));
3199 /* Our patch code uses LSTAR for TPR caching. */
3200 pCtx->msrLSTAR = u8LastTPR;
3201
3202 /** @todo r=ramshankar: we should check for MSR-bitmap support here. */
3203 if (fPending)
3204 {
3205 /* A TPR change could activate a pending interrupt, so catch lstar writes. */
3206 hmR0VmxSetMSRPermission(pVCpu, MSR_K8_LSTAR, true, false);
3207 }
3208 else
3209 {
3210 /*
3211 * No interrupts are pending, so we don't need to be explicitely notified.
3212 * There are enough world switches for detecting pending interrupts.
3213 */
3214 hmR0VmxSetMSRPermission(pVCpu, MSR_K8_LSTAR, true, true);
3215 }
3216 }
3217 }
3218
3219#ifdef LOG_ENABLED
3220 if ( pVM->hm.s.fNestedPaging
3221 || pVM->hm.s.vmx.fVpid)
3222 {
3223 PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
3224 if (pVCpu->hm.s.idLastCpu != pCpu->idCpu)
3225 {
3226 LogFlow(("Force TLB flush due to rescheduling to a different cpu (%d vs %d)\n", pVCpu->hm.s.idLastCpu,
3227 pCpu->idCpu));
3228 }
3229 else if (pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
3230 {
3231 LogFlow(("Force TLB flush due to changed TLB flush count (%x vs %x)\n", pVCpu->hm.s.cTlbFlushes,
3232 pCpu->cTlbFlushes));
3233 }
3234 else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH))
3235 LogFlow(("Manual TLB flush\n"));
3236 }
3237#endif
3238#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3239 PGMRZDynMapFlushAutoSet(pVCpu);
3240#endif
3241
3242 /*
3243 * NOTE: DO NOT DO ANYTHING AFTER THIS POINT THAT MIGHT JUMP BACK TO RING-3!
3244 * (until the actual world switch)
3245 */
3246#ifdef VBOX_STRICT
3247 idCpuCheck = RTMpCpuId();
3248#endif
3249#ifdef LOG_ENABLED
3250 VMMR0LogFlushDisable(pVCpu);
3251#endif
3252
3253 /*
3254 * Save the host state first.
3255 */
3256 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT)
3257 {
3258 rc = VMXR0SaveHostState(pVM, pVCpu);
3259 if (RT_UNLIKELY(rc != VINF_SUCCESS))
3260 {
3261 VMMR0LogFlushEnable(pVCpu);
3262 goto end;
3263 }
3264 }
3265
3266 /*
3267 * Load the guest state.
3268 */
3269 if (!pVCpu->hm.s.fContextUseFlags)
3270 {
3271 VMXR0LoadMinimalGuestState(pVM, pVCpu, pCtx);
3272 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
3273 if (fNeedTscSetup)
3274 {
3275 VMXR0SetupTscOffsetAndPreemption(pVM, pVCpu);
3276 fNeedTscSetup = false;
3277 }
3278 }
3279 else
3280 {
3281 rc = VMXR0LoadGuestState(pVM, pVCpu, pCtx);
3282 if (RT_UNLIKELY(rc != VINF_SUCCESS))
3283 {
3284 VMMR0LogFlushEnable(pVCpu);
3285 goto end;
3286 }
3287 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
3288 VMXR0SetupTscOffsetAndPreemption(pVM, pVCpu);
3289 }
3290
3291#ifndef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
3292 /*
3293 * Disable interrupts to make sure a poke will interrupt execution.
3294 * This must be done *before* we check for TLB flushes; TLB shootdowns rely on this.
3295 */
3296 uOldEFlags = ASMIntDisableFlags();
3297 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
3298#endif
3299
3300 /* Non-register state Guest Context */
3301 /** @todo change me according to cpu state */
3302 rc2 = VMXWriteVmcs(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
3303 AssertRC(rc2);
3304
3305 /* Set TLB flush state as checked until we return from the world switch. */
3306 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true);
3307 /* Deal with tagged TLB setup and invalidation. */
3308 pVM->hm.s.vmx.pfnFlushTaggedTlb(pVM, pVCpu);
3309
3310 /*
3311 * Manual save and restore:
3312 * - General purpose registers except RIP, RSP
3313 *
3314 * Trashed:
3315 * - CR2 (we don't care)
3316 * - LDTR (reset to 0)
3317 * - DRx (presumably not changed at all)
3318 * - DR7 (reset to 0x400)
3319 * - EFLAGS (reset to RT_BIT(1); not relevant)
3320 */
3321
3322 /* All done! Let's start VM execution. */
3323 Assert(idCpuCheck == RTMpCpuId());
3324
3325#ifdef VBOX_WITH_CRASHDUMP_MAGIC
3326 pVCpu->hm.s.vmx.VMCSCache.cResume = cResume;
3327 pVCpu->hm.s.vmx.VMCSCache.u64TimeSwitch = RTTimeNanoTS();
3328#endif
3329
3330 /*
3331 * Save the current TPR value in the LSTAR MSR so our patches can access it.
3332 */
3333 if (pVM->hm.s.fTPRPatchingActive)
3334 {
3335 Assert(pVM->hm.s.fTPRPatchingActive);
3336 u64OldLSTAR = ASMRdMsr(MSR_K8_LSTAR);
3337 ASMWrMsr(MSR_K8_LSTAR, u8LastTPR);
3338 }
3339
3340 TMNotifyStartOfExecution(pVCpu);
3341
3342#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
3343 /*
3344 * Save the current Host TSC_AUX and write the guest TSC_AUX to the host, so that
3345 * RDTSCPs (that don't cause exits) reads the guest MSR. See @bugref{3324}.
3346 */
3347 if ( (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
3348 && !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
3349 {
3350 pVCpu->hm.s.u64HostTscAux = ASMRdMsr(MSR_K8_TSC_AUX);
3351 uint64_t u64HostTscAux = 0;
3352 rc2 = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &u64HostTscAux);
3353 AssertRC(rc2);
3354 ASMWrMsr(MSR_K8_TSC_AUX, u64HostTscAux);
3355 }
3356#endif
3357
3358 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
3359#ifdef VBOX_WITH_KERNEL_USING_XMM
3360 rc = HMR0VMXStartVMWrapXMM(pVCpu->hm.s.fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
3361#else
3362 rc = pVCpu->hm.s.vmx.pfnStartVM(pVCpu->hm.s.fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
3363#endif
3364 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
3365 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false);
3366 ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits);
3367
3368 /* Possibly the last TSC value seen by the guest (too high) (only when we're in TSC offset mode). */
3369 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
3370 {
3371#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
3372 /* Restore host's TSC_AUX. */
3373 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
3374 ASMWrMsr(MSR_K8_TSC_AUX, pVCpu->hm.s.u64HostTscAux);
3375#endif
3376
3377 TMCpuTickSetLastSeen(pVCpu,
3378 ASMReadTSC() + pVCpu->hm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
3379 }
3380
3381 TMNotifyEndOfExecution(pVCpu);
3382 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
3383 Assert(!(ASMGetFlags() & X86_EFL_IF));
3384
3385 /*
3386 * Restore the host LSTAR MSR if the guest could have changed it.
3387 */
3388 if (pVM->hm.s.fTPRPatchingActive)
3389 {
3390 Assert(pVM->hm.s.fTPRPatchingActive);
3391 pVCpu->hm.s.vmx.pbVirtApic[0x80] = pCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
3392 ASMWrMsr(MSR_K8_LSTAR, u64OldLSTAR);
3393 }
3394
3395 ASMSetFlags(uOldEFlags);
3396#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
3397 uOldEFlags = ~(RTCCUINTREG)0;
3398#endif
3399
3400 AssertMsg(!pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries, ("pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries=%d\n",
3401 pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries));
3402
3403 /* In case we execute a goto ResumeExecution later on. */
3404 pVCpu->hm.s.fResumeVM = true;
3405 pVCpu->hm.s.fForceTLBFlush = false;
3406
3407 /*
3408 * !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
3409 * IMPORTANT: WE CAN'T DO ANY LOGGING OR OPERATIONS THAT CAN DO A LONGJMP BACK TO RING 3 *BEFORE* WE'VE SYNCED BACK (MOST OF) THE GUEST STATE
3410 * !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
3411 */
3412
3413 if (RT_UNLIKELY(rc != VINF_SUCCESS))
3414 {
3415 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx);
3416 VMMR0LogFlushEnable(pVCpu);
3417 goto end;
3418 }
3419
3420 /* Success. Query the guest state and figure out what has happened. */
3421
3422 /* Investigate why there was a VM-exit. */
3423 rc2 = VMXReadCachedVmcs(VMX_VMCS32_RO_EXIT_REASON, &exitReason);
3424 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[exitReason & MASK_EXITREASON_STAT]);
3425
3426 exitReason &= 0xffff; /* bit 0-15 contain the exit code. */
3427 rc2 |= VMXReadCachedVmcs(VMX_VMCS32_RO_VM_INSTR_ERROR, &instrError);
3428 rc2 |= VMXReadCachedVmcs(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &cbInstr);
3429 rc2 |= VMXReadCachedVmcs(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &intInfo);
3430 /* might not be valid; depends on VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID. */
3431 rc2 |= VMXReadCachedVmcs(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &errCode);
3432 rc2 |= VMXReadCachedVmcs(VMX_VMCS32_RO_EXIT_INSTR_INFO, &instrInfo);
3433 rc2 |= VMXReadCachedVmcs(VMX_VMCS_RO_EXIT_QUALIFICATION, &exitQualification);
3434 AssertRC(rc2);
3435
3436 /*
3437 * Sync back the guest state.
3438 */
3439 rc2 = VMXR0SaveGuestState(pVM, pVCpu, pCtx);
3440 AssertRC(rc2);
3441
3442 /* Note! NOW IT'S SAFE FOR LOGGING! */
3443 VMMR0LogFlushEnable(pVCpu);
3444 Log2(("Raw exit reason %08x\n", exitReason));
3445#if ARCH_BITS == 64 /* for the time being */
3446 VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, pCtx, exitReason);
3447#endif
3448
3449 /*
3450 * Check if an injected event was interrupted prematurely.
3451 */
3452 rc2 = VMXReadCachedVmcs(VMX_VMCS32_RO_IDT_INFO, &val);
3453 AssertRC(rc2);
3454 pVCpu->hm.s.Event.u64IntrInfo = VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(val);
3455 if ( VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVCpu->hm.s.Event.u64IntrInfo)
3456 /* Ignore 'int xx' as they'll be restarted anyway. */
3457 && VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo) != VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT
3458 /* Ignore software exceptions (such as int3) as they'll reoccur when we restart the instruction anyway. */
3459 && VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo) != VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
3460 {
3461 Assert(!pVCpu->hm.s.Event.fPending);
3462 pVCpu->hm.s.Event.fPending = true;
3463 /* Error code present? */
3464 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntrInfo))
3465 {
3466 rc2 = VMXReadCachedVmcs(VMX_VMCS32_RO_IDT_ERROR_CODE, &val);
3467 AssertRC(rc2);
3468 pVCpu->hm.s.Event.u32ErrCode = val;
3469 Log(("Pending inject %RX64 at %RGv exit=%08x intInfo=%08x exitQualification=%RGv pending error=%RX64\n",
3470 pVCpu->hm.s.Event.u64IntrInfo, (RTGCPTR)pCtx->rip, exitReason, intInfo, exitQualification, val));
3471 }
3472 else
3473 {
3474 Log(("Pending inject %RX64 at %RGv exit=%08x intInfo=%08x exitQualification=%RGv\n", pVCpu->hm.s.Event.u64IntrInfo,
3475 (RTGCPTR)pCtx->rip, exitReason, intInfo, exitQualification));
3476 pVCpu->hm.s.Event.u32ErrCode = 0;
3477 }
3478 }
3479#ifdef VBOX_STRICT
3480 else if ( VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVCpu->hm.s.Event.u64IntrInfo)
3481 /* Ignore software exceptions (such as int3) as they're reoccur when we restart the instruction anyway. */
3482 && VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
3483 {
3484 Log(("Ignore pending inject %RX64 at %RGv exit=%08x intInfo=%08x exitQualification=%RGv\n",
3485 pVCpu->hm.s.Event.u64IntrInfo, (RTGCPTR)pCtx->rip, exitReason, intInfo, exitQualification));
3486 }
3487
3488 if (exitReason == VMX_EXIT_ERR_INVALID_GUEST_STATE)
3489 HMDumpRegs(pVM, pVCpu, pCtx);
3490#endif
3491
3492 Log2(("E%d: New EIP=%x:%RGv\n", (uint32_t)exitReason, pCtx->cs.Sel, (RTGCPTR)pCtx->rip));
3493 Log2(("Exit reason %d, exitQualification %RGv\n", (uint32_t)exitReason, exitQualification));
3494 Log2(("instrInfo=%d instrError=%d instr length=%d\n", (uint32_t)instrInfo, (uint32_t)instrError, (uint32_t)cbInstr));
3495 Log2(("Interruption error code %d\n", (uint32_t)errCode));
3496 Log2(("IntInfo = %08x\n", (uint32_t)intInfo));
3497
3498 /*
3499 * Sync back the TPR if it was changed.
3500 */
3501 if ( fSetupTPRCaching
3502 && u8LastTPR != pVCpu->hm.s.vmx.pbVirtApic[0x80])
3503 {
3504 rc2 = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
3505 AssertRC(rc2);
3506 }
3507
3508#ifdef DBGFTRACE_ENABLED /** @todo DTrace later. */
3509 RTTraceBufAddMsgF(pVM->CTX_SUFF(hTraceBuf), "vmexit %08x %016RX64 at %04:%08RX64 %RX64",
3510 exitReason, (uint64_t)exitQualification, pCtx->cs.Sel, pCtx->rip, (uint64_t)intInfo);
3511#endif
3512 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
3513
3514 /* Some cases don't need a complete resync of the guest CPU state; handle them here. */
3515 Assert(rc == VINF_SUCCESS); /* might consider VERR_IPE_UNINITIALIZED_STATUS here later... */
3516 switch (exitReason)
3517 {
3518 case VMX_EXIT_XCPT_NMI: /* 0 Exception or non-maskable interrupt (NMI). */
3519 case VMX_EXIT_EXT_INT: /* 1 External interrupt. */
3520 {
3521 uint32_t vector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(intInfo);
3522
3523 if (!VMX_EXIT_INTERRUPTION_INFO_IS_VALID(intInfo))
3524 {
3525 Assert(exitReason == VMX_EXIT_EXT_INT);
3526 /* External interrupt; leave to allow it to be dispatched again. */
3527 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
3528 rc = VINF_EM_RAW_INTERRUPT;
3529 break;
3530 }
3531 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
3532 switch (VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo))
3533 {
3534 case VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI: /* Non-maskable interrupt. */
3535 /* External interrupt; leave to allow it to be dispatched again. */
3536 rc = VINF_EM_RAW_INTERRUPT;
3537 break;
3538
3539 case VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT: /* External hardware interrupt. */
3540 AssertFailed(); /* can't come here; fails the first check. */
3541 break;
3542
3543 case VMX_EXIT_INTERRUPTION_INFO_TYPE_DB_XCPT: /* Unknown why we get this type for #DB */
3544 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
3545 Assert(vector == 1 || vector == 3 || vector == 4);
3546 /* no break */
3547 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT: /* Hardware exception. */
3548 Log2(("Hardware/software interrupt %d\n", vector));
3549 switch (vector)
3550 {
3551 case X86_XCPT_NM:
3552 {
3553 Log(("#NM fault at %RGv error code %x\n", (RTGCPTR)pCtx->rip, errCode));
3554
3555 /** @todo don't intercept #NM exceptions anymore when we've activated the guest FPU state. */
3556 /* If we sync the FPU/XMM state on-demand, then we can continue execution as if nothing has happened. */
3557 rc = CPUMR0LoadGuestFPU(pVM, pVCpu, pCtx);
3558 if (rc == VINF_SUCCESS)
3559 {
3560 Assert(CPUMIsGuestFPUStateActive(pVCpu));
3561
3562 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
3563
3564 /* Continue execution. */
3565 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_CR0;
3566
3567 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
3568 goto ResumeExecution;
3569 }
3570
3571 Log(("Forward #NM fault to the guest\n"));
3572 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
3573 rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
3574 cbInstr, 0);
3575 AssertRC(rc2);
3576 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
3577 goto ResumeExecution;
3578 }
3579
3580 case X86_XCPT_PF: /* Page fault */
3581 {
3582#ifdef VBOX_ALWAYS_TRAP_PF
3583 if (pVM->hm.s.fNestedPaging)
3584 {
3585 /*
3586 * A genuine pagefault. Forward the trap to the guest by injecting the exception and resuming execution.
3587 */
3588 Log(("Guest page fault at %RGv cr2=%RGv error code %RGv rsp=%RGv\n", (RTGCPTR)pCtx->rip, exitQualification,
3589 errCode, (RTGCPTR)pCtx->rsp));
3590
3591 Assert(CPUMIsGuestInPagedProtectedModeEx(pCtx));
3592
3593 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
3594
3595 /* Now we must update CR2. */
3596 pCtx->cr2 = exitQualification;
3597 rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
3598 cbInstr, errCode);
3599 AssertRC(rc2);
3600
3601 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
3602 goto ResumeExecution;
3603 }
3604#else
3605 Assert(!pVM->hm.s.fNestedPaging);
3606#endif
3607
3608#ifdef VBOX_HM_WITH_GUEST_PATCHING
3609 /* Shortcut for APIC TPR reads and writes; 32 bits guests only */
3610 if ( pVM->hm.s.fTRPPatchingAllowed
3611 && pVM->hm.s.pGuestPatchMem
3612 && (exitQualification & 0xfff) == 0x080
3613 && !(errCode & X86_TRAP_PF_P) /* not present */
3614 && CPUMGetGuestCPL(pVCpu) == 0
3615 && !CPUMIsGuestInLongModeEx(pCtx)
3616 && pVM->hm.s.cPatches < RT_ELEMENTS(pVM->hm.s.aPatches))
3617 {
3618 RTGCPHYS GCPhysApicBase, GCPhys;
3619 GCPhysApicBase = pCtx->msrApicBase;
3620 GCPhysApicBase &= PAGE_BASE_GC_MASK;
3621
3622 rc = PGMGstGetPage(pVCpu, (RTGCPTR)exitQualification, NULL, &GCPhys);
3623 if ( rc == VINF_SUCCESS
3624 && GCPhys == GCPhysApicBase)
3625 {
3626 /* Only attempt to patch the instruction once. */
3627 PHMTPRPATCH pPatch = (PHMTPRPATCH)RTAvloU32Get(&pVM->hm.s.PatchTree, (AVLOU32KEY)pCtx->eip);
3628 if (!pPatch)
3629 {
3630 rc = VINF_EM_HM_PATCH_TPR_INSTR;
3631 break;
3632 }
3633 }
3634 }
3635#endif
3636
3637 Log2(("Page fault at %RGv error code %x\n", exitQualification, errCode));
3638 /* Exit qualification contains the linear address of the page fault. */
3639 TRPMAssertTrap(pVCpu, X86_XCPT_PF, TRPM_TRAP);
3640 TRPMSetErrorCode(pVCpu, errCode);
3641 TRPMSetFaultAddress(pVCpu, exitQualification);
3642
3643 /* Shortcut for APIC TPR reads and writes. */
3644 if ( (exitQualification & 0xfff) == 0x080
3645 && !(errCode & X86_TRAP_PF_P) /* not present */
3646 && fSetupTPRCaching
3647 && (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
3648 {
3649 RTGCPHYS GCPhysApicBase, GCPhys;
3650 GCPhysApicBase = pCtx->msrApicBase;
3651 GCPhysApicBase &= PAGE_BASE_GC_MASK;
3652
3653 rc = PGMGstGetPage(pVCpu, (RTGCPTR)exitQualification, NULL, &GCPhys);
3654 if ( rc == VINF_SUCCESS
3655 && GCPhys == GCPhysApicBase)
3656 {
3657 Log(("Enable VT-x virtual APIC access filtering\n"));
3658 rc2 = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess,
3659 X86_PTE_RW | X86_PTE_P);
3660 AssertRC(rc2);
3661 }
3662 }
3663
3664 /* Forward it to our trap handler first, in case our shadow pages are out of sync. */
3665 rc = PGMTrap0eHandler(pVCpu, errCode, CPUMCTX2CORE(pCtx), (RTGCPTR)exitQualification);
3666 Log2(("PGMTrap0eHandler %RGv returned %Rrc\n", (RTGCPTR)pCtx->rip, VBOXSTRICTRC_VAL(rc)));
3667
3668 if (rc == VINF_SUCCESS)
3669 { /* We've successfully synced our shadow pages, so let's just continue execution. */
3670 Log2(("Shadow page fault at %RGv cr2=%RGv error code %x\n", (RTGCPTR)pCtx->rip, exitQualification ,errCode));
3671 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
3672
3673 TRPMResetTrap(pVCpu);
3674 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
3675 goto ResumeExecution;
3676 }
3677 else if (rc == VINF_EM_RAW_GUEST_TRAP)
3678 {
3679 /*
3680 * A genuine pagefault. Forward the trap to the guest by injecting the exception and resuming execution.
3681 */
3682 Log2(("Forward page fault to the guest\n"));
3683
3684 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
3685 /* The error code might have been changed. */
3686 errCode = TRPMGetErrorCode(pVCpu);
3687
3688 TRPMResetTrap(pVCpu);
3689
3690 /* Now we must update CR2. */
3691 pCtx->cr2 = exitQualification;
3692 rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
3693 cbInstr, errCode);
3694 AssertRC(rc2);
3695
3696 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
3697 goto ResumeExecution;
3698 }
3699#ifdef VBOX_STRICT
3700 if (rc != VINF_EM_RAW_EMULATE_INSTR && rc != VINF_EM_RAW_EMULATE_IO_BLOCK)
3701 Log2(("PGMTrap0eHandler failed with %d\n", VBOXSTRICTRC_VAL(rc)));
3702#endif
3703 /* Need to go back to the recompiler to emulate the instruction. */
3704 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
3705 TRPMResetTrap(pVCpu);
3706
3707 /* If event delivery caused the #PF (shadow or not), tell TRPM. */
3708 hmR0VmxCheckPendingEvent(pVCpu);
3709 break;
3710 }
3711
3712 case X86_XCPT_MF: /* Floating point exception. */
3713 {
3714 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
3715 if (!(pCtx->cr0 & X86_CR0_NE))
3716 {
3717 /* old style FPU error reporting needs some extra work. */
3718 /** @todo don't fall back to the recompiler, but do it manually. */
3719 rc = VINF_EM_RAW_EMULATE_INSTR;
3720 break;
3721 }
3722 Log(("Trap %x at %04X:%RGv\n", vector, pCtx->cs.Sel, (RTGCPTR)pCtx->rip));
3723 rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
3724 cbInstr, errCode);
3725 AssertRC(rc2);
3726
3727 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
3728 goto ResumeExecution;
3729 }
3730
3731 case X86_XCPT_DB: /* Debug exception. */
3732 {
3733 uint64_t uDR6;
3734
3735 /*
3736 * DR6, DR7.GD and IA32_DEBUGCTL.LBR are not updated yet.
3737 *
3738 * Exit qualification bits:
3739 * 3:0 B0-B3 which breakpoint condition was met
3740 * 12:4 Reserved (0)
3741 * 13 BD - debug register access detected
3742 * 14 BS - single step execution or branch taken
3743 * 63:15 Reserved (0)
3744 */
3745 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
3746
3747 /* Note that we don't support guest and host-initiated debugging at the same time. */
3748
3749 uDR6 = X86_DR6_INIT_VAL;
3750 uDR6 |= (exitQualification & (X86_DR6_B0|X86_DR6_B1|X86_DR6_B2|X86_DR6_B3|X86_DR6_BD|X86_DR6_BS));
3751 rc = DBGFRZTrap01Handler(pVM, pVCpu, CPUMCTX2CORE(pCtx), uDR6);
3752 if (rc == VINF_EM_RAW_GUEST_TRAP)
3753 {
3754 /* Update DR6 here. */
3755 pCtx->dr[6] = uDR6;
3756
3757 /* Resync DR6 if the debug state is active. */
3758 if (CPUMIsGuestDebugStateActive(pVCpu))
3759 ASMSetDR6(pCtx->dr[6]);
3760
3761 /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
3762 pCtx->dr[7] &= ~X86_DR7_GD;
3763
3764 /* Paranoia. */
3765 pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
3766 pCtx->dr[7] &= ~(RT_BIT(11) | RT_BIT(12) | RT_BIT(14) | RT_BIT(15)); /* must be zero */
3767 pCtx->dr[7] |= 0x400; /* must be one */
3768
3769 /* Resync DR7 */
3770 rc2 = VMXWriteVmcs64(VMX_VMCS_GUEST_DR7, pCtx->dr[7]);
3771 AssertRC(rc2);
3772
3773 Log(("Trap %x (debug) at %RGv exit qualification %RX64 dr6=%x dr7=%x\n", vector, (RTGCPTR)pCtx->rip,
3774 exitQualification, (uint32_t)pCtx->dr[6], (uint32_t)pCtx->dr[7]));
3775 rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
3776 cbInstr, errCode);
3777 AssertRC(rc2);
3778
3779 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
3780 goto ResumeExecution;
3781 }
3782 /* Return to ring 3 to deal with the debug exit code. */
3783 Log(("Debugger hardware BP at %04x:%RGv (rc=%Rrc)\n", pCtx->cs.Sel, pCtx->rip, VBOXSTRICTRC_VAL(rc)));
3784 break;
3785 }
3786
3787 case X86_XCPT_BP: /* Breakpoint. */
3788 {
3789 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
3790 rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pCtx));
3791 if (rc == VINF_EM_RAW_GUEST_TRAP)
3792 {
3793 Log(("Guest #BP at %04x:%RGv\n", pCtx->cs.Sel, pCtx->rip));
3794 rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
3795 cbInstr, errCode);
3796 AssertRC(rc2);
3797 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
3798 goto ResumeExecution;
3799 }
3800 if (rc == VINF_SUCCESS)
3801 {
3802 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
3803 goto ResumeExecution;
3804 }
3805 Log(("Debugger BP at %04x:%RGv (rc=%Rrc)\n", pCtx->cs.Sel, pCtx->rip, VBOXSTRICTRC_VAL(rc)));
3806 break;
3807 }
3808
3809 case X86_XCPT_GP: /* General protection failure exception. */
3810 {
3811 uint32_t cbOp;
3812 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
3813
3814 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
3815#ifdef VBOX_STRICT
3816 if ( !CPUMIsGuestInRealModeEx(pCtx)
3817 || !pVM->hm.s.vmx.pRealModeTSS)
3818 {
3819 Log(("Trap %x at %04X:%RGv errorCode=%RGv\n", vector, pCtx->cs.Sel, (RTGCPTR)pCtx->rip, errCode));
3820 rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
3821 cbInstr, errCode);
3822 AssertRC(rc2);
3823 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
3824 goto ResumeExecution;
3825 }
3826#endif
3827 Assert(CPUMIsGuestInRealModeEx(pCtx));
3828
3829 LogFlow(("Real mode X86_XCPT_GP instruction emulation at %x:%RGv\n", pCtx->cs.Sel, (RTGCPTR)pCtx->rip));
3830
3831 rc2 = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
3832 if (RT_SUCCESS(rc2))
3833 {
3834 bool fUpdateRIP = true;
3835
3836 rc = VINF_SUCCESS;
3837 Assert(cbOp == pDis->cbInstr);
3838 switch (pDis->pCurInstr->uOpcode)
3839 {
3840 case OP_CLI:
3841 pCtx->eflags.Bits.u1IF = 0;
3842 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
3843 break;
3844
3845 case OP_STI:
3846 pCtx->eflags.Bits.u1IF = 1;
3847 EMSetInhibitInterruptsPC(pVCpu, pCtx->rip + pDis->cbInstr);
3848 Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
3849 rc2 = VMXWriteVmcs(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE,
3850 VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
3851 AssertRC(rc2);
3852 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
3853 break;
3854
3855 case OP_HLT:
3856 fUpdateRIP = false;
3857 rc = VINF_EM_HALT;
3858 pCtx->rip += pDis->cbInstr;
3859 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
3860 break;
3861
3862 case OP_POPF:
3863 {
3864 RTGCPTR GCPtrStack;
3865 uint32_t cbParm;
3866 uint32_t uMask;
3867 X86EFLAGS eflags;
3868
3869 if (pDis->fPrefix & DISPREFIX_OPSIZE)
3870 {
3871 cbParm = 4;
3872 uMask = 0xffffffff;
3873 }
3874 else
3875 {
3876 cbParm = 2;
3877 uMask = 0xffff;
3878 }
3879
3880 rc2 = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pCtx), pCtx->esp & uMask, 0, &GCPtrStack);
3881 if (RT_FAILURE(rc2))
3882 {
3883 rc = VERR_EM_INTERPRETER;
3884 break;
3885 }
3886 eflags.u = 0;
3887 rc2 = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &eflags.u, cbParm);
3888 if (RT_FAILURE(rc2))
3889 {
3890 rc = VERR_EM_INTERPRETER;
3891 break;
3892 }
3893 LogFlow(("POPF %x -> %RGv mask=%x\n", eflags.u, pCtx->rsp, uMask));
3894 pCtx->eflags.u = (pCtx->eflags.u & ~(X86_EFL_POPF_BITS & uMask))
3895 | (eflags.u & X86_EFL_POPF_BITS & uMask);
3896 /* RF cleared when popped in real mode; see pushf description in AMD manual. */
3897 pCtx->eflags.Bits.u1RF = 0;
3898 pCtx->esp += cbParm;
3899 pCtx->esp &= uMask;
3900
3901 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
3902 break;
3903 }
3904
3905 case OP_PUSHF:
3906 {
3907 RTGCPTR GCPtrStack;
3908 uint32_t cbParm;
3909 uint32_t uMask;
3910 X86EFLAGS eflags;
3911
3912 if (pDis->fPrefix & DISPREFIX_OPSIZE)
3913 {
3914 cbParm = 4;
3915 uMask = 0xffffffff;
3916 }
3917 else
3918 {
3919 cbParm = 2;
3920 uMask = 0xffff;
3921 }
3922
3923 rc2 = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pCtx), (pCtx->esp - cbParm) & uMask, 0,
3924 &GCPtrStack);
3925 if (RT_FAILURE(rc2))
3926 {
3927 rc = VERR_EM_INTERPRETER;
3928 break;
3929 }
3930 eflags = pCtx->eflags;
3931 /* RF & VM cleared when pushed in real mode; see pushf description in AMD manual. */
3932 eflags.Bits.u1RF = 0;
3933 eflags.Bits.u1VM = 0;
3934
3935 rc2 = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &eflags.u, cbParm);
3936 if (RT_FAILURE(rc2))
3937 {
3938 rc = VERR_EM_INTERPRETER;
3939 break;
3940 }
3941 LogFlow(("PUSHF %x -> %RGv\n", eflags.u, GCPtrStack));
3942 pCtx->esp -= cbParm;
3943 pCtx->esp &= uMask;
3944 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
3945 break;
3946 }
3947
3948 case OP_IRET:
3949 {
3950 RTGCPTR GCPtrStack;
3951 uint32_t uMask = 0xffff;
3952 uint16_t aIretFrame[3];
3953
3954 if (pDis->fPrefix & (DISPREFIX_OPSIZE | DISPREFIX_ADDRSIZE))
3955 {
3956 rc = VERR_EM_INTERPRETER;
3957 break;
3958 }
3959
3960 rc2 = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pCtx), pCtx->esp & uMask, 0, &GCPtrStack);
3961 if (RT_FAILURE(rc2))
3962 {
3963 rc = VERR_EM_INTERPRETER;
3964 break;
3965 }
3966 rc2 = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
3967 if (RT_FAILURE(rc2))
3968 {
3969 rc = VERR_EM_INTERPRETER;
3970 break;
3971 }
3972 pCtx->ip = aIretFrame[0];
3973 pCtx->cs.Sel = aIretFrame[1];
3974 pCtx->cs.ValidSel = aIretFrame[1];
3975 pCtx->cs.u64Base = (uint32_t)pCtx->cs.Sel << 4;
3976 pCtx->eflags.u = (pCtx->eflags.u & ~(X86_EFL_POPF_BITS & uMask))
3977 | (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
3978 pCtx->sp += sizeof(aIretFrame);
3979
3980 LogFlow(("iret to %04x:%x\n", pCtx->cs.Sel, pCtx->ip));
3981 fUpdateRIP = false;
3982 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
3983 break;
3984 }
3985
3986 case OP_INT:
3987 {
3988 uint32_t intInfo2;
3989
3990 LogFlow(("Realmode: INT %x\n", pDis->Param1.uValue & 0xff));
3991 intInfo2 = pDis->Param1.uValue & 0xff;
3992 intInfo2 |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
3993 intInfo2 |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
3994
3995 rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo2, cbOp, 0);
3996 AssertRC(VBOXSTRICTRC_VAL(rc));
3997 fUpdateRIP = false;
3998 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
3999 break;
4000 }
4001
4002 case OP_INTO:
4003 {
4004 if (pCtx->eflags.Bits.u1OF)
4005 {
4006 uint32_t intInfo2;
4007
4008 LogFlow(("Realmode: INTO\n"));
4009 intInfo2 = X86_XCPT_OF;
4010 intInfo2 |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
4011 intInfo2 |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
4012
4013 rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo2, cbOp, 0);
4014 AssertRC(VBOXSTRICTRC_VAL(rc));
4015 fUpdateRIP = false;
4016 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
4017 }
4018 break;
4019 }
4020
4021 case OP_INT3:
4022 {
4023 uint32_t intInfo2;
4024
4025 LogFlow(("Realmode: INT 3\n"));
4026 intInfo2 = 3;
4027 intInfo2 |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
4028 intInfo2 |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
4029
4030 rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo2, cbOp, 0);
4031 AssertRC(VBOXSTRICTRC_VAL(rc));
4032 fUpdateRIP = false;
4033 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
4034 break;
4035 }
4036
4037 default:
4038 rc = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pCtx), 0, EMCODETYPE_SUPERVISOR);
4039 fUpdateRIP = false;
4040 break;
4041 }
4042
4043 if (rc == VINF_SUCCESS)
4044 {
4045 if (fUpdateRIP)
4046 pCtx->rip += cbOp; /* Move on to the next instruction. */
4047
4048 /*
4049 * LIDT, LGDT can end up here. In the future CRx changes as well. Just reload the
4050 * whole context to be done with it.
4051 */
4052 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_ALL;
4053
4054 /* Only resume if successful. */
4055 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
4056 goto ResumeExecution;
4057 }
4058 }
4059 else
4060 rc = VERR_EM_INTERPRETER;
4061
4062 AssertMsg(rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_EM_HALT,
4063 ("Unexpected rc=%Rrc\n", VBOXSTRICTRC_VAL(rc)));
4064 break;
4065 }
4066
4067#ifdef VBOX_STRICT
4068 case X86_XCPT_XF: /* SIMD exception. */
4069 case X86_XCPT_DE: /* Divide error. */
4070 case X86_XCPT_UD: /* Unknown opcode exception. */
4071 case X86_XCPT_SS: /* Stack segment exception. */
4072 case X86_XCPT_NP: /* Segment not present exception. */
4073 {
4074 switch (vector)
4075 {
4076 case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE); break;
4077 case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD); break;
4078 case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS); break;
4079 case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP); break;
4080 case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF); break;
4081 }
4082
4083 Log(("Trap %x at %04X:%RGv\n", vector, pCtx->cs.Sel, (RTGCPTR)pCtx->rip));
4084 rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
4085 cbInstr, errCode);
4086 AssertRC(rc2);
4087
4088 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
4089 goto ResumeExecution;
4090 }
4091#endif
4092 default:
4093 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
4094 if ( CPUMIsGuestInRealModeEx(pCtx)
4095 && pVM->hm.s.vmx.pRealModeTSS)
4096 {
4097 Log(("Real Mode Trap %x at %04x:%04X error code %x\n", vector, pCtx->cs.Sel, pCtx->eip, errCode));
4098 rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
4099 cbInstr, errCode);
4100 AssertRC(VBOXSTRICTRC_VAL(rc)); /* Strict RC check below. */
4101
4102 /* Go back to ring-3 in case of a triple fault. */
4103 if ( vector == X86_XCPT_DF
4104 && rc == VINF_EM_RESET)
4105 {
4106 break;
4107 }
4108
4109 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
4110 goto ResumeExecution;
4111 }
4112 AssertMsgFailed(("Unexpected vm-exit caused by exception %x\n", vector));
4113 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
4114 break;
4115 } /* switch (vector) */
4116
4117 break;
4118
4119 default:
4120 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_CODE;
4121 AssertMsgFailed(("Unexpected interruption code %x\n", intInfo));
4122 break;
4123 }
4124
4125 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
4126 break;
4127 }
4128
4129 /*
4130 * 48 EPT violation. An attempt to access memory with a guest-physical address was disallowed
4131 * by the configuration of the EPT paging structures.
4132 */
4133 case VMX_EXIT_EPT_VIOLATION:
4134 {
4135 RTGCPHYS GCPhys;
4136
4137 Assert(pVM->hm.s.fNestedPaging);
4138
4139 rc2 = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
4140 AssertRC(rc2);
4141 Assert(((exitQualification >> 7) & 3) != 2);
4142
4143 /* Determine the kind of violation. */
4144 errCode = 0;
4145 if (exitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
4146 errCode |= X86_TRAP_PF_ID;
4147
4148 if (exitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
4149 errCode |= X86_TRAP_PF_RW;
4150
4151 /* If the page is present, then it's a page level protection fault. */
4152 if (exitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
4153 errCode |= X86_TRAP_PF_P;
4154 else
4155 {
4156 /* Shortcut for APIC TPR reads and writes. */
4157 if ( (GCPhys & 0xfff) == 0x080
4158 && GCPhys > 0x1000000 /* to skip VGA frame buffer accesses */
4159 && fSetupTPRCaching
4160 && (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
4161 {
4162 RTGCPHYS GCPhysApicBase;
4163 GCPhysApicBase = pCtx->msrApicBase;
4164 GCPhysApicBase &= PAGE_BASE_GC_MASK;
4165 if (GCPhys == GCPhysApicBase + 0x80)
4166 {
4167 Log(("Enable VT-x virtual APIC access filtering\n"));
4168 rc2 = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess,
4169 X86_PTE_RW | X86_PTE_P);
4170 AssertRC(rc2);
4171 }
4172 }
4173 }
4174 Log(("EPT Page fault %x at %RGp error code %x\n", (uint32_t)exitQualification, GCPhys, errCode));
4175
4176 /* GCPhys contains the guest physical address of the page fault. */
4177 TRPMAssertTrap(pVCpu, X86_XCPT_PF, TRPM_TRAP);
4178 TRPMSetErrorCode(pVCpu, errCode);
4179 TRPMSetFaultAddress(pVCpu, GCPhys);
4180
4181 /* Handle the pagefault trap for the nested shadow table. */
4182 rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, errCode, CPUMCTX2CORE(pCtx), GCPhys);
4183
4184 /*
4185 * Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment below, @bugref{6043}.
4186 */
4187 if ( rc == VINF_SUCCESS
4188 || rc == VERR_PAGE_TABLE_NOT_PRESENT
4189 || rc == VERR_PAGE_NOT_PRESENT)
4190 {
4191 /* We've successfully synced our shadow pages, so let's just continue execution. */
4192 Log2(("Shadow page fault at %RGv cr2=%RGp error code %x\n", (RTGCPTR)pCtx->rip, exitQualification , errCode));
4193 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
4194
4195 TRPMResetTrap(pVCpu);
4196 goto ResumeExecution;
4197 }
4198
4199#ifdef VBOX_STRICT
4200 if (rc != VINF_EM_RAW_EMULATE_INSTR)
4201 LogFlow(("PGMTrap0eHandlerNestedPaging at %RGv failed with %Rrc\n", (RTGCPTR)pCtx->rip, VBOXSTRICTRC_VAL(rc)));
4202#endif
4203 /* Need to go back to the recompiler to emulate the instruction. */
4204 TRPMResetTrap(pVCpu);
4205 break;
4206 }
4207
4208 case VMX_EXIT_EPT_MISCONFIG:
4209 {
4210 RTGCPHYS GCPhys;
4211
4212 Assert(pVM->hm.s.fNestedPaging);
4213
4214 rc2 = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
4215 AssertRC(rc2);
4216 Log(("VMX_EXIT_EPT_MISCONFIG for %RGp\n", GCPhys));
4217
4218 /* Shortcut for APIC TPR reads and writes. */
4219 if ( (GCPhys & 0xfff) == 0x080
4220 && GCPhys > 0x1000000 /* to skip VGA frame buffer accesses */
4221 && fSetupTPRCaching
4222 && (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
4223 {
4224 RTGCPHYS GCPhysApicBase = pCtx->msrApicBase;
4225 GCPhysApicBase &= PAGE_BASE_GC_MASK;
4226 if (GCPhys == GCPhysApicBase + 0x80)
4227 {
4228 Log(("Enable VT-x virtual APIC access filtering\n"));
4229 rc2 = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess,
4230 X86_PTE_RW | X86_PTE_P);
4231 AssertRC(rc2);
4232 }
4233 }
4234
4235 rc = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pCtx), GCPhys, UINT32_MAX);
4236
4237 /*
4238 * If we succeed, resume execution.
4239 * Or, if fail in interpreting the instruction because we couldn't get the guest physical address
4240 * of the page containing the instruction via the guest's page tables (we would invalidate the guest page
4241 * in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
4242 * weird case. See @bugref{6043}.
4243 */
4244 if ( rc == VINF_SUCCESS
4245 || rc == VERR_PAGE_TABLE_NOT_PRESENT
4246 || rc == VERR_PAGE_NOT_PRESENT)
4247 {
4248 Log2(("PGMR0Trap0eHandlerNPMisconfig(,,,%RGp) at %RGv -> resume\n", GCPhys, (RTGCPTR)pCtx->rip));
4249 goto ResumeExecution;
4250 }
4251
4252 Log2(("PGMR0Trap0eHandlerNPMisconfig(,,,%RGp) at %RGv -> %Rrc\n", GCPhys, (RTGCPTR)pCtx->rip, VBOXSTRICTRC_VAL(rc)));
4253 break;
4254 }
4255
4256 case VMX_EXIT_INT_WINDOW: /* 7 Interrupt window exiting. */
4257 /* Clear VM-exit on IF=1 change. */
4258 LogFlow(("VMX_EXIT_INT_WINDOW %RGv pending=%d IF=%d\n", (RTGCPTR)pCtx->rip,
4259 VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC|VMCPU_FF_INTERRUPT_PIC)), pCtx->eflags.Bits.u1IF));
4260 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
4261 rc2 = VMXWriteVmcs(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4262 AssertRC(rc2);
4263 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
4264 goto ResumeExecution; /* we check for pending guest interrupts there */
4265
4266 case VMX_EXIT_WBINVD: /* 54 Guest software attempted to execute WBINVD. (conditional) */
4267 case VMX_EXIT_INVD: /* 13 Guest software attempted to execute INVD. (unconditional) */
4268 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
4269 /* Skip instruction and continue directly. */
4270 pCtx->rip += cbInstr;
4271 /* Continue execution.*/
4272 goto ResumeExecution;
4273
4274 case VMX_EXIT_CPUID: /* 10 Guest software attempted to execute CPUID. */
4275 {
4276 Log2(("VMX: Cpuid %x\n", pCtx->eax));
4277 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
4278 rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pCtx));
4279 if (rc == VINF_SUCCESS)
4280 {
4281 /* Update EIP and continue execution. */
4282 Assert(cbInstr == 2);
4283 pCtx->rip += cbInstr;
4284 goto ResumeExecution;
4285 }
4286 AssertMsgFailed(("EMU: cpuid failed with %Rrc\n", VBOXSTRICTRC_VAL(rc)));
4287 rc = VINF_EM_RAW_EMULATE_INSTR;
4288 break;
4289 }
4290
4291 case VMX_EXIT_RDPMC: /* 15 Guest software attempted to execute RDPMC. */
4292 {
4293 Log2(("VMX: Rdpmc %x\n", pCtx->ecx));
4294 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
4295 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pCtx));
4296 if (rc == VINF_SUCCESS)
4297 {
4298 /* Update EIP and continue execution. */
4299 Assert(cbInstr == 2);
4300 pCtx->rip += cbInstr;
4301 goto ResumeExecution;
4302 }
4303 rc = VINF_EM_RAW_EMULATE_INSTR;
4304 break;
4305 }
4306
4307 case VMX_EXIT_RDTSC: /* 16 Guest software attempted to execute RDTSC. */
4308 {
4309 Log2(("VMX: Rdtsc\n"));
4310 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
4311 rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pCtx));
4312 if (rc == VINF_SUCCESS)
4313 {
4314 /* Update EIP and continue execution. */
4315 Assert(cbInstr == 2);
4316 pCtx->rip += cbInstr;
4317 fNeedTscSetup = true; /* See @bugref{6634}. */
4318 goto ResumeExecution;
4319 }
4320 rc = VINF_EM_RAW_EMULATE_INSTR;
4321 break;
4322 }
4323
4324 case VMX_EXIT_RDTSCP: /* 51 Guest software attempted to execute RDTSCP. */
4325 {
4326 Log2(("VMX: Rdtscp\n"));
4327 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtscp);
4328 rc = EMInterpretRdtscp(pVM, pVCpu, pCtx);
4329 if (rc == VINF_SUCCESS)
4330 {
4331 /* Update EIP and continue execution. */
4332 Assert(cbInstr == 3);
4333 pCtx->rip += cbInstr;
4334 fNeedTscSetup = true; /* See @bugref{6634}. */
4335 goto ResumeExecution;
4336 }
4337 rc = VINF_EM_RAW_EMULATE_INSTR;
4338 break;
4339 }
4340
4341 case VMX_EXIT_INVLPG: /* 14 Guest software attempted to execute INVLPG. */
4342 {
4343 Log2(("VMX: invlpg\n"));
4344 Assert(!pVM->hm.s.fNestedPaging);
4345
4346 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
4347 rc = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pCtx), exitQualification);
4348 if (rc == VINF_SUCCESS)
4349 {
4350 /* Update EIP and continue execution. */
4351 pCtx->rip += cbInstr;
4352 goto ResumeExecution;
4353 }
4354 AssertMsg(rc == VERR_EM_INTERPRETER, ("EMU: invlpg %RGv failed with %Rrc\n", exitQualification, VBOXSTRICTRC_VAL(rc)));
4355 break;
4356 }
4357
4358 case VMX_EXIT_MONITOR: /* 39 Guest software attempted to execute MONITOR. */
4359 {
4360 Log2(("VMX: monitor\n"));
4361
4362 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
4363 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pCtx));
4364 if (rc == VINF_SUCCESS)
4365 {
4366 /* Update EIP and continue execution. */
4367 pCtx->rip += cbInstr;
4368 goto ResumeExecution;
4369 }
4370 AssertMsg(rc == VERR_EM_INTERPRETER, ("EMU: monitor failed with %Rrc\n", VBOXSTRICTRC_VAL(rc)));
4371 break;
4372 }
4373
4374 case VMX_EXIT_WRMSR: /* 32 WRMSR. Guest software attempted to execute WRMSR. */
4375 /* When an interrupt is pending, we'll let MSR_K8_LSTAR writes fault in our TPR patch code. */
4376 if ( pVM->hm.s.fTPRPatchingActive
4377 && pCtx->ecx == MSR_K8_LSTAR)
4378 {
4379 Assert(!CPUMIsGuestInLongModeEx(pCtx));
4380 if ((pCtx->eax & 0xff) != u8LastTPR)
4381 {
4382 Log(("VMX: Faulting MSR_K8_LSTAR write with new TPR value %x\n", pCtx->eax & 0xff));
4383
4384 /* Our patch code uses LSTAR for TPR caching. */
4385 rc2 = PDMApicSetTPR(pVCpu, pCtx->eax & 0xff);
4386 AssertRC(rc2);
4387 }
4388
4389 /* Skip the instruction and continue. */
4390 pCtx->rip += cbInstr; /* wrmsr = [0F 30] */
4391
4392 /* Only resume if successful. */
4393 goto ResumeExecution;
4394 }
4395 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_MSR;
4396 /* no break */
4397 case VMX_EXIT_RDMSR: /* 31 RDMSR. Guest software attempted to execute RDMSR. */
4398 {
4399 STAM_COUNTER_INC((exitReason == VMX_EXIT_RDMSR) ? &pVCpu->hm.s.StatExitRdmsr : &pVCpu->hm.s.StatExitWrmsr);
4400
4401 Log2(("VMX: %s\n", (exitReason == VMX_EXIT_RDMSR) ? "rdmsr" : "wrmsr"));
4402 rc = EMInterpretInstruction(pVCpu, CPUMCTX2CORE(pCtx), 0);
4403 if (rc == VINF_SUCCESS)
4404 {
4405 /* EIP has been updated already. */
4406 /* Only resume if successful. */
4407 goto ResumeExecution;
4408 }
4409 AssertMsg(rc == VERR_EM_INTERPRETER, ("EMU: %s failed with %Rrc\n",
4410 (exitReason == VMX_EXIT_RDMSR) ? "rdmsr" : "wrmsr", VBOXSTRICTRC_VAL(rc)));
4411 break;
4412 }
4413
4414 case VMX_EXIT_MOV_CRX: /* 28 Control-register accesses. */
4415 {
4416 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
4417
4418 switch (VMX_EXIT_QUALIFICATION_CRX_ACCESS(exitQualification))
4419 {
4420 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE:
4421 {
4422 Log2(("VMX: %RGv mov cr%d, x\n", (RTGCPTR)pCtx->rip, VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification)));
4423 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification)]);
4424 rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
4425 VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification),
4426 VMX_EXIT_QUALIFICATION_CRX_GENREG(exitQualification));
4427 switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification))
4428 {
4429 case 0:
4430 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_CR0 | HM_CHANGED_GUEST_CR3;
4431 break;
4432 case 2:
4433 break;
4434 case 3:
4435 Assert(!pVM->hm.s.fNestedPaging || !CPUMIsGuestInPagedProtectedModeEx(pCtx));
4436 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_CR3;
4437 break;
4438 case 4:
4439 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_CR4;
4440 break;
4441 case 8:
4442 /* CR8 contains the APIC TPR */
4443 Assert(!(pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1
4444 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
4445 break;
4446
4447 default:
4448 AssertFailed();
4449 break;
4450 }
4451 break;
4452 }
4453
4454 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ:
4455 {
4456 Log2(("VMX: mov x, crx\n"));
4457 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification)]);
4458
4459 Assert( !pVM->hm.s.fNestedPaging
4460 || !CPUMIsGuestInPagedProtectedModeEx(pCtx)
4461 || VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification) != DISCREG_CR3);
4462
4463 /* CR8 reads only cause an exit when the TPR shadow feature isn't present. */
4464 Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification) != 8
4465 || !(pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
4466
4467 rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
4468 VMX_EXIT_QUALIFICATION_CRX_GENREG(exitQualification),
4469 VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification));
4470 break;
4471 }
4472
4473 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS:
4474 {
4475 Log2(("VMX: clts\n"));
4476 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
4477 rc = EMInterpretCLTS(pVM, pVCpu);
4478 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_CR0;
4479 break;
4480 }
4481
4482 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW:
4483 {
4484 Log2(("VMX: lmsw %x\n", VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(exitQualification)));
4485 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
4486 rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(exitQualification));
4487 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_CR0;
4488 break;
4489 }
4490 }
4491
4492 /* Update EIP if no error occurred. */
4493 if (RT_SUCCESS(rc))
4494 pCtx->rip += cbInstr;
4495
4496 if (rc == VINF_SUCCESS)
4497 {
4498 /* Only resume if successful. */
4499 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
4500 goto ResumeExecution;
4501 }
4502 Assert(rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_PGM_SYNC_CR3);
4503 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
4504 break;
4505 }
4506
4507 case VMX_EXIT_MOV_DRX: /* 29 Debug-register accesses. */
4508 {
4509 if ( !DBGFIsStepping(pVCpu)
4510 && !CPUMIsHyperDebugStateActive(pVCpu))
4511 {
4512 /* Disable DRx move intercepts. */
4513 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4514 rc2 = VMXWriteVmcs(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4515 AssertRC(rc2);
4516
4517 /* Save the host and load the guest debug state. */
4518 rc2 = CPUMR0LoadGuestDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
4519 AssertRC(rc2);
4520
4521#ifdef LOG_ENABLED
4522 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
4523 {
4524 Log(("VMX_EXIT_MOV_DRX: write DR%d genreg %d\n", VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification),
4525 VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification)));
4526 }
4527 else
4528 Log(("VMX_EXIT_MOV_DRX: read DR%d\n", VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification)));
4529#endif
4530
4531#ifdef VBOX_WITH_STATISTICS
4532 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
4533 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
4534 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
4535 else
4536 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
4537#endif
4538
4539 goto ResumeExecution;
4540 }
4541
4542 /** @todo clear VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT after the first
4543 * time and restore DRx registers afterwards */
4544 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
4545 {
4546 Log2(("VMX: mov DRx%d, genreg%d\n", VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification),
4547 VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification)));
4548 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
4549 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
4550 VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification),
4551 VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification));
4552 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_DEBUG;
4553 Log2(("DR7=%08x\n", pCtx->dr[7]));
4554 }
4555 else
4556 {
4557 Log2(("VMX: mov x, DRx\n"));
4558 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
4559 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
4560 VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification),
4561 VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification));
4562 }
4563 /* Update EIP if no error occurred. */
4564 if (RT_SUCCESS(rc))
4565 pCtx->rip += cbInstr;
4566
4567 if (rc == VINF_SUCCESS)
4568 {
4569 /* Only resume if successful. */
4570 goto ResumeExecution;
4571 }
4572 Assert(rc == VERR_EM_INTERPRETER);
4573 break;
4574 }
4575
4576 /* Note: We'll get a #GP if the IO instruction isn't allowed (IOPL or TSS bitmap); no need to double check. */
4577 case VMX_EXIT_IO_INSTR: /* 30 I/O instruction. */
4578 {
4579 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
4580 uint32_t uPort;
4581 uint32_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(exitQualification);
4582 bool fIOWrite = (VMX_EXIT_QUALIFICATION_IO_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
4583
4584 /** @todo necessary to make the distinction? */
4585 if (VMX_EXIT_QUALIFICATION_IO_ENCODING(exitQualification) == VMX_EXIT_QUALIFICATION_IO_ENCODING_DX)
4586 uPort = pCtx->edx & 0xffff;
4587 else
4588 uPort = VMX_EXIT_QUALIFICATION_IO_PORT(exitQualification); /* Immediate encoding. */
4589
4590 if (RT_UNLIKELY(uIOWidth == 2 || uIOWidth >= 4)) /* paranoia */
4591 {
4592 rc = fIOWrite ? VINF_IOM_R3_IOPORT_WRITE : VINF_IOM_R3_IOPORT_READ;
4593 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
4594 break;
4595 }
4596
4597 uint32_t cbSize = g_aIOSize[uIOWidth];
4598 if (VMX_EXIT_QUALIFICATION_IO_STRING(exitQualification))
4599 {
4600 /* ins/outs */
4601 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
4602
4603 /* Disassemble manually to deal with segment prefixes. */
4604 /** @todo VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR contains the flat pointer
4605 * operand of the instruction. */
4606 /** @todo VMX_VMCS32_RO_EXIT_INSTR_INFO also contains segment prefix info. */
4607 rc2 = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL);
4608 if (RT_SUCCESS(rc))
4609 {
4610 if (fIOWrite)
4611 {
4612 Log2(("IOMInterpretOUTSEx %RGv %x size=%d\n", (RTGCPTR)pCtx->rip, uPort, cbSize));
4613 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
4614 rc = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pCtx), uPort, pDis->fPrefix,
4615 (DISCPUMODE)pDis->uAddrMode, cbSize);
4616 }
4617 else
4618 {
4619 Log2(("IOMInterpretINSEx %RGv %x size=%d\n", (RTGCPTR)pCtx->rip, uPort, cbSize));
4620 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
4621 rc = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pCtx), uPort, pDis->fPrefix,
4622 (DISCPUMODE)pDis->uAddrMode, cbSize);
4623 }
4624 }
4625 else
4626 rc = VINF_EM_RAW_EMULATE_INSTR;
4627 }
4628 else
4629 {
4630 /* Normal in/out */
4631 uint32_t uAndVal = g_aIOOpAnd[uIOWidth];
4632
4633 Assert(!VMX_EXIT_QUALIFICATION_IO_REP(exitQualification));
4634
4635 if (fIOWrite)
4636 {
4637 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
4638 rc = IOMIOPortWrite(pVM, pVCpu, uPort, pCtx->eax & uAndVal, cbSize);
4639 if (rc == VINF_IOM_R3_IOPORT_WRITE)
4640 HMR0SavePendingIOPortWrite(pVCpu, pCtx->rip, pCtx->rip + cbInstr, uPort, uAndVal, cbSize);
4641 }
4642 else
4643 {
4644 uint32_t u32Val = 0;
4645
4646 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
4647 rc = IOMIOPortRead(pVM, pVCpu, uPort, &u32Val, cbSize);
4648 if (IOM_SUCCESS(rc))
4649 {
4650 /* Write back to the EAX register. */
4651 pCtx->eax = (pCtx->eax & ~uAndVal) | (u32Val & uAndVal);
4652 }
4653 else
4654 if (rc == VINF_IOM_R3_IOPORT_READ)
4655 HMR0SavePendingIOPortRead(pVCpu, pCtx->rip, pCtx->rip + cbInstr, uPort, uAndVal, cbSize);
4656 }
4657 }
4658
4659 /*
4660 * Handled the I/O return codes.
4661 * (The unhandled cases end up with rc == VINF_EM_RAW_EMULATE_INSTR.)
4662 */
4663 if (IOM_SUCCESS(rc))
4664 {
4665 /* Update EIP and continue execution. */
4666 pCtx->rip += cbInstr;
4667 if (RT_LIKELY(rc == VINF_SUCCESS))
4668 {
4669 /* If any IO breakpoints are armed, then we should check if a debug trap needs to be generated. */
4670 if (pCtx->dr[7] & X86_DR7_ENABLED_MASK)
4671 {
4672 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
4673 for (unsigned i = 0; i < 4; i++)
4674 {
4675 unsigned uBPLen = g_aIOSize[X86_DR7_GET_LEN(pCtx->dr[7], i)];
4676
4677 if ( (uPort >= pCtx->dr[i] && uPort < pCtx->dr[i] + uBPLen)
4678 && (pCtx->dr[7] & (X86_DR7_L(i) | X86_DR7_G(i)))
4679 && (pCtx->dr[7] & X86_DR7_RW(i, X86_DR7_RW_IO)) == X86_DR7_RW(i, X86_DR7_RW_IO))
4680 {
4681 uint64_t uDR6;
4682
4683 Assert(CPUMIsGuestDebugStateActive(pVCpu));
4684
4685 uDR6 = ASMGetDR6();
4686
4687 /* Clear all breakpoint status flags and set the one we just hit. */
4688 uDR6 &= ~(X86_DR6_B0|X86_DR6_B1|X86_DR6_B2|X86_DR6_B3);
4689 uDR6 |= (uint64_t)RT_BIT(i);
4690
4691 /*
4692 * Note: AMD64 Architecture Programmer's Manual 13.1:
4693 * Bits 15:13 of the DR6 register is never cleared by the processor and must
4694 * be cleared by software after the contents have been read.
4695 */
4696 ASMSetDR6(uDR6);
4697
4698 /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
4699 pCtx->dr[7] &= ~X86_DR7_GD;
4700
4701 /* Paranoia. */
4702 pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
4703 pCtx->dr[7] &= ~(RT_BIT(11) | RT_BIT(12) | RT_BIT(14) | RT_BIT(15)); /* must be zero */
4704 pCtx->dr[7] |= 0x400; /* must be one */
4705
4706 /* Resync DR7 */
4707 rc2 = VMXWriteVmcs64(VMX_VMCS_GUEST_DR7, pCtx->dr[7]);
4708 AssertRC(rc2);
4709
4710 /* Construct inject info. */
4711 intInfo = X86_XCPT_DB;
4712 intInfo |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
4713 intInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
4714
4715 Log(("Inject IO debug trap at %RGv\n", (RTGCPTR)pCtx->rip));
4716 rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
4717 0 /* cbInstr */, 0 /* errCode */);
4718 AssertRC(rc2);
4719
4720 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
4721 goto ResumeExecution;
4722 }
4723 }
4724 }
4725 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
4726 goto ResumeExecution;
4727 }
4728 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
4729 break;
4730 }
4731
4732#ifdef VBOX_STRICT
4733 if (rc == VINF_IOM_R3_IOPORT_READ)
4734 Assert(!fIOWrite);
4735 else if (rc == VINF_IOM_R3_IOPORT_WRITE)
4736 Assert(fIOWrite);
4737 else
4738 {
4739 AssertMsg( RT_FAILURE(rc)
4740 || rc == VINF_EM_RAW_EMULATE_INSTR
4741 || rc == VINF_EM_RAW_GUEST_TRAP
4742 || rc == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rc)));
4743 }
4744#endif
4745 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
4746 break;
4747 }
4748
4749 case VMX_EXIT_TPR_BELOW_THRESHOLD: /* 43 TPR below threshold. Guest software executed MOV to CR8. */
4750 LogFlow(("VMX_EXIT_TPR_BELOW_THRESHOLD\n"));
4751 /* RIP is already set to the next instruction and the TPR has been synced back. Just resume. */
4752 goto ResumeExecution;
4753
4754 case VMX_EXIT_APIC_ACCESS: /* 44 APIC access. Guest software attempted to access memory at a physical address
4755 on the APIC-access page. */
4756 {
4757 LogFlow(("VMX_EXIT_APIC_ACCESS\n"));
4758 unsigned uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(exitQualification);
4759
4760 switch (uAccessType)
4761 {
4762 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
4763 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
4764 {
4765 RTGCPHYS GCPhys = pCtx->msrApicBase;
4766 GCPhys &= PAGE_BASE_GC_MASK;
4767 GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(exitQualification);
4768
4769 LogFlow(("Apic access at %RGp\n", GCPhys));
4770 rc = IOMMMIOPhysHandler(pVM, pVCpu, (uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW,
4771 CPUMCTX2CORE(pCtx), GCPhys);
4772 if (rc == VINF_SUCCESS)
4773 goto ResumeExecution; /* rip already updated */
4774 break;
4775 }
4776
4777 default:
4778 rc = VINF_EM_RAW_EMULATE_INSTR;
4779 break;
4780 }
4781 break;
4782 }
4783
4784 case VMX_EXIT_PREEMPT_TIMER: /* 52 VMX-preemption timer expired. The preemption timer counted down to zero. */
4785 if (!TMTimerPollBool(pVM, pVCpu))
4786 goto ResumeExecution;
4787 rc = VINF_EM_RAW_TIMER_PENDING;
4788 break;
4789
4790 default:
4791 /* The rest is handled after syncing the entire CPU state. */
4792 break;
4793 }
4794
4795
4796 /*
4797 * Note: The guest state is not entirely synced back at this stage!
4798 */
4799
4800 /* Investigate why there was a VM-exit. (part 2) */
4801 switch (exitReason)
4802 {
4803 case VMX_EXIT_XCPT_NMI: /* 0 Exception or non-maskable interrupt (NMI). */
4804 case VMX_EXIT_EXT_INT: /* 1 External interrupt. */
4805 case VMX_EXIT_EPT_VIOLATION:
4806 case VMX_EXIT_EPT_MISCONFIG: /* 49 EPT misconfig is used by the PGM/MMIO optimizations. */
4807 case VMX_EXIT_PREEMPT_TIMER: /* 52 VMX-preemption timer expired. The preemption timer counted down to zero. */
4808 /* Already handled above. */
4809 break;
4810
4811 case VMX_EXIT_TRIPLE_FAULT: /* 2 Triple fault. */
4812 rc = VINF_EM_RESET; /* Triple fault equals a reset. */
4813 break;
4814
4815 case VMX_EXIT_INIT_SIGNAL: /* 3 INIT signal. */
4816 case VMX_EXIT_SIPI: /* 4 Start-up IPI (SIPI). */
4817 rc = VINF_EM_RAW_INTERRUPT;
4818 AssertFailed(); /* Can't happen. Yet. */
4819 break;
4820
4821 case VMX_EXIT_IO_SMI: /* 5 I/O system-management interrupt (SMI). */
4822 case VMX_EXIT_SMI: /* 6 Other SMI. */
4823 rc = VINF_EM_RAW_INTERRUPT;
4824 AssertFailed(); /* Can't happen afaik. */
4825 break;
4826
4827 case VMX_EXIT_TASK_SWITCH: /* 9 Task switch: too complicated to emulate, so fall back to the recompiler */
4828 Log(("VMX_EXIT_TASK_SWITCH: exit=%RX64\n", exitQualification));
4829 if ( (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(exitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
4830 && pVCpu->hm.s.Event.fPending)
4831 {
4832 /* Caused by an injected interrupt. */
4833 pVCpu->hm.s.Event.fPending = false;
4834
4835 Log(("VMX_EXIT_TASK_SWITCH: reassert trap %d\n", VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVCpu->hm.s.Event.u64IntrInfo)));
4836 Assert(!VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntrInfo));
4837 //@todo: Why do we assume this had to be a hardware interrupt? What about software interrupts or exceptions?
4838 rc2 = TRPMAssertTrap(pVCpu, VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVCpu->hm.s.Event.u64IntrInfo), TRPM_HARDWARE_INT);
4839 AssertRC(rc2);
4840 }
4841 /* else Exceptions and software interrupts can just be restarted. */
4842 rc = VERR_EM_INTERPRETER;
4843 break;
4844
4845 case VMX_EXIT_HLT: /* 12 Guest software attempted to execute HLT. */
4846 /* Check if external interrupts are pending; if so, don't switch back. */
4847 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
4848 pCtx->rip++; /* skip hlt */
4849 if (EMShouldContinueAfterHalt(pVCpu, pCtx))
4850 goto ResumeExecution;
4851
4852 rc = VINF_EM_HALT;
4853 break;
4854
4855 case VMX_EXIT_MWAIT: /* 36 Guest software executed MWAIT. */
4856 Log2(("VMX: mwait\n"));
4857 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
4858 rc = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pCtx));
4859 if ( rc == VINF_EM_HALT
4860 || rc == VINF_SUCCESS)
4861 {
4862 /* Update EIP and continue execution. */
4863 pCtx->rip += cbInstr;
4864
4865 /* Check if external interrupts are pending; if so, don't switch back. */
4866 if ( rc == VINF_SUCCESS
4867 || ( rc == VINF_EM_HALT
4868 && EMShouldContinueAfterHalt(pVCpu, pCtx))
4869 )
4870 goto ResumeExecution;
4871 }
4872 AssertMsg(rc == VERR_EM_INTERPRETER || rc == VINF_EM_HALT, ("EMU: mwait failed with %Rrc\n", VBOXSTRICTRC_VAL(rc)));
4873 break;
4874
4875 case VMX_EXIT_RSM: /* 17 Guest software attempted to execute RSM in SMM. */
4876 AssertFailed(); /* can't happen. */
4877 rc = VERR_EM_INTERPRETER;
4878 break;
4879
4880 case VMX_EXIT_MTF: /* 37 Exit due to Monitor Trap Flag. */
4881 LogFlow(("VMX_EXIT_MTF at %RGv\n", (RTGCPTR)pCtx->rip));
4882 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
4883 rc2 = VMXWriteVmcs(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4884 AssertRC(rc2);
4885 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
4886#if 0
4887 DBGFDoneStepping(pVCpu);
4888#endif
4889 rc = VINF_EM_DBG_STOP;
4890 break;
4891
4892 case VMX_EXIT_VMCALL: /* 18 Guest software executed VMCALL. */
4893 case VMX_EXIT_VMCLEAR: /* 19 Guest software executed VMCLEAR. */
4894 case VMX_EXIT_VMLAUNCH: /* 20 Guest software executed VMLAUNCH. */
4895 case VMX_EXIT_VMPTRLD: /* 21 Guest software executed VMPTRLD. */
4896 case VMX_EXIT_VMPTRST: /* 22 Guest software executed VMPTRST. */
4897 case VMX_EXIT_VMREAD: /* 23 Guest software executed VMREAD. */
4898 case VMX_EXIT_VMRESUME: /* 24 Guest software executed VMRESUME. */
4899 case VMX_EXIT_VMWRITE: /* 25 Guest software executed VMWRITE. */
4900 case VMX_EXIT_VMXOFF: /* 26 Guest software executed VMXOFF. */
4901 case VMX_EXIT_VMXON: /* 27 Guest software executed VMXON. */
4902 /** @todo inject #UD immediately */
4903 rc = VERR_EM_INTERPRETER;
4904 break;
4905
4906 case VMX_EXIT_CPUID: /* 10 Guest software attempted to execute CPUID. */
4907 case VMX_EXIT_RDTSC: /* 16 Guest software attempted to execute RDTSC. */
4908 case VMX_EXIT_INVLPG: /* 14 Guest software attempted to execute INVLPG. */
4909 case VMX_EXIT_MOV_CRX: /* 28 Control-register accesses. */
4910 case VMX_EXIT_MOV_DRX: /* 29 Debug-register accesses. */
4911 case VMX_EXIT_IO_INSTR: /* 30 I/O instruction. */
4912 case VMX_EXIT_RDPMC: /* 15 Guest software attempted to execute RDPMC. */
4913 case VMX_EXIT_RDTSCP: /* 51 Guest software attempted to execute RDTSCP. */
4914 /* already handled above */
4915 AssertMsg( rc == VINF_PGM_CHANGE_MODE
4916 || rc == VINF_EM_RAW_INTERRUPT
4917 || rc == VERR_EM_INTERPRETER
4918 || rc == VINF_EM_RAW_EMULATE_INSTR
4919 || rc == VINF_PGM_SYNC_CR3
4920 || rc == VINF_IOM_R3_IOPORT_READ
4921 || rc == VINF_IOM_R3_IOPORT_WRITE
4922 || rc == VINF_EM_RAW_GUEST_TRAP
4923 || rc == VINF_TRPM_XCPT_DISPATCHED
4924 || rc == VINF_EM_RESCHEDULE_REM,
4925 ("rc = %d\n", VBOXSTRICTRC_VAL(rc)));
4926 break;
4927
4928 case VMX_EXIT_TPR_BELOW_THRESHOLD: /* 43 TPR below threshold. Guest software executed MOV to CR8. */
4929 case VMX_EXIT_RDMSR: /* 31 RDMSR. Guest software attempted to execute RDMSR. */
4930 case VMX_EXIT_WRMSR: /* 32 WRMSR. Guest software attempted to execute WRMSR. */
4931 case VMX_EXIT_PAUSE: /* 40 Guest software attempted to execute PAUSE. */
4932 case VMX_EXIT_MONITOR: /* 39 Guest software attempted to execute MONITOR. */
4933 case VMX_EXIT_APIC_ACCESS: /* 44 APIC access. Guest software attempted to access memory at a physical address
4934 on the APIC-access page. */
4935 {
4936 /*
4937 * If we decided to emulate them here, then we must sync the MSRs that could have been changed (sysenter, FS/GS base)
4938 */
4939 rc = VERR_EM_INTERPRETER;
4940 break;
4941 }
4942
4943 case VMX_EXIT_INT_WINDOW: /* 7 Interrupt window. */
4944 Assert(rc == VINF_EM_RAW_INTERRUPT);
4945 break;
4946
4947 case VMX_EXIT_ERR_INVALID_GUEST_STATE: /* 33 VM-entry failure due to invalid guest state. */
4948 {
4949#ifdef VBOX_STRICT
4950 RTCCUINTREG val2 = 0;
4951
4952 Log(("VMX_EXIT_ERR_INVALID_GUEST_STATE\n"));
4953
4954 VMXReadVmcs(VMX_VMCS_GUEST_RIP, &val2);
4955 Log(("Old eip %RGv new %RGv\n", (RTGCPTR)pCtx->rip, (RTGCPTR)val2));
4956
4957 VMXReadVmcs(VMX_VMCS_GUEST_CR0, &val2);
4958 Log(("VMX_VMCS_GUEST_CR0 %RX64\n", (uint64_t)val2));
4959
4960 VMXReadVmcs(VMX_VMCS_GUEST_CR3, &val2);
4961 Log(("VMX_VMCS_GUEST_CR3 %RX64\n", (uint64_t)val2));
4962
4963 VMXReadVmcs(VMX_VMCS_GUEST_CR4, &val2);
4964 Log(("VMX_VMCS_GUEST_CR4 %RX64\n", (uint64_t)val2));
4965
4966 VMXReadVmcs(VMX_VMCS_GUEST_RFLAGS, &val2);
4967 Log(("VMX_VMCS_GUEST_RFLAGS %08x\n", val2));
4968
4969 VMX_LOG_SELREG(CS, "CS", val2);
4970 VMX_LOG_SELREG(DS, "DS", val2);
4971 VMX_LOG_SELREG(ES, "ES", val2);
4972 VMX_LOG_SELREG(FS, "FS", val2);
4973 VMX_LOG_SELREG(GS, "GS", val2);
4974 VMX_LOG_SELREG(SS, "SS", val2);
4975 VMX_LOG_SELREG(TR, "TR", val2);
4976 VMX_LOG_SELREG(LDTR, "LDTR", val2);
4977
4978 VMXReadVmcs(VMX_VMCS_GUEST_GDTR_BASE, &val2);
4979 Log(("VMX_VMCS_GUEST_GDTR_BASE %RX64\n", (uint64_t)val2));
4980 VMXReadVmcs(VMX_VMCS_GUEST_IDTR_BASE, &val2);
4981 Log(("VMX_VMCS_GUEST_IDTR_BASE %RX64\n", (uint64_t)val2));
4982#endif /* VBOX_STRICT */
4983 rc = VERR_VMX_INVALID_GUEST_STATE;
4984 break;
4985 }
4986
4987 case VMX_EXIT_ERR_MSR_LOAD: /* 34 VM-entry failure due to MSR loading. */
4988 case VMX_EXIT_ERR_MACHINE_CHECK: /* 41 VM-entry failure due to machine-check. */
4989 default:
4990 rc = VERR_VMX_UNEXPECTED_EXIT_CODE;
4991 AssertMsgFailed(("Unexpected exit code %d\n", exitReason)); /* Can't happen. */
4992 break;
4993
4994 }
4995
4996end:
4997 /* We now going back to ring-3, so clear the action flag. */
4998 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
4999
5000 /*
5001 * Signal changes for the recompiler.
5002 */
5003 CPUMSetChangedFlags(pVCpu,
5004 CPUM_CHANGED_SYSENTER_MSR
5005 | CPUM_CHANGED_LDTR
5006 | CPUM_CHANGED_GDTR
5007 | CPUM_CHANGED_IDTR
5008 | CPUM_CHANGED_TR
5009 | CPUM_CHANGED_HIDDEN_SEL_REGS);
5010
5011 /*
5012 * If we executed vmlaunch/vmresume and an external IRQ was pending, then we don't have to do a full sync the next time.
5013 */
5014 if ( exitReason == VMX_EXIT_EXT_INT
5015 && !VMX_EXIT_INTERRUPTION_INFO_IS_VALID(intInfo))
5016 {
5017 STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
5018 /* On the next entry we'll only sync the host context. */
5019 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_HOST_CONTEXT;
5020 }
5021 else
5022 {
5023 /* On the next entry we'll sync everything. */
5024 /** @todo we can do better than this */
5025 /* Not in the VINF_PGM_CHANGE_MODE though! */
5026 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_ALL;
5027 }
5028
5029 /* Translate into a less severe return code */
5030 if (rc == VERR_EM_INTERPRETER)
5031 rc = VINF_EM_RAW_EMULATE_INSTR;
5032 else if (rc == VERR_VMX_INVALID_VMCS_PTR)
5033 {
5034 /* Try to extract more information about what might have gone wrong here. */
5035 VMXGetActivateVMCS(&pVCpu->hm.s.vmx.lasterror.u64VMCSPhys);
5036 pVCpu->hm.s.vmx.lasterror.u32VMCSRevision = *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs;
5037 pVCpu->hm.s.vmx.lasterror.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
5038 pVCpu->hm.s.vmx.lasterror.idCurrentCpu = RTMpCpuId();
5039 }
5040
5041 /* Just set the correct state here instead of trying to catch every goto above. */
5042 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
5043
5044#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
5045 /* Restore interrupts if we exited after disabling them. */
5046 if (uOldEFlags != ~(RTCCUINTREG)0)
5047 ASMSetFlags(uOldEFlags);
5048#endif
5049
5050 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
5051 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
5052 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
5053 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
5054 Log2(("X"));
5055 return VBOXSTRICTRC_TODO(rc);
5056}
5057
5058
5059/**
5060 * Enters the VT-x session.
5061 *
5062 * @returns VBox status code.
5063 * @param pVM Pointer to the VM.
5064 * @param pVCpu Pointer to the VMCPU.
5065 * @param pCpu Pointer to the CPU info struct.
5066 */
5067VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBLCPUINFO pCpu)
5068{
5069 Assert(pVM->hm.s.vmx.fSupported);
5070 NOREF(pCpu);
5071
5072 unsigned cr4 = ASMGetCR4();
5073 if (!(cr4 & X86_CR4_VMXE))
5074 {
5075 AssertMsgFailed(("X86_CR4_VMXE should be set!\n"));
5076 return VERR_VMX_X86_CR4_VMXE_CLEARED;
5077 }
5078
5079 /* Activate the VMCS. */
5080 int rc = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
5081 if (RT_FAILURE(rc))
5082 return rc;
5083
5084 pVCpu->hm.s.fResumeVM = false;
5085 return VINF_SUCCESS;
5086}
5087
5088
5089/**
5090 * Leaves the VT-x session.
5091 *
5092 * @returns VBox status code.
5093 * @param pVM Pointer to the VM.
5094 * @param pVCpu Pointer to the VMCPU.
5095 * @param pCtx Pointer to the guests CPU context.
5096 */
5097VMMR0DECL(int) VMXR0Leave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
5098{
5099 Assert(pVM->hm.s.vmx.fSupported);
5100
5101#ifdef DEBUG
5102 if (CPUMIsHyperDebugStateActive(pVCpu))
5103 {
5104 CPUMR0LoadHostDebugState(pVM, pVCpu);
5105 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
5106 }
5107 else
5108#endif
5109
5110 /*
5111 * Save the guest debug state if necessary.
5112 */
5113 if (CPUMIsGuestDebugStateActive(pVCpu))
5114 {
5115 CPUMR0SaveGuestDebugState(pVM, pVCpu, pCtx, true /* save DR6 */);
5116
5117 /* Enable DRx move intercepts again. */
5118 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
5119 int rc = VMXWriteVmcs(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
5120 AssertRC(rc);
5121
5122 /* Resync the debug registers the next time. */
5123 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_DEBUG;
5124 }
5125 else
5126 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
5127
5128 /*
5129 * Clear VMCS, marking it inactive, clearing implementation-specific data and writing
5130 * VMCS data back to memory.
5131 */
5132 int rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
5133 AssertRC(rc);
5134
5135 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
5136 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
5137 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
5138 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
5139 return VINF_SUCCESS;
5140}
5141
5142
5143/**
5144 * Flush the TLB using EPT.
5145 *
5146 * @returns VBox status code.
5147 * @param pVM Pointer to the VM.
5148 * @param pVCpu Pointer to the VMCPU.
5149 * @param enmFlush Type of flush.
5150 */
5151static void hmR0VmxFlushEPT(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush)
5152{
5153 uint64_t descriptor[2];
5154
5155 LogFlow(("hmR0VmxFlushEPT %d\n", enmFlush));
5156 Assert(pVM->hm.s.fNestedPaging);
5157 descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
5158 descriptor[1] = 0; /* MBZ. Intel spec. 33.3 VMX Instructions */
5159 int rc = VMXR0InvEPT(enmFlush, &descriptor[0]);
5160 AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %x %RGv failed with %d\n", enmFlush, pVCpu->hm.s.vmx.HCPhysEPTP, rc));
5161#ifdef VBOX_WITH_STATISTICS
5162 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
5163#endif
5164}
5165
5166
5167/**
5168 * Flush the TLB using VPID.
5169 *
5170 * @returns VBox status code.
5171 * @param pVM Pointer to the VM.
5172 * @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
5173 * enmFlush).
5174 * @param enmFlush Type of flush.
5175 * @param GCPtr Virtual address of the page to flush (can be 0 depending
5176 * on @a enmFlush).
5177 */
5178static void hmR0VmxFlushVPID(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr)
5179{
5180 uint64_t descriptor[2];
5181
5182 Assert(pVM->hm.s.vmx.fVpid);
5183 if (enmFlush == VMX_FLUSH_VPID_ALL_CONTEXTS)
5184 {
5185 descriptor[0] = 0;
5186 descriptor[1] = 0;
5187 }
5188 else
5189 {
5190 AssertPtr(pVCpu);
5191 AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
5192 AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
5193 descriptor[0] = pVCpu->hm.s.uCurrentAsid;
5194 descriptor[1] = GCPtr;
5195 }
5196 int rc = VMXR0InvVPID(enmFlush, &descriptor[0]); NOREF(rc);
5197 AssertMsg(rc == VINF_SUCCESS,
5198 ("VMXR0InvVPID %x %x %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
5199#ifdef VBOX_WITH_STATISTICS
5200 if (pVCpu)
5201 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
5202#endif
5203}
5204
5205
5206/**
5207 * Invalidates a guest page by guest virtual address. Only relevant for
5208 * EPT/VPID, otherwise there is nothing really to invalidate.
5209 *
5210 * @returns VBox status code.
5211 * @param pVM Pointer to the VM.
5212 * @param pVCpu Pointer to the VMCPU.
5213 * @param GCVirt Guest virtual address of the page to invalidate.
5214 */
5215VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
5216{
5217 bool fFlushPending = VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
5218
5219 Log2(("VMXR0InvalidatePage %RGv\n", GCVirt));
5220
5221 if (!fFlushPending)
5222 {
5223 /*
5224 * We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
5225 * See @bugref{6043} and @bugref{6177}
5226 *
5227 * Set the VMCPU_FF_TLB_FLUSH force flag and flush before VMENTRY in hmR0VmxSetupTLB*() as this
5228 * function maybe called in a loop with individual addresses.
5229 */
5230 if (pVM->hm.s.vmx.fVpid)
5231 {
5232 /* If we can flush just this page do it, otherwise flush as little as possible. */
5233 if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
5234 hmR0VmxFlushVPID(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, GCVirt);
5235 else
5236 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
5237 }
5238 else if (pVM->hm.s.fNestedPaging)
5239 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
5240 }
5241
5242 return VINF_SUCCESS;
5243}
5244
5245
5246/**
5247 * Invalidates a guest page by physical address. Only relevant for EPT/VPID,
5248 * otherwise there is nothing really to invalidate.
5249 *
5250 * NOTE: Assumes the current instruction references this physical page though a virtual address!!
5251 *
5252 * @returns VBox status code.
5253 * @param pVM Pointer to the VM.
5254 * @param pVCpu Pointer to the VMCPU.
5255 * @param GCPhys Guest physical address of the page to invalidate.
5256 */
5257VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
5258{
5259 LogFlow(("VMXR0InvalidatePhysPage %RGp\n", GCPhys));
5260
5261 /*
5262 * We cannot flush a page by guest-physical address. invvpid takes only a linear address
5263 * while invept only flushes by EPT not individual addresses. We update the force flag here
5264 * and flush before VMENTRY in hmR0VmxSetupTLB*(). This function might be called in a loop.
5265 */
5266 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
5267 return VINF_SUCCESS;
5268}
5269
5270
5271/**
5272 * Report world switch error and dump some useful debug info.
5273 *
5274 * @param pVM Pointer to the VM.
5275 * @param pVCpu Pointer to the VMCPU.
5276 * @param rc Return code.
5277 * @param pCtx Pointer to the current guest CPU context (not updated).
5278 */
5279static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, VBOXSTRICTRC rc, PCPUMCTX pCtx)
5280{
5281 NOREF(pVM);
5282
5283 switch (VBOXSTRICTRC_VAL(rc))
5284 {
5285 case VERR_VMX_INVALID_VMXON_PTR:
5286 AssertFailed();
5287 break;
5288
5289 case VERR_VMX_UNABLE_TO_START_VM:
5290 {
5291 int rc2;
5292 RTCCUINTREG exitReason, instrError;
5293
5294 rc2 = VMXReadVmcs(VMX_VMCS32_RO_EXIT_REASON, &exitReason);
5295 rc2 |= VMXReadVmcs(VMX_VMCS32_RO_VM_INSTR_ERROR, &instrError);
5296 AssertRC(rc2);
5297 if (rc2 == VINF_SUCCESS)
5298 {
5299 Log(("Unable to start/resume VM for reason: %x. Instruction error %x\n", (uint32_t)exitReason,
5300 (uint32_t)instrError));
5301 Log(("Current stack %08x\n", &rc2));
5302
5303 pVCpu->hm.s.vmx.lasterror.u32InstrError = instrError;
5304 pVCpu->hm.s.vmx.lasterror.u32ExitReason = exitReason;
5305
5306#ifdef VBOX_STRICT
5307 RTGDTR gdtr;
5308 PCX86DESCHC pDesc;
5309 RTCCUINTREG val;
5310
5311 ASMGetGDTR(&gdtr);
5312
5313 VMXReadVmcs(VMX_VMCS_GUEST_RIP, &val);
5314 Log(("Old eip %RGv new %RGv\n", (RTGCPTR)pCtx->rip, (RTGCPTR)val));
5315 VMXReadVmcs(VMX_VMCS32_CTRL_PIN_EXEC, &val);
5316 Log(("VMX_VMCS_CTRL_PIN_EXEC %08x\n", val));
5317 VMXReadVmcs(VMX_VMCS32_CTRL_PROC_EXEC, &val);
5318 Log(("VMX_VMCS_CTRL_PROC_EXEC %08x\n", val));
5319 VMXReadVmcs(VMX_VMCS32_CTRL_ENTRY, &val);
5320 Log(("VMX_VMCS_CTRL_ENTRY %08x\n", val));
5321 VMXReadVmcs(VMX_VMCS32_CTRL_EXIT, &val);
5322 Log(("VMX_VMCS_CTRL_EXIT %08x\n", val));
5323
5324 VMXReadVmcs(VMX_VMCS_HOST_CR0, &val);
5325 Log(("VMX_VMCS_HOST_CR0 %08x\n", val));
5326 VMXReadVmcs(VMX_VMCS_HOST_CR3, &val);
5327 Log(("VMX_VMCS_HOST_CR3 %08x\n", val));
5328 VMXReadVmcs(VMX_VMCS_HOST_CR4, &val);
5329 Log(("VMX_VMCS_HOST_CR4 %08x\n", val));
5330
5331 VMXReadVmcs(VMX_VMCS16_HOST_FIELD_CS, &val);
5332 Log(("VMX_VMCS_HOST_FIELD_CS %08x\n", val));
5333 VMXReadVmcs(VMX_VMCS_GUEST_RFLAGS, &val);
5334 Log(("VMX_VMCS_GUEST_RFLAGS %08x\n", val));
5335
5336 if (val < gdtr.cbGdt)
5337 {
5338 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
5339 HMR0DumpDescriptor(pDesc, val, "CS: ");
5340 }
5341
5342 VMXReadVmcs(VMX_VMCS16_HOST_FIELD_DS, &val);
5343 Log(("VMX_VMCS_HOST_FIELD_DS %08x\n", val));
5344 if (val < gdtr.cbGdt)
5345 {
5346 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
5347 HMR0DumpDescriptor(pDesc, val, "DS: ");
5348 }
5349
5350 VMXReadVmcs(VMX_VMCS16_HOST_FIELD_ES, &val);
5351 Log(("VMX_VMCS_HOST_FIELD_ES %08x\n", val));
5352 if (val < gdtr.cbGdt)
5353 {
5354 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
5355 HMR0DumpDescriptor(pDesc, val, "ES: ");
5356 }
5357
5358 VMXReadVmcs(VMX_VMCS16_HOST_FIELD_FS, &val);
5359 Log(("VMX_VMCS16_HOST_FIELD_FS %08x\n", val));
5360 if (val < gdtr.cbGdt)
5361 {
5362 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
5363 HMR0DumpDescriptor(pDesc, val, "FS: ");
5364 }
5365
5366 VMXReadVmcs(VMX_VMCS16_HOST_FIELD_GS, &val);
5367 Log(("VMX_VMCS16_HOST_FIELD_GS %08x\n", val));
5368 if (val < gdtr.cbGdt)
5369 {
5370 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
5371 HMR0DumpDescriptor(pDesc, val, "GS: ");
5372 }
5373
5374 VMXReadVmcs(VMX_VMCS16_HOST_FIELD_SS, &val);
5375 Log(("VMX_VMCS16_HOST_FIELD_SS %08x\n", val));
5376 if (val < gdtr.cbGdt)
5377 {
5378 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
5379 HMR0DumpDescriptor(pDesc, val, "SS: ");
5380 }
5381
5382 VMXReadVmcs(VMX_VMCS16_HOST_FIELD_TR, &val);
5383 Log(("VMX_VMCS16_HOST_FIELD_TR %08x\n", val));
5384 if (val < gdtr.cbGdt)
5385 {
5386 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
5387 HMR0DumpDescriptor(pDesc, val, "TR: ");
5388 }
5389
5390 VMXReadVmcs(VMX_VMCS_HOST_TR_BASE, &val);
5391 Log(("VMX_VMCS_HOST_TR_BASE %RHv\n", val));
5392 VMXReadVmcs(VMX_VMCS_HOST_GDTR_BASE, &val);
5393 Log(("VMX_VMCS_HOST_GDTR_BASE %RHv\n", val));
5394 VMXReadVmcs(VMX_VMCS_HOST_IDTR_BASE, &val);
5395 Log(("VMX_VMCS_HOST_IDTR_BASE %RHv\n", val));
5396 VMXReadVmcs(VMX_VMCS32_HOST_SYSENTER_CS, &val);
5397 Log(("VMX_VMCS_HOST_SYSENTER_CS %08x\n", val));
5398 VMXReadVmcs(VMX_VMCS_HOST_SYSENTER_EIP, &val);
5399 Log(("VMX_VMCS_HOST_SYSENTER_EIP %RHv\n", val));
5400 VMXReadVmcs(VMX_VMCS_HOST_SYSENTER_ESP, &val);
5401 Log(("VMX_VMCS_HOST_SYSENTER_ESP %RHv\n", val));
5402 VMXReadVmcs(VMX_VMCS_HOST_RSP, &val);
5403 Log(("VMX_VMCS_HOST_RSP %RHv\n", val));
5404 VMXReadVmcs(VMX_VMCS_HOST_RIP, &val);
5405 Log(("VMX_VMCS_HOST_RIP %RHv\n", val));
5406# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
5407 if (VMX_IS_64BIT_HOST_MODE())
5408 {
5409 Log(("MSR_K6_EFER = %RX64\n", ASMRdMsr(MSR_K6_EFER)));
5410 Log(("MSR_K6_STAR = %RX64\n", ASMRdMsr(MSR_K6_STAR)));
5411 Log(("MSR_K8_LSTAR = %RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
5412 Log(("MSR_K8_CSTAR = %RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
5413 Log(("MSR_K8_SF_MASK = %RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
5414 Log(("MSR_K8_KERNEL_GS_BASE = %RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
5415 }
5416# endif
5417#endif /* VBOX_STRICT */
5418 }
5419 break;
5420 }
5421
5422 default:
5423 /* impossible */
5424 AssertMsgFailed(("%Rrc (%#x)\n", VBOXSTRICTRC_VAL(rc), VBOXSTRICTRC_VAL(rc)));
5425 break;
5426 }
5427}
5428
5429
5430#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
5431/**
5432 * Prepares for and executes VMLAUNCH (64 bits guest mode).
5433 *
5434 * @returns VBox status code.
5435 * @param fResume Whether to vmlauch/vmresume.
5436 * @param pCtx Pointer to the guest CPU context.
5437 * @param pCache Pointer to the VMCS cache.
5438 * @param pVM Pointer to the VM.
5439 * @param pVCpu Pointer to the VMCPU.
5440 */
5441DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
5442{
5443 uint32_t aParam[6];
5444 PHMGLOBLCPUINFO pCpu;
5445 RTHCPHYS HCPhysCpuPage;
5446 int rc;
5447
5448 pCpu = HMR0GetCurrentCpu();
5449 HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
5450
5451#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5452 pCache->uPos = 1;
5453 pCache->interPD = PGMGetInterPaeCR3(pVM);
5454 pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
5455#endif
5456
5457#ifdef DEBUG
5458 pCache->TestIn.HCPhysCpuPage= 0;
5459 pCache->TestIn.HCPhysVmcs = 0;
5460 pCache->TestIn.pCache = 0;
5461 pCache->TestOut.HCPhysVmcs = 0;
5462 pCache->TestOut.pCache = 0;
5463 pCache->TestOut.pCtx = 0;
5464 pCache->TestOut.eflags = 0;
5465#endif
5466
5467 aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5468 aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
5469 aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
5470 aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
5471 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
5472 aParam[5] = 0;
5473
5474#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5475 pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
5476 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
5477#endif
5478 rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, 6, &aParam[0]);
5479
5480#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5481 Assert(*(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
5482 Assert(pCtx->dr[4] == 10);
5483 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
5484#endif
5485
5486#ifdef DEBUG
5487 AssertMsg(pCache->TestIn.HCPhysCpuPage== HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5488 AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5489 pVCpu->hm.s.vmx.HCPhysVmcs));
5490 AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5491 pCache->TestOut.HCPhysVmcs));
5492 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5493 pCache->TestOut.pCache));
5494 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5495 ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5496 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5497 pCache->TestOut.pCtx));
5498 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5499#endif
5500 return rc;
5501}
5502
5503
5504#ifdef VBOX_STRICT
5505static bool hmR0VmxIsValidReadField(uint32_t idxField)
5506{
5507 switch (idxField)
5508 {
5509 case VMX_VMCS_GUEST_RIP:
5510 case VMX_VMCS_GUEST_RSP:
5511 case VMX_VMCS_GUEST_RFLAGS:
5512 case VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE:
5513 case VMX_VMCS_CTRL_CR0_READ_SHADOW:
5514 case VMX_VMCS_GUEST_CR0:
5515 case VMX_VMCS_CTRL_CR4_READ_SHADOW:
5516 case VMX_VMCS_GUEST_CR4:
5517 case VMX_VMCS_GUEST_DR7:
5518 case VMX_VMCS32_GUEST_SYSENTER_CS:
5519 case VMX_VMCS_GUEST_SYSENTER_EIP:
5520 case VMX_VMCS_GUEST_SYSENTER_ESP:
5521 case VMX_VMCS32_GUEST_GDTR_LIMIT:
5522 case VMX_VMCS_GUEST_GDTR_BASE:
5523 case VMX_VMCS32_GUEST_IDTR_LIMIT:
5524 case VMX_VMCS_GUEST_IDTR_BASE:
5525 case VMX_VMCS16_GUEST_FIELD_CS:
5526 case VMX_VMCS32_GUEST_CS_LIMIT:
5527 case VMX_VMCS_GUEST_CS_BASE:
5528 case VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS:
5529 case VMX_VMCS16_GUEST_FIELD_DS:
5530 case VMX_VMCS32_GUEST_DS_LIMIT:
5531 case VMX_VMCS_GUEST_DS_BASE:
5532 case VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS:
5533 case VMX_VMCS16_GUEST_FIELD_ES:
5534 case VMX_VMCS32_GUEST_ES_LIMIT:
5535 case VMX_VMCS_GUEST_ES_BASE:
5536 case VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS:
5537 case VMX_VMCS16_GUEST_FIELD_FS:
5538 case VMX_VMCS32_GUEST_FS_LIMIT:
5539 case VMX_VMCS_GUEST_FS_BASE:
5540 case VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS:
5541 case VMX_VMCS16_GUEST_FIELD_GS:
5542 case VMX_VMCS32_GUEST_GS_LIMIT:
5543 case VMX_VMCS_GUEST_GS_BASE:
5544 case VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS:
5545 case VMX_VMCS16_GUEST_FIELD_SS:
5546 case VMX_VMCS32_GUEST_SS_LIMIT:
5547 case VMX_VMCS_GUEST_SS_BASE:
5548 case VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS:
5549 case VMX_VMCS16_GUEST_FIELD_LDTR:
5550 case VMX_VMCS32_GUEST_LDTR_LIMIT:
5551 case VMX_VMCS_GUEST_LDTR_BASE:
5552 case VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS:
5553 case VMX_VMCS16_GUEST_FIELD_TR:
5554 case VMX_VMCS32_GUEST_TR_LIMIT:
5555 case VMX_VMCS_GUEST_TR_BASE:
5556 case VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS:
5557 case VMX_VMCS32_RO_EXIT_REASON:
5558 case VMX_VMCS32_RO_VM_INSTR_ERROR:
5559 case VMX_VMCS32_RO_EXIT_INSTR_LENGTH:
5560 case VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE:
5561 case VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO:
5562 case VMX_VMCS32_RO_EXIT_INSTR_INFO:
5563 case VMX_VMCS_RO_EXIT_QUALIFICATION:
5564 case VMX_VMCS32_RO_IDT_INFO:
5565 case VMX_VMCS32_RO_IDT_ERROR_CODE:
5566 case VMX_VMCS_GUEST_CR3:
5567 case VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL:
5568 return true;
5569 }
5570 return false;
5571}
5572
5573
5574static bool hmR0VmxIsValidWriteField(uint32_t idxField)
5575{
5576 switch (idxField)
5577 {
5578 case VMX_VMCS_GUEST_LDTR_BASE:
5579 case VMX_VMCS_GUEST_TR_BASE:
5580 case VMX_VMCS_GUEST_GDTR_BASE:
5581 case VMX_VMCS_GUEST_IDTR_BASE:
5582 case VMX_VMCS_GUEST_SYSENTER_EIP:
5583 case VMX_VMCS_GUEST_SYSENTER_ESP:
5584 case VMX_VMCS_GUEST_CR0:
5585 case VMX_VMCS_GUEST_CR4:
5586 case VMX_VMCS_GUEST_CR3:
5587 case VMX_VMCS_GUEST_DR7:
5588 case VMX_VMCS_GUEST_RIP:
5589 case VMX_VMCS_GUEST_RSP:
5590 case VMX_VMCS_GUEST_CS_BASE:
5591 case VMX_VMCS_GUEST_DS_BASE:
5592 case VMX_VMCS_GUEST_ES_BASE:
5593 case VMX_VMCS_GUEST_FS_BASE:
5594 case VMX_VMCS_GUEST_GS_BASE:
5595 case VMX_VMCS_GUEST_SS_BASE:
5596 return true;
5597 }
5598 return false;
5599}
5600#endif /* VBOX_STRICT */
5601
5602
5603/**
5604 * Executes the specified handler in 64-bit mode.
5605 *
5606 * @returns VBox status code.
5607 * @param pVM Pointer to the VM.
5608 * @param pVCpu Pointer to the VMCPU.
5609 * @param pCtx Pointer to the guest CPU context.
5610 * @param enmOp The operation to perform.
5611 * @param cbParam Number of parameters.
5612 * @param paParam Array of 32-bit parameters.
5613 */
5614VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp, uint32_t cbParam,
5615 uint32_t *paParam)
5616{
5617 int rc, rc2;
5618 PHMGLOBLCPUINFO pCpu;
5619 RTHCPHYS HCPhysCpuPage;
5620 RTHCUINTREG uOldEFlags;
5621
5622 AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
5623 Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
5624 Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
5625 Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
5626
5627#ifdef VBOX_STRICT
5628 for (unsigned i=0;i<pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries;i++)
5629 Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
5630
5631 for (unsigned i=0;i<pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries;i++)
5632 Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
5633#endif
5634
5635 /* Disable interrupts. */
5636 uOldEFlags = ASMIntDisableFlags();
5637
5638#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
5639 RTCPUID idHostCpu = RTMpCpuId();
5640 CPUMR0SetLApic(pVM, idHostCpu);
5641#endif
5642
5643 pCpu = HMR0GetCurrentCpu();
5644 HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
5645
5646 /* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
5647 VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
5648
5649 /* Leave VMX Root Mode. */
5650 VMXDisable();
5651
5652 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
5653
5654 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
5655 CPUMSetHyperEIP(pVCpu, enmOp);
5656 for (int i=(int)cbParam-1;i>=0;i--)
5657 CPUMPushHyper(pVCpu, paParam[i]);
5658
5659 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
5660
5661 /* Call switcher. */
5662 rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
5663 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
5664
5665 /* Make sure the VMX instructions don't cause #UD faults. */
5666 ASMSetCR4(ASMGetCR4() | X86_CR4_VMXE);
5667
5668 /* Enter VMX Root Mode */
5669 rc2 = VMXEnable(HCPhysCpuPage);
5670 if (RT_FAILURE(rc2))
5671 {
5672 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
5673 ASMSetFlags(uOldEFlags);
5674 return VERR_VMX_VMXON_FAILED;
5675 }
5676
5677 rc2 = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
5678 AssertRC(rc2);
5679 Assert(!(ASMGetFlags() & X86_EFL_IF));
5680 ASMSetFlags(uOldEFlags);
5681 return rc;
5682}
5683#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
5684
5685
5686#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
5687/**
5688 * Executes VMWRITE.
5689 *
5690 * @returns VBox status code
5691 * @param pVCpu Pointer to the VMCPU.
5692 * @param idxField VMCS field index.
5693 * @param u64Val 16, 32 or 64 bits value.
5694 */
5695VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5696{
5697 int rc;
5698 switch (idxField)
5699 {
5700 case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5701 case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5702 case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5703 case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5704 case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5705 case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5706 case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5707 case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
5708 case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5709 case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5710 case VMX_VMCS64_GUEST_PDPTE0_FULL:
5711 case VMX_VMCS64_GUEST_PDPTE1_FULL:
5712 case VMX_VMCS64_GUEST_PDPTE2_FULL:
5713 case VMX_VMCS64_GUEST_PDPTE3_FULL:
5714 case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5715 case VMX_VMCS64_GUEST_EFER_FULL:
5716 case VMX_VMCS64_CTRL_EPTP_FULL:
5717 /* These fields consist of two parts, which are both writable in 32 bits mode. */
5718 rc = VMXWriteVmcs32(idxField, u64Val);
5719 rc |= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32ULL));
5720 AssertRC(rc);
5721 return rc;
5722
5723 case VMX_VMCS_GUEST_LDTR_BASE:
5724 case VMX_VMCS_GUEST_TR_BASE:
5725 case VMX_VMCS_GUEST_GDTR_BASE:
5726 case VMX_VMCS_GUEST_IDTR_BASE:
5727 case VMX_VMCS_GUEST_SYSENTER_EIP:
5728 case VMX_VMCS_GUEST_SYSENTER_ESP:
5729 case VMX_VMCS_GUEST_CR0:
5730 case VMX_VMCS_GUEST_CR4:
5731 case VMX_VMCS_GUEST_CR3:
5732 case VMX_VMCS_GUEST_DR7:
5733 case VMX_VMCS_GUEST_RIP:
5734 case VMX_VMCS_GUEST_RSP:
5735 case VMX_VMCS_GUEST_CS_BASE:
5736 case VMX_VMCS_GUEST_DS_BASE:
5737 case VMX_VMCS_GUEST_ES_BASE:
5738 case VMX_VMCS_GUEST_FS_BASE:
5739 case VMX_VMCS_GUEST_GS_BASE:
5740 case VMX_VMCS_GUEST_SS_BASE:
5741 /* Queue a 64 bits value as we can't set it in 32 bits host mode. */
5742 if (u64Val >> 32ULL)
5743 rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5744 else
5745 rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
5746
5747 return rc;
5748
5749 default:
5750 AssertMsgFailed(("Unexpected field %x\n", idxField));
5751 return VERR_INVALID_PARAMETER;
5752 }
5753}
5754
5755
5756/**
5757 * Cache VMCS writes for running 64 bits guests on 32 bits hosts.
5758 *
5759 * @param pVCpu Pointer to the VMCPU.
5760 * @param idxField VMCS field index.
5761 * @param u64Val 16, 32 or 64 bits value.
5762 */
5763VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5764{
5765 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5766
5767 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5768 ("entries=%x\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5769
5770 /* Make sure there are no duplicates. */
5771 for (unsigned i = 0; i < pCache->Write.cValidEntries; i++)
5772 {
5773 if (pCache->Write.aField[i] == idxField)
5774 {
5775 pCache->Write.aFieldVal[i] = u64Val;
5776 return VINF_SUCCESS;
5777 }
5778 }
5779
5780 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5781 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5782 pCache->Write.cValidEntries++;
5783 return VINF_SUCCESS;
5784}
5785
5786#endif /* HC_ARCH_BITS == 32 && !VBOX_WITH_HYBRID_32BIT_KERNEL */
5787
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette