VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HWVMXR0.cpp@ 38684

Last change on this file since 38684 was 38684, checked in by vboxsync, 13 years ago

HM: some file headers
  • Property svn:eol-style set to native
  • Property svn:keywords set to Id
File size: 199.3 KB
Line 
1/* $Id: HWVMXR0.cpp 38684 2011-09-08 08:36:48Z vboxsync $ */
2/** @file
3 * HM VMX (VT-x) - Host Context Ring 0.
4 */
5
6/*
7 * Copyright (C) 2006-2011 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18
19/*******************************************************************************
20* Header Files *
21*******************************************************************************/
22#define LOG_GROUP LOG_GROUP_HWACCM
23#include <iprt/asm-amd64-x86.h>
24#include <VBox/vmm/hwaccm.h>
25#include <VBox/vmm/pgm.h>
26#include <VBox/vmm/dbgf.h>
27#include <VBox/vmm/selm.h>
28#include <VBox/vmm/iom.h>
29#include <VBox/vmm/rem.h>
30#include <VBox/vmm/tm.h>
31#include "HWACCMInternal.h"
32#include <VBox/vmm/vm.h>
33#include <VBox/vmm/pdmapi.h>
34#include <VBox/err.h>
35#include <VBox/log.h>
36#include <iprt/assert.h>
37#include <iprt/param.h>
38#include <iprt/string.h>
39#include <iprt/time.h>
40#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
41# include <iprt/thread.h>
42#endif
43#include <iprt/x86.h>
44#include "HWVMXR0.h"
45
46/*******************************************************************************
47* Defined Constants And Macros *
48*******************************************************************************/
49#if defined(RT_ARCH_AMD64)
50# define VMX_IS_64BIT_HOST_MODE() (true)
51#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
52# define VMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
53#else
54# define VMX_IS_64BIT_HOST_MODE() (false)
55#endif
56
57/*******************************************************************************
58* Global Variables *
59*******************************************************************************/
60/* IO operation lookup arrays. */
61static uint32_t const g_aIOSize[4] = {1, 2, 0, 4};
62static uint32_t const g_aIOOpAnd[4] = {0xff, 0xffff, 0, 0xffffffff};
63
64#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
65/** See HWACCMR0A.asm. */
66extern "C" uint32_t g_fVMXIs64bitHost;
67#endif
68
69/*******************************************************************************
70* Local Functions *
71*******************************************************************************/
72static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, VBOXSTRICTRC rc, PCPUMCTX pCtx);
73static void hmR0VmxSetupTLBEPT(PVM pVM, PVMCPU pVCpu);
74static void hmR0VmxSetupTLBVPID(PVM pVM, PVMCPU pVCpu);
75static void hmR0VmxSetupTLBDummy(PVM pVM, PVMCPU pVCpu);
76static void hmR0VmxFlushEPT(PVM pVM, PVMCPU pVCpu, VMX_FLUSH enmFlush, RTGCPHYS GCPhys);
77static void hmR0VmxFlushVPID(PVM pVM, PVMCPU pVCpu, VMX_FLUSH enmFlush, RTGCPTR GCPtr);
78static void hmR0VmxUpdateExceptionBitmap(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
79#ifdef VBOX_STRICT
80static bool hmR0VmxIsValidReadField(uint32_t idxField);
81static bool hmR0VmxIsValidWriteField(uint32_t idxField);
82#endif
83static void hmR0VmxSetMSRPermission(PVMCPU pVCpu, unsigned ulMSR, bool fRead, bool fWrite);
84
85
86static void hmR0VmxCheckError(PVM pVM, PVMCPU pVCpu, int rc)
87{
88 if (rc == VERR_VMX_GENERIC)
89 {
90 RTCCUINTREG instrError;
91
92 VMXReadVMCS(VMX_VMCS32_RO_VM_INSTR_ERROR, &instrError);
93 pVCpu->hwaccm.s.vmx.lasterror.ulInstrError = instrError;
94 }
95 pVM->hwaccm.s.lLastError = rc;
96}
97
98/**
99 * Sets up and activates VT-x on the current CPU
100 *
101 * @returns VBox status code.
102 * @param pCpu CPU info struct
103 * @param pVM The VM to operate on. (can be NULL after a resume!!)
104 * @param pvCpuPage Pointer to the global cpu page.
105 * @param HCPhysCpuPage Physical address of the global cpu page.
106 */
107VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBLCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
108{
109 AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
110 AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
111
112 if (pVM)
113 {
114 /* Set revision dword at the beginning of the VMXON structure. */
115 *(uint32_t *)pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hwaccm.s.vmx.msr.vmx_basic_info);
116 }
117
118 /** @todo we should unmap the two pages from the virtual address space in order to prevent accidental corruption.
119 * (which can have very bad consequences!!!)
120 */
121
122 if (ASMGetCR4() & X86_CR4_VMXE)
123 return VERR_VMX_IN_VMX_ROOT_MODE;
124
125 /* Make sure the VMX instructions don't cause #UD faults. */
126 ASMSetCR4(ASMGetCR4() | X86_CR4_VMXE);
127
128 /* Enter VMX Root Mode. */
129 int rc = VMXEnable(HCPhysCpuPage);
130 if (RT_FAILURE(rc))
131 {
132 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
133 return VERR_VMX_VMXON_FAILED;
134 }
135 return VINF_SUCCESS;
136}
137
138/**
139 * Deactivates VT-x on the current CPU
140 *
141 * @returns VBox status code.
142 * @param pCpu CPU info struct
143 * @param pvCpuPage Pointer to the global cpu page.
144 * @param HCPhysCpuPage Physical address of the global cpu page.
145 */
146VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBLCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
147{
148 AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
149 AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
150
151 /* If we're somehow not in VMX root mode, then we shouldn't dare leaving it. */
152 if (!(ASMGetCR4() & X86_CR4_VMXE))
153 return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
154
155 /* Leave VMX Root Mode. */
156 VMXDisable();
157
158 /* And clear the X86_CR4_VMXE bit. */
159 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
160 return VINF_SUCCESS;
161}
162
163/**
164 * Does Ring-0 per VM VT-x init.
165 *
166 * @returns VBox status code.
167 * @param pVM The VM to operate on.
168 */
169VMMR0DECL(int) VMXR0InitVM(PVM pVM)
170{
171 int rc;
172
173#ifdef LOG_ENABLED
174 SUPR0Printf("VMXR0InitVM %x\n", pVM);
175#endif
176
177 pVM->hwaccm.s.vmx.pMemObjAPIC = NIL_RTR0MEMOBJ;
178
179 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
180 {
181 /* Allocate one page for the APIC physical page (serves for filtering accesses). */
182 rc = RTR0MemObjAllocCont(&pVM->hwaccm.s.vmx.pMemObjAPIC, PAGE_SIZE, true /* executable R0 mapping */);
183 AssertRC(rc);
184 if (RT_FAILURE(rc))
185 return rc;
186
187 pVM->hwaccm.s.vmx.pAPIC = (uint8_t *)RTR0MemObjAddress(pVM->hwaccm.s.vmx.pMemObjAPIC);
188 pVM->hwaccm.s.vmx.pAPICPhys = RTR0MemObjGetPagePhysAddr(pVM->hwaccm.s.vmx.pMemObjAPIC, 0);
189 ASMMemZero32(pVM->hwaccm.s.vmx.pAPIC, PAGE_SIZE);
190 }
191 else
192 {
193 pVM->hwaccm.s.vmx.pMemObjAPIC = 0;
194 pVM->hwaccm.s.vmx.pAPIC = 0;
195 pVM->hwaccm.s.vmx.pAPICPhys = 0;
196 }
197
198#ifdef VBOX_WITH_CRASHDUMP_MAGIC
199 {
200 rc = RTR0MemObjAllocCont(&pVM->hwaccm.s.vmx.pMemObjScratch, PAGE_SIZE, true /* executable R0 mapping */);
201 AssertRC(rc);
202 if (RT_FAILURE(rc))
203 return rc;
204
205 pVM->hwaccm.s.vmx.pScratch = (uint8_t *)RTR0MemObjAddress(pVM->hwaccm.s.vmx.pMemObjScratch);
206 pVM->hwaccm.s.vmx.pScratchPhys = RTR0MemObjGetPagePhysAddr(pVM->hwaccm.s.vmx.pMemObjScratch, 0);
207
208 ASMMemZero32(pVM->hwaccm.s.vmx.pScratch, PAGE_SIZE);
209 strcpy((char *)pVM->hwaccm.s.vmx.pScratch, "SCRATCH Magic");
210 *(uint64_t *)(pVM->hwaccm.s.vmx.pScratch + 16) = UINT64_C(0xDEADBEEFDEADBEEF);
211 }
212#endif
213
214 /* Allocate VMCBs for all guest CPUs. */
215 for (VMCPUID i = 0; i < pVM->cCpus; i++)
216 {
217 PVMCPU pVCpu = &pVM->aCpus[i];
218
219 pVCpu->hwaccm.s.vmx.hMemObjVMCS = NIL_RTR0MEMOBJ;
220
221 /* Allocate one page for the VM control structure (VMCS). */
222 rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.hMemObjVMCS, PAGE_SIZE, true /* executable R0 mapping */);
223 AssertRC(rc);
224 if (RT_FAILURE(rc))
225 return rc;
226
227 pVCpu->hwaccm.s.vmx.pvVMCS = RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.hMemObjVMCS);
228 pVCpu->hwaccm.s.vmx.HCPhysVMCS = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.hMemObjVMCS, 0);
229 ASMMemZeroPage(pVCpu->hwaccm.s.vmx.pvVMCS);
230
231 pVCpu->hwaccm.s.vmx.cr0_mask = 0;
232 pVCpu->hwaccm.s.vmx.cr4_mask = 0;
233
234 /* Allocate one page for the virtual APIC page for TPR caching. */
235 rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.hMemObjVAPIC, PAGE_SIZE, true /* executable R0 mapping */);
236 AssertRC(rc);
237 if (RT_FAILURE(rc))
238 return rc;
239
240 pVCpu->hwaccm.s.vmx.pbVAPIC = (uint8_t *)RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.hMemObjVAPIC);
241 pVCpu->hwaccm.s.vmx.HCPhysVAPIC = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.hMemObjVAPIC, 0);
242 ASMMemZeroPage(pVCpu->hwaccm.s.vmx.pbVAPIC);
243
244 /* Allocate the MSR bitmap if this feature is supported. */
245 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
246 {
247 rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap, PAGE_SIZE, true /* executable R0 mapping */);
248 AssertRC(rc);
249 if (RT_FAILURE(rc))
250 return rc;
251
252 pVCpu->hwaccm.s.vmx.pMSRBitmap = (uint8_t *)RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap);
253 pVCpu->hwaccm.s.vmx.pMSRBitmapPhys = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap, 0);
254 memset(pVCpu->hwaccm.s.vmx.pMSRBitmap, 0xff, PAGE_SIZE);
255 }
256
257#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
258 /* Allocate one page for the guest MSR load area (for preloading guest MSRs during the world switch). */
259 rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.pMemObjGuestMSR, PAGE_SIZE, true /* executable R0 mapping */);
260 AssertRC(rc);
261 if (RT_FAILURE(rc))
262 return rc;
263
264 pVCpu->hwaccm.s.vmx.pGuestMSR = (uint8_t *)RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.pMemObjGuestMSR);
265 pVCpu->hwaccm.s.vmx.pGuestMSRPhys = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.pMemObjGuestMSR, 0);
266 memset(pVCpu->hwaccm.s.vmx.pGuestMSR, 0, PAGE_SIZE);
267
268 /* Allocate one page for the host MSR load area (for restoring host MSRs after the world switch back). */
269 rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.pMemObjHostMSR, PAGE_SIZE, true /* executable R0 mapping */);
270 AssertRC(rc);
271 if (RT_FAILURE(rc))
272 return rc;
273
274 pVCpu->hwaccm.s.vmx.pHostMSR = (uint8_t *)RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.pMemObjHostMSR);
275 pVCpu->hwaccm.s.vmx.pHostMSRPhys = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.pMemObjHostMSR, 0);
276 memset(pVCpu->hwaccm.s.vmx.pHostMSR, 0, PAGE_SIZE);
277#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
278
279 /* Current guest paging mode. */
280 pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode = PGMMODE_REAL;
281
282#ifdef LOG_ENABLED
283 SUPR0Printf("VMXR0InitVM %x VMCS=%x (%x)\n", pVM, pVCpu->hwaccm.s.vmx.pvVMCS, (uint32_t)pVCpu->hwaccm.s.vmx.HCPhysVMCS);
284#endif
285 }
286
287 return VINF_SUCCESS;
288}
289
290/**
291 * Does Ring-0 per VM VT-x termination.
292 *
293 * @returns VBox status code.
294 * @param pVM The VM to operate on.
295 */
296VMMR0DECL(int) VMXR0TermVM(PVM pVM)
297{
298 for (VMCPUID i = 0; i < pVM->cCpus; i++)
299 {
300 PVMCPU pVCpu = &pVM->aCpus[i];
301
302 if (pVCpu->hwaccm.s.vmx.hMemObjVMCS != NIL_RTR0MEMOBJ)
303 {
304 RTR0MemObjFree(pVCpu->hwaccm.s.vmx.hMemObjVMCS, false);
305 pVCpu->hwaccm.s.vmx.hMemObjVMCS = NIL_RTR0MEMOBJ;
306 pVCpu->hwaccm.s.vmx.pvVMCS = 0;
307 pVCpu->hwaccm.s.vmx.HCPhysVMCS = 0;
308 }
309 if (pVCpu->hwaccm.s.vmx.hMemObjVAPIC != NIL_RTR0MEMOBJ)
310 {
311 RTR0MemObjFree(pVCpu->hwaccm.s.vmx.hMemObjVAPIC, false);
312 pVCpu->hwaccm.s.vmx.hMemObjVAPIC = NIL_RTR0MEMOBJ;
313 pVCpu->hwaccm.s.vmx.pbVAPIC = 0;
314 pVCpu->hwaccm.s.vmx.HCPhysVAPIC = 0;
315 }
316 if (pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap != NIL_RTR0MEMOBJ)
317 {
318 RTR0MemObjFree(pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap, false);
319 pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap = NIL_RTR0MEMOBJ;
320 pVCpu->hwaccm.s.vmx.pMSRBitmap = 0;
321 pVCpu->hwaccm.s.vmx.pMSRBitmapPhys = 0;
322 }
323#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
324 if (pVCpu->hwaccm.s.vmx.pMemObjHostMSR != NIL_RTR0MEMOBJ)
325 {
326 RTR0MemObjFree(pVCpu->hwaccm.s.vmx.pMemObjHostMSR, false);
327 pVCpu->hwaccm.s.vmx.pMemObjHostMSR = NIL_RTR0MEMOBJ;
328 pVCpu->hwaccm.s.vmx.pHostMSR = 0;
329 pVCpu->hwaccm.s.vmx.pHostMSRPhys = 0;
330 }
331 if (pVCpu->hwaccm.s.vmx.pMemObjGuestMSR != NIL_RTR0MEMOBJ)
332 {
333 RTR0MemObjFree(pVCpu->hwaccm.s.vmx.pMemObjGuestMSR, false);
334 pVCpu->hwaccm.s.vmx.pMemObjGuestMSR = NIL_RTR0MEMOBJ;
335 pVCpu->hwaccm.s.vmx.pGuestMSR = 0;
336 pVCpu->hwaccm.s.vmx.pGuestMSRPhys = 0;
337 }
338#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
339 }
340 if (pVM->hwaccm.s.vmx.pMemObjAPIC != NIL_RTR0MEMOBJ)
341 {
342 RTR0MemObjFree(pVM->hwaccm.s.vmx.pMemObjAPIC, false);
343 pVM->hwaccm.s.vmx.pMemObjAPIC = NIL_RTR0MEMOBJ;
344 pVM->hwaccm.s.vmx.pAPIC = 0;
345 pVM->hwaccm.s.vmx.pAPICPhys = 0;
346 }
347#ifdef VBOX_WITH_CRASHDUMP_MAGIC
348 if (pVM->hwaccm.s.vmx.pMemObjScratch != NIL_RTR0MEMOBJ)
349 {
350 ASMMemZero32(pVM->hwaccm.s.vmx.pScratch, PAGE_SIZE);
351 RTR0MemObjFree(pVM->hwaccm.s.vmx.pMemObjScratch, false);
352 pVM->hwaccm.s.vmx.pMemObjScratch = NIL_RTR0MEMOBJ;
353 pVM->hwaccm.s.vmx.pScratch = 0;
354 pVM->hwaccm.s.vmx.pScratchPhys = 0;
355 }
356#endif
357 return VINF_SUCCESS;
358}
359
360/**
361 * Sets up VT-x for the specified VM
362 *
363 * @returns VBox status code.
364 * @param pVM The VM to operate on.
365 */
366VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
367{
368 int rc = VINF_SUCCESS;
369 uint32_t val;
370
371 AssertReturn(pVM, VERR_INVALID_PARAMETER);
372
373 for (VMCPUID i = 0; i < pVM->cCpus; i++)
374 {
375 PVMCPU pVCpu = &pVM->aCpus[i];
376
377 AssertPtr(pVCpu->hwaccm.s.vmx.pvVMCS);
378
379 /* Set revision dword at the beginning of the VMCS structure. */
380 *(uint32_t *)pVCpu->hwaccm.s.vmx.pvVMCS = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hwaccm.s.vmx.msr.vmx_basic_info);
381
382 /* Clear VM Control Structure. */
383 Log(("HCPhysVMCS = %RHp\n", pVCpu->hwaccm.s.vmx.HCPhysVMCS));
384 rc = VMXClearVMCS(pVCpu->hwaccm.s.vmx.HCPhysVMCS);
385 if (RT_FAILURE(rc))
386 goto vmx_end;
387
388 /* Activate the VM Control Structure. */
389 rc = VMXActivateVMCS(pVCpu->hwaccm.s.vmx.HCPhysVMCS);
390 if (RT_FAILURE(rc))
391 goto vmx_end;
392
393 /* VMX_VMCS_CTRL_PIN_EXEC_CONTROLS
394 * Set required bits to one and zero according to the MSR capabilities.
395 */
396 val = pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.disallowed0;
397 /* External and non-maskable interrupts cause VM-exits. */
398 val |= VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_EXT_INT_EXIT | VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_NMI_EXIT;
399 /* enable the preemption timer. */
400 if (pVM->hwaccm.s.vmx.fUsePreemptTimer)
401 val |= VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_PREEMPT_TIMER;
402 val &= pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.allowed1;
403
404 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PIN_EXEC_CONTROLS, val);
405 AssertRC(rc);
406
407 /* VMX_VMCS_CTRL_PROC_EXEC_CONTROLS
408 * Set required bits to one and zero according to the MSR capabilities.
409 */
410 val = pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.disallowed0;
411 /* Program which event cause VM-exits and which features we want to use. */
412 val = val | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_HLT_EXIT
413 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_TSC_OFFSET
414 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT
415 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_UNCOND_IO_EXIT
416 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDPMC_EXIT
417 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_EXIT
418 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MWAIT_EXIT; /* don't execute mwait or else we'll idle inside the guest (host thinks the cpu load is high) */
419
420 /* Without nested paging we should intercept invlpg and cr3 mov instructions. */
421 if (!pVM->hwaccm.s.fNestedPaging)
422 val |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT
423 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
424 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
425
426 /* Note: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MWAIT_EXIT might cause a vmlaunch failure with an invalid control fields error. (combined with some other exit reasons) */
427 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
428 {
429 /* CR8 reads from the APIC shadow page; writes cause an exit is they lower the TPR below the threshold */
430 val |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW;
431 Assert(pVM->hwaccm.s.vmx.pAPIC);
432 }
433 else
434 /* Exit on CR8 reads & writes in case the TPR shadow feature isn't present. */
435 val |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_STORE_EXIT | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_LOAD_EXIT;
436
437 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
438 {
439 Assert(pVCpu->hwaccm.s.vmx.pMSRBitmapPhys);
440 val |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS;
441 }
442
443 /* We will use the secondary control if it's present. */
444 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
445
446 /* Mask away the bits that the CPU doesn't support */
447 /** @todo make sure they don't conflict with the above requirements. */
448 val &= pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1;
449 pVCpu->hwaccm.s.vmx.proc_ctls = val;
450
451 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, val);
452 AssertRC(rc);
453
454 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
455 {
456 /* VMX_VMCS_CTRL_PROC_EXEC_CONTROLS2
457 * Set required bits to one and zero according to the MSR capabilities.
458 */
459 val = pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0;
460 val |= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT;
461
462#ifdef HWACCM_VTX_WITH_EPT
463 if (pVM->hwaccm.s.fNestedPaging)
464 val |= VMX_VMCS_CTRL_PROC_EXEC2_EPT;
465#endif /* HWACCM_VTX_WITH_EPT */
466#ifdef HWACCM_VTX_WITH_VPID
467 else
468 if (pVM->hwaccm.s.vmx.fVPID)
469 val |= VMX_VMCS_CTRL_PROC_EXEC2_VPID;
470#endif /* HWACCM_VTX_WITH_VPID */
471
472 if (pVM->hwaccm.s.fHasIoApic)
473 val |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC;
474
475 if (pVM->hwaccm.s.vmx.fUnrestrictedGuest)
476 val |= VMX_VMCS_CTRL_PROC_EXEC2_REAL_MODE;
477
478 /* Mask away the bits that the CPU doesn't support */
479 /** @todo make sure they don't conflict with the above requirements. */
480 val &= pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1;
481 pVCpu->hwaccm.s.vmx.proc_ctls2 = val;
482 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS2, val);
483 AssertRC(rc);
484 }
485
486 /* VMX_VMCS_CTRL_CR3_TARGET_COUNT
487 * Set required bits to one and zero according to the MSR capabilities.
488 */
489 rc = VMXWriteVMCS(VMX_VMCS_CTRL_CR3_TARGET_COUNT, 0);
490 AssertRC(rc);
491
492 /* Forward all exception except #NM & #PF to the guest.
493 * We always need to check pagefaults since our shadow page table can be out of sync.
494 * And we always lazily sync the FPU & XMM state.
495 */
496
497 /** @todo Possible optimization:
498 * Keep the FPU and XMM state current in the EM thread. That way there's no need to
499 * lazily sync anything, but the downside is that we can't use the FPU stack or XMM
500 * registers ourselves of course.
501 *
502 * Note: only possible if the current state is actually ours (X86_CR0_TS flag)
503 */
504
505 /* Don't filter page faults; all of them should cause a switch. */
506 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PAGEFAULT_ERROR_MASK, 0);
507 rc |= VMXWriteVMCS(VMX_VMCS_CTRL_PAGEFAULT_ERROR_MATCH, 0);
508 AssertRC(rc);
509
510 /* Init TSC offset to zero. */
511 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_TSC_OFFSET_FULL, 0);
512 AssertRC(rc);
513
514 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_IO_BITMAP_A_FULL, 0);
515 AssertRC(rc);
516
517 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_IO_BITMAP_B_FULL, 0);
518 AssertRC(rc);
519
520 /* Set the MSR bitmap address. */
521 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
522 {
523 Assert(pVCpu->hwaccm.s.vmx.pMSRBitmapPhys);
524
525 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_MSR_BITMAP_FULL, pVCpu->hwaccm.s.vmx.pMSRBitmapPhys);
526 AssertRC(rc);
527
528 /* Allow the guest to directly modify these MSRs; they are restored and saved automatically. */
529 hmR0VmxSetMSRPermission(pVCpu, MSR_IA32_SYSENTER_CS, true, true);
530 hmR0VmxSetMSRPermission(pVCpu, MSR_IA32_SYSENTER_ESP, true, true);
531 hmR0VmxSetMSRPermission(pVCpu, MSR_IA32_SYSENTER_EIP, true, true);
532 hmR0VmxSetMSRPermission(pVCpu, MSR_K8_LSTAR, true, true);
533 hmR0VmxSetMSRPermission(pVCpu, MSR_K6_STAR, true, true);
534 hmR0VmxSetMSRPermission(pVCpu, MSR_K8_SF_MASK, true, true);
535 hmR0VmxSetMSRPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, true, true);
536 hmR0VmxSetMSRPermission(pVCpu, MSR_K8_GS_BASE, true, true);
537 hmR0VmxSetMSRPermission(pVCpu, MSR_K8_FS_BASE, true, true);
538 }
539
540#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
541 /* Set the guest & host MSR load/store physical addresses. */
542 Assert(pVCpu->hwaccm.s.vmx.pGuestMSRPhys);
543 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_VMENTRY_MSR_LOAD_FULL, pVCpu->hwaccm.s.vmx.pGuestMSRPhys);
544 AssertRC(rc);
545 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_VMEXIT_MSR_STORE_FULL, pVCpu->hwaccm.s.vmx.pGuestMSRPhys);
546 AssertRC(rc);
547
548 Assert(pVCpu->hwaccm.s.vmx.pHostMSRPhys);
549 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_VMEXIT_MSR_LOAD_FULL, pVCpu->hwaccm.s.vmx.pHostMSRPhys);
550 AssertRC(rc);
551#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
552
553 rc = VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
554 AssertRC(rc);
555
556 rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_MSR_STORE_COUNT, 0);
557 AssertRC(rc);
558
559 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
560 {
561 Assert(pVM->hwaccm.s.vmx.pMemObjAPIC);
562 /* Optional */
563 rc = VMXWriteVMCS(VMX_VMCS_CTRL_TPR_THRESHOLD, 0);
564 rc |= VMXWriteVMCS64(VMX_VMCS_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hwaccm.s.vmx.HCPhysVAPIC);
565
566 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
567 rc |= VMXWriteVMCS64(VMX_VMCS_CTRL_APIC_ACCESSADDR_FULL, pVM->hwaccm.s.vmx.pAPICPhys);
568
569 AssertRC(rc);
570 }
571
572 /* Set link pointer to -1. Not currently used. */
573 rc = VMXWriteVMCS64(VMX_VMCS_GUEST_LINK_PTR_FULL, 0xFFFFFFFFFFFFFFFFULL);
574 AssertRC(rc);
575
576 /* Clear VM Control Structure. Marking it inactive, clearing implementation specific data and writing back VMCS data to memory. */
577 rc = VMXClearVMCS(pVCpu->hwaccm.s.vmx.HCPhysVMCS);
578 AssertRC(rc);
579
580 /* Configure the VMCS read cache. */
581 PVMCSCACHE pCache = &pVCpu->hwaccm.s.vmx.VMCSCache;
582
583 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_RIP);
584 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_RSP);
585 VMXSetupCachedReadVMCS(pCache, VMX_VMCS_GUEST_RFLAGS);
586 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE);
587 VMXSetupCachedReadVMCS(pCache, VMX_VMCS_CTRL_CR0_READ_SHADOW);
588 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_CR0);
589 VMXSetupCachedReadVMCS(pCache, VMX_VMCS_CTRL_CR4_READ_SHADOW);
590 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_CR4);
591 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_DR7);
592 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_SYSENTER_CS);
593 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_SYSENTER_EIP);
594 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_SYSENTER_ESP);
595 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_GDTR_LIMIT);
596 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_GDTR_BASE);
597 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_IDTR_LIMIT);
598 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_IDTR_BASE);
599
600 VMX_SETUP_SELREG(ES, pCache);
601 VMX_SETUP_SELREG(SS, pCache);
602 VMX_SETUP_SELREG(CS, pCache);
603 VMX_SETUP_SELREG(DS, pCache);
604 VMX_SETUP_SELREG(FS, pCache);
605 VMX_SETUP_SELREG(GS, pCache);
606 VMX_SETUP_SELREG(LDTR, pCache);
607 VMX_SETUP_SELREG(TR, pCache);
608
609 /* Status code VMCS reads. */
610 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_REASON);
611 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_VM_INSTR_ERROR);
612 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INSTR_LENGTH);
613 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INTERRUPTION_ERRCODE);
614 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO);
615 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INSTR_INFO);
616 VMXSetupCachedReadVMCS(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
617 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_IDT_INFO);
618 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_IDT_ERRCODE);
619
620 if (pVM->hwaccm.s.fNestedPaging)
621 {
622 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_CR3);
623 VMXSetupCachedReadVMCS(pCache, VMX_VMCS_EXIT_PHYS_ADDR_FULL);
624 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
625 }
626 else
627 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
628 } /* for each VMCPU */
629
630 /* Choose the right TLB setup function. */
631 if (pVM->hwaccm.s.fNestedPaging)
632 {
633 pVM->hwaccm.s.vmx.pfnSetupTaggedTLB = hmR0VmxSetupTLBEPT;
634
635 /* Default values for flushing. */
636 pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_ALL_CONTEXTS;
637 pVM->hwaccm.s.vmx.enmFlushContext = VMX_FLUSH_ALL_CONTEXTS;
638
639 /* If the capabilities specify we can do more, then make use of it. */
640 if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVEPT_CAPS_INDIV)
641 pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_PAGE;
642 else
643 if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVEPT_CAPS_CONTEXT)
644 pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_SINGLE_CONTEXT;
645
646 if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVEPT_CAPS_CONTEXT)
647 pVM->hwaccm.s.vmx.enmFlushContext = VMX_FLUSH_SINGLE_CONTEXT;
648 }
649#ifdef HWACCM_VTX_WITH_VPID
650 else
651 if (pVM->hwaccm.s.vmx.fVPID)
652 {
653 pVM->hwaccm.s.vmx.pfnSetupTaggedTLB = hmR0VmxSetupTLBVPID;
654
655 /* Default values for flushing. */
656 pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_ALL_CONTEXTS;
657 pVM->hwaccm.s.vmx.enmFlushContext = VMX_FLUSH_ALL_CONTEXTS;
658
659 /* If the capabilities specify we can do more, then make use of it. */
660 if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_INDIV)
661 pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_PAGE;
662 else
663 if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_CONTEXT)
664 pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_SINGLE_CONTEXT;
665
666 if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_CONTEXT)
667 pVM->hwaccm.s.vmx.enmFlushContext = VMX_FLUSH_SINGLE_CONTEXT;
668 }
669#endif /* HWACCM_VTX_WITH_VPID */
670 else
671 pVM->hwaccm.s.vmx.pfnSetupTaggedTLB = hmR0VmxSetupTLBDummy;
672
673vmx_end:
674 hmR0VmxCheckError(pVM, &pVM->aCpus[0], rc);
675 return rc;
676}
677
678/**
679 * Sets the permission bits for the specified MSR
680 *
681 * @param pVCpu The VMCPU to operate on.
682 * @param ulMSR MSR value
683 * @param fRead Reading allowed/disallowed
684 * @param fWrite Writing allowed/disallowed
685 */
686static void hmR0VmxSetMSRPermission(PVMCPU pVCpu, unsigned ulMSR, bool fRead, bool fWrite)
687{
688 unsigned ulBit;
689 uint8_t *pMSRBitmap = (uint8_t *)pVCpu->hwaccm.s.vmx.pMSRBitmap;
690
691 /* Layout:
692 * 0x000 - 0x3ff - Low MSR read bits
693 * 0x400 - 0x7ff - High MSR read bits
694 * 0x800 - 0xbff - Low MSR write bits
695 * 0xc00 - 0xfff - High MSR write bits
696 */
697 if (ulMSR <= 0x00001FFF)
698 {
699 /* Pentium-compatible MSRs */
700 ulBit = ulMSR;
701 }
702 else
703 if ( ulMSR >= 0xC0000000
704 && ulMSR <= 0xC0001FFF)
705 {
706 /* AMD Sixth Generation x86 Processor MSRs */
707 ulBit = (ulMSR - 0xC0000000);
708 pMSRBitmap += 0x400;
709 }
710 else
711 {
712 AssertFailed();
713 return;
714 }
715
716 Assert(ulBit <= 0x1fff);
717 if (fRead)
718 ASMBitClear(pMSRBitmap, ulBit);
719 else
720 ASMBitSet(pMSRBitmap, ulBit);
721
722 if (fWrite)
723 ASMBitClear(pMSRBitmap + 0x800, ulBit);
724 else
725 ASMBitSet(pMSRBitmap + 0x800, ulBit);
726}
727
728
729/**
730 * Injects an event (trap or external interrupt)
731 *
732 * @returns VBox status code. Note that it may return VINF_EM_RESET to
733 * indicate a triple fault when injecting X86_XCPT_DF.
734 *
735 * @param pVM The VM to operate on.
736 * @param pVCpu The VMCPU to operate on.
737 * @param pCtx CPU Context
738 * @param intInfo VMX interrupt info
739 * @param cbInstr Opcode length of faulting instruction
740 * @param errCode Error code (optional)
741 */
742static int hmR0VmxInjectEvent(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, uint32_t intInfo, uint32_t cbInstr, uint32_t errCode)
743{
744 int rc;
745 uint32_t iGate = VMX_EXIT_INTERRUPTION_INFO_VECTOR(intInfo);
746
747#ifdef VBOX_WITH_STATISTICS
748 STAM_COUNTER_INC(&pVCpu->hwaccm.s.paStatInjectedIrqsR0[iGate & MASK_INJECT_IRQ_STAT]);
749#endif
750
751#ifdef VBOX_STRICT
752 if (iGate == 0xE)
753 LogFlow(("hmR0VmxInjectEvent: Injecting interrupt %d at %RGv error code=%08x CR2=%RGv intInfo=%08x\n", iGate, (RTGCPTR)pCtx->rip, errCode, pCtx->cr2, intInfo));
754 else
755 if (iGate < 0x20)
756 LogFlow(("hmR0VmxInjectEvent: Injecting interrupt %d at %RGv error code=%08x\n", iGate, (RTGCPTR)pCtx->rip, errCode));
757 else
758 {
759 LogFlow(("INJ-EI: %x at %RGv\n", iGate, (RTGCPTR)pCtx->rip));
760 Assert(VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW || !VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
761 Assert(VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW || pCtx->eflags.u32 & X86_EFL_IF);
762 }
763#endif
764
765 if ( CPUMIsGuestInRealModeEx(pCtx)
766 && pVM->hwaccm.s.vmx.pRealModeTSS)
767 {
768 RTGCPHYS GCPhysHandler;
769 uint16_t offset, ip;
770 RTSEL sel;
771
772 /* Injecting events doesn't work right with real mode emulation.
773 * (#GP if we try to inject external hardware interrupts)
774 * Inject the interrupt or trap directly instead.
775 *
776 * ASSUMES no access handlers for the bits we read or write below (should be safe).
777 */
778 Log(("Manual interrupt/trap '%x' inject (real mode)\n", iGate));
779
780 /* Check if the interrupt handler is present. */
781 if (iGate * 4 + 3 > pCtx->idtr.cbIdt)
782 {
783 Log(("IDT cbIdt violation\n"));
784 if (iGate != X86_XCPT_DF)
785 {
786 uint32_t intInfo2;
787
788 intInfo2 = (iGate == X86_XCPT_GP) ? (uint32_t)X86_XCPT_DF : iGate;
789 intInfo2 |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
790 intInfo2 |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
791 intInfo2 |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
792
793 return hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo2, 0, 0 /* no error code according to the Intel docs */);
794 }
795 Log(("Triple fault -> reset the VM!\n"));
796 return VINF_EM_RESET;
797 }
798 if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW
799 || iGate == 3 /* Both #BP and #OF point to the instruction after. */
800 || iGate == 4)
801 {
802 ip = pCtx->ip + cbInstr;
803 }
804 else
805 ip = pCtx->ip;
806
807 /* Read the selector:offset pair of the interrupt handler. */
808 GCPhysHandler = (RTGCPHYS)pCtx->idtr.pIdt + iGate * 4;
809 rc = PGMPhysSimpleReadGCPhys(pVM, &offset, GCPhysHandler, sizeof(offset)); AssertRC(rc);
810 rc = PGMPhysSimpleReadGCPhys(pVM, &sel, GCPhysHandler + 2, sizeof(sel)); AssertRC(rc);
811
812 LogFlow(("IDT handler %04X:%04X\n", sel, offset));
813
814 /* Construct the stack frame. */
815 /** @todo should check stack limit. */
816 pCtx->sp -= 2;
817 LogFlow(("ss:sp %04X:%04X eflags=%x\n", pCtx->ss, pCtx->sp, pCtx->eflags.u));
818 rc = PGMPhysSimpleWriteGCPhys(pVM, pCtx->ssHid.u64Base + pCtx->sp, &pCtx->eflags, sizeof(uint16_t)); AssertRC(rc);
819 pCtx->sp -= 2;
820 LogFlow(("ss:sp %04X:%04X cs=%x\n", pCtx->ss, pCtx->sp, pCtx->cs));
821 rc = PGMPhysSimpleWriteGCPhys(pVM, pCtx->ssHid.u64Base + pCtx->sp, &pCtx->cs, sizeof(uint16_t)); AssertRC(rc);
822 pCtx->sp -= 2;
823 LogFlow(("ss:sp %04X:%04X ip=%x\n", pCtx->ss, pCtx->sp, ip));
824 rc = PGMPhysSimpleWriteGCPhys(pVM, pCtx->ssHid.u64Base + pCtx->sp, &ip, sizeof(ip)); AssertRC(rc);
825
826 /* Update the CPU state for executing the handler. */
827 pCtx->rip = offset;
828 pCtx->cs = sel;
829 pCtx->csHid.u64Base = sel << 4;
830 pCtx->eflags.u &= ~(X86_EFL_IF|X86_EFL_TF|X86_EFL_RF|X86_EFL_AC);
831
832 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_SEGMENT_REGS;
833 return VINF_SUCCESS;
834 }
835
836 /* Set event injection state. */
837 rc = VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_IRQ_INFO, intInfo | (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT));
838
839 rc |= VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
840 rc |= VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_EXCEPTION_ERRCODE, errCode);
841
842 AssertRC(rc);
843 return rc;
844}
845
846
847/**
848 * Checks for pending guest interrupts and injects them
849 *
850 * @returns VBox status code.
851 * @param pVM The VM to operate on.
852 * @param pVCpu The VMCPU to operate on.
853 * @param pCtx CPU Context
854 */
855static int hmR0VmxCheckPendingInterrupt(PVM pVM, PVMCPU pVCpu, CPUMCTX *pCtx)
856{
857 int rc;
858
859 /* Dispatch any pending interrupts. (injected before, but a VM exit occurred prematurely) */
860 if (pVCpu->hwaccm.s.Event.fPending)
861 {
862 Log(("CPU%d: Reinjecting event %RX64 %08x at %RGv cr2=%RX64\n", pVCpu->idCpu, pVCpu->hwaccm.s.Event.intInfo, pVCpu->hwaccm.s.Event.errCode, (RTGCPTR)pCtx->rip, pCtx->cr2));
863 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatIntReinject);
864 rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, pVCpu->hwaccm.s.Event.intInfo, 0, pVCpu->hwaccm.s.Event.errCode);
865 AssertRC(rc);
866
867 pVCpu->hwaccm.s.Event.fPending = false;
868 return VINF_SUCCESS;
869 }
870
871 /* If an active trap is already pending, then we must forward it first! */
872 if (!TRPMHasTrap(pVCpu))
873 {
874 if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI))
875 {
876 RTGCUINTPTR intInfo;
877
878 Log(("CPU%d: injecting #NMI\n", pVCpu->idCpu));
879
880 intInfo = X86_XCPT_NMI;
881 intInfo |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
882 intInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
883
884 rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo, 0, 0);
885 AssertRC(rc);
886
887 return VINF_SUCCESS;
888 }
889
890 /* @todo SMI interrupts. */
891
892 /* When external interrupts are pending, we should exit the VM when IF is set. */
893 if (VMCPU_FF_ISPENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC|VMCPU_FF_INTERRUPT_PIC)))
894 {
895 if (!(pCtx->eflags.u32 & X86_EFL_IF))
896 {
897 if (!(pVCpu->hwaccm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_IRQ_WINDOW_EXIT))
898 {
899 LogFlow(("Enable irq window exit!\n"));
900 pVCpu->hwaccm.s.vmx.proc_ctls |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_IRQ_WINDOW_EXIT;
901 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
902 AssertRC(rc);
903 }
904 /* else nothing to do but wait */
905 }
906 else
907 if (!VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
908 {
909 uint8_t u8Interrupt;
910
911 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
912 Log(("CPU%d: Dispatch interrupt: u8Interrupt=%x (%d) rc=%Rrc cs:rip=%04X:%RGv\n", pVCpu->idCpu, u8Interrupt, u8Interrupt, rc, pCtx->cs, (RTGCPTR)pCtx->rip));
913 if (RT_SUCCESS(rc))
914 {
915 rc = TRPMAssertTrap(pVCpu, u8Interrupt, TRPM_HARDWARE_INT);
916 AssertRC(rc);
917 }
918 else
919 {
920 /* Can only happen in rare cases where a pending interrupt is cleared behind our back */
921 Assert(!VMCPU_FF_ISPENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC|VMCPU_FF_INTERRUPT_PIC)));
922 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatSwitchGuestIrq);
923 /* Just continue */
924 }
925 }
926 else
927 Log(("Pending interrupt blocked at %RGv by VM_FF_INHIBIT_INTERRUPTS!!\n", (RTGCPTR)pCtx->rip));
928 }
929 }
930
931#ifdef VBOX_STRICT
932 if (TRPMHasTrap(pVCpu))
933 {
934 uint8_t u8Vector;
935 rc = TRPMQueryTrapAll(pVCpu, &u8Vector, 0, 0, 0);
936 AssertRC(rc);
937 }
938#endif
939
940 if ( (pCtx->eflags.u32 & X86_EFL_IF)
941 && (!VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
942 && TRPMHasTrap(pVCpu)
943 )
944 {
945 uint8_t u8Vector;
946 TRPMEVENT enmType;
947 RTGCUINTPTR intInfo;
948 RTGCUINT errCode;
949
950 /* If a new event is pending, then dispatch it now. */
951 rc = TRPMQueryTrapAll(pVCpu, &u8Vector, &enmType, &errCode, 0);
952 AssertRC(rc);
953 Assert(pCtx->eflags.Bits.u1IF == 1 || enmType == TRPM_TRAP);
954 Assert(enmType != TRPM_SOFTWARE_INT);
955
956 /* Clear the pending trap. */
957 rc = TRPMResetTrap(pVCpu);
958 AssertRC(rc);
959
960 intInfo = u8Vector;
961 intInfo |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
962
963 if (enmType == TRPM_TRAP)
964 {
965 switch (u8Vector) {
966 case 8:
967 case 10:
968 case 11:
969 case 12:
970 case 13:
971 case 14:
972 case 17:
973 /* Valid error codes. */
974 intInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
975 break;
976 default:
977 break;
978 }
979 if (u8Vector == X86_XCPT_BP || u8Vector == X86_XCPT_OF)
980 intInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
981 else
982 intInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
983 }
984 else
985 intInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
986
987 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatIntInject);
988 rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo, 0, errCode);
989 AssertRC(rc);
990 } /* if (interrupts can be dispatched) */
991
992 return VINF_SUCCESS;
993}
994
995/**
996 * Save the host state
997 *
998 * @returns VBox status code.
999 * @param pVM The VM to operate on.
1000 * @param pVCpu The VMCPU to operate on.
1001 */
1002VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
1003{
1004 int rc = VINF_SUCCESS;
1005
1006 /*
1007 * Host CPU Context
1008 */
1009 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_HOST_CONTEXT)
1010 {
1011 RTIDTR idtr;
1012 RTGDTR gdtr;
1013 RTSEL SelTR;
1014 PCX86DESCHC pDesc;
1015 uintptr_t trBase;
1016 RTSEL cs;
1017 RTSEL ss;
1018 uint64_t cr3;
1019
1020 /* Control registers */
1021 rc = VMXWriteVMCS(VMX_VMCS_HOST_CR0, ASMGetCR0());
1022#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1023 if (VMX_IS_64BIT_HOST_MODE())
1024 {
1025 cr3 = hwaccmR0Get64bitCR3();
1026 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_CR3, cr3);
1027 }
1028 else
1029#endif
1030 {
1031 cr3 = ASMGetCR3();
1032 rc |= VMXWriteVMCS(VMX_VMCS_HOST_CR3, cr3);
1033 }
1034 rc |= VMXWriteVMCS(VMX_VMCS_HOST_CR4, ASMGetCR4());
1035 AssertRC(rc);
1036 Log2(("VMX_VMCS_HOST_CR0 %08x\n", ASMGetCR0()));
1037 Log2(("VMX_VMCS_HOST_CR3 %08RX64\n", cr3));
1038 Log2(("VMX_VMCS_HOST_CR4 %08x\n", ASMGetCR4()));
1039
1040 /* Selector registers. */
1041#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1042 if (VMX_IS_64BIT_HOST_MODE())
1043 {
1044 cs = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
1045 ss = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
1046 }
1047 else
1048 {
1049 /* sysenter loads LDT cs & ss, VMX doesn't like this. Load the GDT ones (safe). */
1050 cs = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
1051 ss = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
1052 }
1053#else
1054 cs = ASMGetCS();
1055 ss = ASMGetSS();
1056#endif
1057 Assert(!(cs & X86_SEL_LDT)); Assert((cs & X86_SEL_RPL) == 0);
1058 Assert(!(ss & X86_SEL_LDT)); Assert((ss & X86_SEL_RPL) == 0);
1059 rc = VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_CS, cs);
1060 /* Note: VMX is (again) very picky about the RPL of the selectors here; we'll restore them manually. */
1061 rc |= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_DS, 0);
1062 rc |= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_ES, 0);
1063#if HC_ARCH_BITS == 32
1064 if (!VMX_IS_64BIT_HOST_MODE())
1065 {
1066 rc |= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_FS, 0);
1067 rc |= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_GS, 0);
1068 }
1069#endif
1070 rc |= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_SS, ss);
1071 SelTR = ASMGetTR();
1072 rc |= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_TR, SelTR);
1073 AssertRC(rc);
1074 Log2(("VMX_VMCS_HOST_FIELD_CS %08x (%08x)\n", cs, ASMGetSS()));
1075 Log2(("VMX_VMCS_HOST_FIELD_DS 00000000 (%08x)\n", ASMGetDS()));
1076 Log2(("VMX_VMCS_HOST_FIELD_ES 00000000 (%08x)\n", ASMGetES()));
1077 Log2(("VMX_VMCS_HOST_FIELD_FS 00000000 (%08x)\n", ASMGetFS()));
1078 Log2(("VMX_VMCS_HOST_FIELD_GS 00000000 (%08x)\n", ASMGetGS()));
1079 Log2(("VMX_VMCS_HOST_FIELD_SS %08x (%08x)\n", ss, ASMGetSS()));
1080 Log2(("VMX_VMCS_HOST_FIELD_TR %08x\n", ASMGetTR()));
1081
1082 /* GDTR & IDTR */
1083#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1084 if (VMX_IS_64BIT_HOST_MODE())
1085 {
1086 X86XDTR64 gdtr64, idtr64;
1087 hwaccmR0Get64bitGDTRandIDTR(&gdtr64, &idtr64);
1088 rc = VMXWriteVMCS64(VMX_VMCS_HOST_GDTR_BASE, gdtr64.uAddr);
1089 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_IDTR_BASE, gdtr64.uAddr);
1090 AssertRC(rc);
1091 Log2(("VMX_VMCS_HOST_GDTR_BASE %RX64\n", gdtr64.uAddr));
1092 Log2(("VMX_VMCS_HOST_IDTR_BASE %RX64\n", idtr64.uAddr));
1093 gdtr.cbGdt = gdtr64.cb;
1094 gdtr.pGdt = (uintptr_t)gdtr64.uAddr;
1095 }
1096 else
1097#endif
1098 {
1099 ASMGetGDTR(&gdtr);
1100 rc = VMXWriteVMCS(VMX_VMCS_HOST_GDTR_BASE, gdtr.pGdt);
1101 ASMGetIDTR(&idtr);
1102 rc |= VMXWriteVMCS(VMX_VMCS_HOST_IDTR_BASE, idtr.pIdt);
1103 AssertRC(rc);
1104 Log2(("VMX_VMCS_HOST_GDTR_BASE %RHv\n", gdtr.pGdt));
1105 Log2(("VMX_VMCS_HOST_IDTR_BASE %RHv\n", idtr.pIdt));
1106 }
1107
1108 /* Save the base address of the TR selector. */
1109 if (SelTR > gdtr.cbGdt)
1110 {
1111 AssertMsgFailed(("Invalid TR selector %x. GDTR.cbGdt=%x\n", SelTR, gdtr.cbGdt));
1112 return VERR_VMX_INVALID_HOST_STATE;
1113 }
1114
1115 pDesc = (PCX86DESCHC)(gdtr.pGdt + (SelTR & X86_SEL_MASK));
1116#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1117 if (VMX_IS_64BIT_HOST_MODE())
1118 {
1119 uint64_t trBase64 = X86DESC64_BASE(*(PX86DESC64)pDesc);
1120 rc = VMXWriteVMCS64(VMX_VMCS_HOST_TR_BASE, trBase64);
1121 Log2(("VMX_VMCS_HOST_TR_BASE %RX64\n", trBase64));
1122 AssertRC(rc);
1123 }
1124 else
1125#endif
1126 {
1127#if HC_ARCH_BITS == 64
1128 trBase = X86DESC64_BASE(*pDesc);
1129#else
1130 trBase = X86DESC_BASE(*pDesc);
1131#endif
1132 rc = VMXWriteVMCS(VMX_VMCS_HOST_TR_BASE, trBase);
1133 AssertRC(rc);
1134 Log2(("VMX_VMCS_HOST_TR_BASE %RHv\n", trBase));
1135 }
1136
1137 /* FS and GS base. */
1138#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1139 if (VMX_IS_64BIT_HOST_MODE())
1140 {
1141 Log2(("MSR_K8_FS_BASE = %RX64\n", ASMRdMsr(MSR_K8_FS_BASE)));
1142 Log2(("MSR_K8_GS_BASE = %RX64\n", ASMRdMsr(MSR_K8_GS_BASE)));
1143 rc = VMXWriteVMCS64(VMX_VMCS_HOST_FS_BASE, ASMRdMsr(MSR_K8_FS_BASE));
1144 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_GS_BASE, ASMRdMsr(MSR_K8_GS_BASE));
1145 }
1146#endif
1147 AssertRC(rc);
1148
1149 /* Sysenter MSRs. */
1150 /** @todo expensive!! */
1151 rc = VMXWriteVMCS(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
1152 Log2(("VMX_VMCS_HOST_SYSENTER_CS %08x\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_CS)));
1153#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1154 if (VMX_IS_64BIT_HOST_MODE())
1155 {
1156 Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_EIP)));
1157 Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_ESP)));
1158 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
1159 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
1160 }
1161 else
1162 {
1163 rc |= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
1164 rc |= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
1165 Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP)));
1166 Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP)));
1167 }
1168#elif HC_ARCH_BITS == 32
1169 rc |= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
1170 rc |= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
1171 Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP)));
1172 Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP)));
1173#else
1174 Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_EIP)));
1175 Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_ESP)));
1176 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
1177 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
1178#endif
1179 AssertRC(rc);
1180
1181#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
1182 /* Store all host MSRs in the VM-Exit load area, so they will be reloaded after the world switch back to the host. */
1183 PVMXMSR pMsr = (PVMXMSR)pVCpu->hwaccm.s.vmx.pHostMSR;
1184 unsigned idxMsr = 0;
1185
1186 /* EFER MSR present? */
1187 if (ASMCpuId_EDX(0x80000001) & (X86_CPUID_AMD_FEATURE_EDX_NX|X86_CPUID_AMD_FEATURE_EDX_LONG_MODE))
1188 {
1189 if (ASMCpuId_EDX(0x80000001) & X86_CPUID_AMD_FEATURE_EDX_SEP)
1190 {
1191 pMsr->u32IndexMSR = MSR_K6_STAR;
1192 pMsr->u32Reserved = 0;
1193 pMsr->u64Value = ASMRdMsr(MSR_K6_STAR); /* legacy syscall eip, cs & ss */
1194 pMsr++; idxMsr++;
1195 }
1196
1197 pMsr->u32IndexMSR = MSR_K6_EFER;
1198 pMsr->u32Reserved = 0;
1199# if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1200 if (CPUMIsGuestInLongMode(pVCpu))
1201 {
1202 /* Must match the efer value in our 64 bits switcher. */
1203 pMsr->u64Value = ASMRdMsr(MSR_K6_EFER) | MSR_K6_EFER_LME | MSR_K6_EFER_SCE | MSR_K6_EFER_NXE;
1204 }
1205 else
1206# endif
1207 pMsr->u64Value = ASMRdMsr(MSR_K6_EFER);
1208 pMsr++; idxMsr++;
1209 }
1210
1211# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1212 if (VMX_IS_64BIT_HOST_MODE())
1213 {
1214 pMsr->u32IndexMSR = MSR_K8_LSTAR;
1215 pMsr->u32Reserved = 0;
1216 pMsr->u64Value = ASMRdMsr(MSR_K8_LSTAR); /* 64 bits mode syscall rip */
1217 pMsr++; idxMsr++;
1218 pMsr->u32IndexMSR = MSR_K8_SF_MASK;
1219 pMsr->u32Reserved = 0;
1220 pMsr->u64Value = ASMRdMsr(MSR_K8_SF_MASK); /* syscall flag mask */
1221 pMsr++; idxMsr++;
1222 pMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
1223 pMsr->u32Reserved = 0;
1224 pMsr->u64Value = ASMRdMsr(MSR_K8_KERNEL_GS_BASE); /* swapgs exchange value */
1225 pMsr++; idxMsr++;
1226 }
1227# endif
1228 rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_MSR_LOAD_COUNT, idxMsr);
1229 AssertRC(rc);
1230#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
1231
1232 pVCpu->hwaccm.s.fContextUseFlags &= ~HWACCM_CHANGED_HOST_CONTEXT;
1233 }
1234 return rc;
1235}
1236
1237/**
1238 * Prefetch the 4 PDPT pointers (PAE and nested paging only)
1239 *
1240 * @returns VINF_SUCCESS or fatal error.
1241 * @param pVM The VM to operate on.
1242 * @param pVCpu The VMCPU to operate on.
1243 * @param pCtx Guest context
1244 */
1245static int hmR0VmxPrefetchPAEPdptrs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1246{
1247 if (CPUMIsGuestInPAEModeEx(pCtx))
1248 {
1249 for (unsigned i=0;i<4;i++)
1250 {
1251 X86PDPE Pdpe;
1252 int rc = PGMGstQueryPaePDPtr(pVCpu, i, &Pdpe);
1253 AssertRCReturn(rc, rc);
1254
1255 rc = VMXWriteVMCS64(VMX_VMCS_GUEST_PDPTR0_FULL + i*2, Pdpe.u);
1256 AssertRC(rc);
1257 }
1258 }
1259 return VINF_SUCCESS;
1260}
1261
1262/**
1263 * Update the exception bitmap according to the current CPU state
1264 *
1265 * @param pVM The VM to operate on.
1266 * @param pVCpu The VMCPU to operate on.
1267 * @param pCtx Guest context
1268 */
1269static void hmR0VmxUpdateExceptionBitmap(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1270{
1271 uint32_t u32TrapMask;
1272 Assert(pCtx);
1273
1274 u32TrapMask = HWACCM_VMX_TRAP_MASK;
1275#ifndef DEBUG
1276 if (pVM->hwaccm.s.fNestedPaging)
1277 u32TrapMask &= ~RT_BIT(X86_XCPT_PF); /* no longer need to intercept #PF. */
1278#endif
1279
1280 /* Also catch floating point exceptions as we need to report them to the guest in a different way. */
1281 if ( CPUMIsGuestFPUStateActive(pVCpu) == true
1282 && !(pCtx->cr0 & X86_CR0_NE)
1283 && !pVCpu->hwaccm.s.fFPUOldStyleOverride)
1284 {
1285 u32TrapMask |= RT_BIT(X86_XCPT_MF);
1286 pVCpu->hwaccm.s.fFPUOldStyleOverride = true;
1287 }
1288
1289#ifdef VBOX_STRICT
1290 Assert(u32TrapMask & RT_BIT(X86_XCPT_GP));
1291#endif
1292
1293 /* Intercept all exceptions in real mode as none of them can be injected directly (#GP otherwise). */
1294 if ( CPUMIsGuestInRealModeEx(pCtx)
1295 && pVM->hwaccm.s.vmx.pRealModeTSS)
1296 u32TrapMask |= HWACCM_VMX_TRAP_MASK_REALMODE;
1297
1298 int rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXCEPTION_BITMAP, u32TrapMask);
1299 AssertRC(rc);
1300}
1301
1302/**
1303 * Loads a minimal guest state
1304 *
1305 * NOTE: Don't do anything here that can cause a jump back to ring 3!!!!!
1306 *
1307 * @param pVM The VM to operate on.
1308 * @param pVCpu The VMCPU to operate on.
1309 * @param pCtx Guest context
1310 */
1311VMMR0DECL(void) VMXR0LoadMinimalGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1312{
1313 int rc;
1314 X86EFLAGS eflags;
1315
1316 Assert(!(pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_ALL_GUEST));
1317
1318 /* EIP, ESP and EFLAGS */
1319 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_RIP, pCtx->rip);
1320 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_RSP, pCtx->rsp);
1321 AssertRC(rc);
1322
1323 /* Bits 22-31, 15, 5 & 3 must be zero. Bit 1 must be 1. */
1324 eflags = pCtx->eflags;
1325 eflags.u32 &= VMX_EFLAGS_RESERVED_0;
1326 eflags.u32 |= VMX_EFLAGS_RESERVED_1;
1327
1328 /* Real mode emulation using v86 mode. */
1329 if ( CPUMIsGuestInRealModeEx(pCtx)
1330 && pVM->hwaccm.s.vmx.pRealModeTSS)
1331 {
1332 pVCpu->hwaccm.s.vmx.RealMode.eflags = eflags;
1333
1334 eflags.Bits.u1VM = 1;
1335 eflags.Bits.u2IOPL = 0; /* must always be 0 or else certain instructions won't cause faults. */
1336 }
1337 rc = VMXWriteVMCS(VMX_VMCS_GUEST_RFLAGS, eflags.u32);
1338 AssertRC(rc);
1339}
1340
1341/**
1342 * Loads the guest state
1343 *
1344 * NOTE: Don't do anything here that can cause a jump back to ring 3!!!!!
1345 *
1346 * @returns VBox status code.
1347 * @param pVM The VM to operate on.
1348 * @param pVCpu The VMCPU to operate on.
1349 * @param pCtx Guest context
1350 */
1351VMMR0DECL(int) VMXR0LoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1352{
1353 int rc = VINF_SUCCESS;
1354 RTGCUINTPTR val;
1355
1356 /* VMX_VMCS_CTRL_ENTRY_CONTROLS
1357 * Set required bits to one and zero according to the MSR capabilities.
1358 */
1359 val = pVM->hwaccm.s.vmx.msr.vmx_entry.n.disallowed0;
1360 /* Load guest debug controls (dr7 & IA32_DEBUGCTL_MSR) (forced to 1 on the 'first' VT-x capable CPUs; this actually includes the newest Nehalem CPUs) */
1361 val |= VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_DEBUG;
1362 /* 64 bits guest mode? */
1363 if (CPUMIsGuestInLongModeEx(pCtx))
1364 val |= VMX_VMCS_CTRL_ENTRY_CONTROLS_IA64_MODE;
1365 /* else Must be zero when AMD64 is not available. */
1366
1367 /* Mask away the bits that the CPU doesn't support */
1368 val &= pVM->hwaccm.s.vmx.msr.vmx_entry.n.allowed1;
1369 rc = VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_CONTROLS, val);
1370 AssertRC(rc);
1371
1372 /* VMX_VMCS_CTRL_EXIT_CONTROLS
1373 * Set required bits to one and zero according to the MSR capabilities.
1374 */
1375 val = pVM->hwaccm.s.vmx.msr.vmx_exit.n.disallowed0;
1376
1377 /* Save debug controls (dr7 & IA32_DEBUGCTL_MSR) (forced to 1 on the 'first' VT-x capable CPUs; this actually includes the newest Nehalem CPUs) */
1378 val |= VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_DEBUG;
1379
1380#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1381 if (VMX_IS_64BIT_HOST_MODE())
1382 val |= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_AMD64;
1383 /* else: Must be zero when AMD64 is not available. */
1384#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
1385 if (CPUMIsGuestInLongModeEx(pCtx))
1386 val |= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_AMD64; /* our switcher goes to long mode */
1387 else
1388 Assert(!(val & VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_AMD64));
1389#endif
1390 val &= pVM->hwaccm.s.vmx.msr.vmx_exit.n.allowed1;
1391 /* Don't acknowledge external interrupts on VM-exit. */
1392 rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_CONTROLS, val);
1393 AssertRC(rc);
1394
1395 /* Guest CPU context: ES, CS, SS, DS, FS, GS. */
1396 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_SEGMENT_REGS)
1397 {
1398 if (pVM->hwaccm.s.vmx.pRealModeTSS)
1399 {
1400 PGMMODE enmGuestMode = PGMGetGuestMode(pVCpu);
1401 if (pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode != enmGuestMode)
1402 {
1403 /* Correct weird requirements for switching to protected mode. */
1404 if ( pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode == PGMMODE_REAL
1405 && enmGuestMode >= PGMMODE_PROTECTED)
1406 {
1407 /* Flush the recompiler code cache as it's not unlikely
1408 * the guest will rewrite code it will later execute in real
1409 * mode (OpenBSD 4.0 is one such example)
1410 */
1411 REMFlushTBs(pVM);
1412
1413 /* DPL of all hidden selector registers must match the current CPL (0). */
1414 pCtx->csHid.Attr.n.u2Dpl = 0;
1415 pCtx->csHid.Attr.n.u4Type = X86_SEL_TYPE_CODE | X86_SEL_TYPE_RW_ACC;
1416
1417 pCtx->dsHid.Attr.n.u2Dpl = 0;
1418 pCtx->esHid.Attr.n.u2Dpl = 0;
1419 pCtx->fsHid.Attr.n.u2Dpl = 0;
1420 pCtx->gsHid.Attr.n.u2Dpl = 0;
1421 pCtx->ssHid.Attr.n.u2Dpl = 0;
1422
1423 /* The limit must correspond to the 32 bits setting. */
1424 if (!pCtx->csHid.Attr.n.u1DefBig)
1425 pCtx->csHid.u32Limit &= 0xffff;
1426 if (!pCtx->dsHid.Attr.n.u1DefBig)
1427 pCtx->dsHid.u32Limit &= 0xffff;
1428 if (!pCtx->esHid.Attr.n.u1DefBig)
1429 pCtx->esHid.u32Limit &= 0xffff;
1430 if (!pCtx->fsHid.Attr.n.u1DefBig)
1431 pCtx->fsHid.u32Limit &= 0xffff;
1432 if (!pCtx->gsHid.Attr.n.u1DefBig)
1433 pCtx->gsHid.u32Limit &= 0xffff;
1434 if (!pCtx->ssHid.Attr.n.u1DefBig)
1435 pCtx->ssHid.u32Limit &= 0xffff;
1436 }
1437 else
1438 /* Switching from protected mode to real mode. */
1439 if ( pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode >= PGMMODE_PROTECTED
1440 && enmGuestMode == PGMMODE_REAL)
1441 {
1442 /* The limit must also be set to 0xffff. */
1443 pCtx->csHid.u32Limit = 0xffff;
1444 pCtx->dsHid.u32Limit = 0xffff;
1445 pCtx->esHid.u32Limit = 0xffff;
1446 pCtx->fsHid.u32Limit = 0xffff;
1447 pCtx->gsHid.u32Limit = 0xffff;
1448 pCtx->ssHid.u32Limit = 0xffff;
1449
1450 Assert(pCtx->csHid.u64Base <= 0xfffff);
1451 Assert(pCtx->dsHid.u64Base <= 0xfffff);
1452 Assert(pCtx->esHid.u64Base <= 0xfffff);
1453 Assert(pCtx->fsHid.u64Base <= 0xfffff);
1454 Assert(pCtx->gsHid.u64Base <= 0xfffff);
1455 }
1456 pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode = enmGuestMode;
1457 }
1458 else
1459 /* VT-x will fail with a guest invalid state otherwise... (CPU state after a reset) */
1460 if ( CPUMIsGuestInRealModeEx(pCtx)
1461 && pCtx->csHid.u64Base == 0xffff0000)
1462 {
1463 pCtx->csHid.u64Base = 0xf0000;
1464 pCtx->cs = 0xf000;
1465 }
1466 }
1467
1468 VMX_WRITE_SELREG(ES, es);
1469 AssertRC(rc);
1470
1471 VMX_WRITE_SELREG(CS, cs);
1472 AssertRC(rc);
1473
1474 VMX_WRITE_SELREG(SS, ss);
1475 AssertRC(rc);
1476
1477 VMX_WRITE_SELREG(DS, ds);
1478 AssertRC(rc);
1479
1480 VMX_WRITE_SELREG(FS, fs);
1481 AssertRC(rc);
1482
1483 VMX_WRITE_SELREG(GS, gs);
1484 AssertRC(rc);
1485 }
1486
1487 /* Guest CPU context: LDTR. */
1488 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_LDTR)
1489 {
1490 if (pCtx->ldtr == 0)
1491 {
1492 rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_LDTR, 0);
1493 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_LIMIT, 0);
1494 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_LDTR_BASE, 0);
1495 /* Note: vmlaunch will fail with 0 or just 0x02. No idea why. */
1496 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, 0x82 /* present, LDT */);
1497 }
1498 else
1499 {
1500 rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_LDTR, pCtx->ldtr);
1501 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtrHid.u32Limit);
1502 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_LDTR_BASE, pCtx->ldtrHid.u64Base);
1503 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, pCtx->ldtrHid.Attr.u);
1504 }
1505 AssertRC(rc);
1506 }
1507 /* Guest CPU context: TR. */
1508 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_TR)
1509 {
1510 /* Real mode emulation using v86 mode with CR4.VME (interrupt redirection using the int bitmap in the TSS) */
1511 if ( CPUMIsGuestInRealModeEx(pCtx)
1512 && pVM->hwaccm.s.vmx.pRealModeTSS)
1513 {
1514 RTGCPHYS GCPhys;
1515
1516 /* We convert it here every time as pci regions could be reconfigured. */
1517 rc = PDMVMMDevHeapR3ToGCPhys(pVM, pVM->hwaccm.s.vmx.pRealModeTSS, &GCPhys);
1518 AssertRC(rc);
1519
1520 rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_TR, 0);
1521 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_TR_LIMIT, HWACCM_VTX_TSS_SIZE);
1522 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_TR_BASE, GCPhys /* phys = virt in this mode */);
1523
1524 X86DESCATTR attr;
1525
1526 attr.u = 0;
1527 attr.n.u1Present = 1;
1528 attr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
1529 val = attr.u;
1530 }
1531 else
1532 {
1533 rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_TR, pCtx->tr);
1534 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_TR_LIMIT, pCtx->trHid.u32Limit);
1535 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_TR_BASE, pCtx->trHid.u64Base);
1536
1537 val = pCtx->trHid.Attr.u;
1538
1539 /* The TSS selector must be busy (REM bugs? see defect #XXXX). */
1540 if (!(val & X86_SEL_TYPE_SYS_TSS_BUSY_MASK))
1541 {
1542 if (val & 0xf)
1543 val |= X86_SEL_TYPE_SYS_TSS_BUSY_MASK;
1544 else
1545 /* Default if no TR selector has been set (otherwise vmlaunch will fail!) */
1546 val = (val & ~0xF) | X86_SEL_TYPE_SYS_386_TSS_BUSY;
1547 }
1548 AssertMsg((val & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY || (val & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("%#x\n", val));
1549 }
1550 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, val);
1551 AssertRC(rc);
1552 }
1553 /* Guest CPU context: GDTR. */
1554 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_GDTR)
1555 {
1556 rc = VMXWriteVMCS(VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt);
1557 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_GDTR_BASE, pCtx->gdtr.pGdt);
1558 AssertRC(rc);
1559 }
1560 /* Guest CPU context: IDTR. */
1561 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_IDTR)
1562 {
1563 rc = VMXWriteVMCS(VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt);
1564 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_IDTR_BASE, pCtx->idtr.pIdt);
1565 AssertRC(rc);
1566 }
1567
1568 /*
1569 * Sysenter MSRs
1570 */
1571 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_MSR)
1572 {
1573 rc = VMXWriteVMCS(VMX_VMCS32_GUEST_SYSENTER_CS, pCtx->SysEnter.cs);
1574 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_SYSENTER_EIP, pCtx->SysEnter.eip);
1575 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_SYSENTER_ESP, pCtx->SysEnter.esp);
1576 AssertRC(rc);
1577 }
1578
1579 /* Control registers */
1580 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_CR0)
1581 {
1582 val = pCtx->cr0;
1583 rc = VMXWriteVMCS(VMX_VMCS_CTRL_CR0_READ_SHADOW, val);
1584 Log2(("Guest CR0-shadow %08x\n", val));
1585 if (CPUMIsGuestFPUStateActive(pVCpu) == false)
1586 {
1587 /* Always use #NM exceptions to load the FPU/XMM state on demand. */
1588 val |= X86_CR0_TS | X86_CR0_ET | X86_CR0_NE | X86_CR0_MP;
1589 }
1590 else
1591 {
1592 /** @todo check if we support the old style mess correctly. */
1593 if (!(val & X86_CR0_NE))
1594 Log(("Forcing X86_CR0_NE!!!\n"));
1595
1596 val |= X86_CR0_NE; /* always turn on the native mechanism to report FPU errors (old style uses interrupts) */
1597 }
1598 /* Note: protected mode & paging are always enabled; we use them for emulating real and protected mode without paging too. */
1599 if (!pVM->hwaccm.s.vmx.fUnrestrictedGuest)
1600 val |= X86_CR0_PE | X86_CR0_PG;
1601
1602 if (pVM->hwaccm.s.fNestedPaging)
1603 {
1604 if (CPUMIsGuestInPagedProtectedModeEx(pCtx))
1605 {
1606 /* Disable cr3 read/write monitoring as we don't need it for EPT. */
1607 pVCpu->hwaccm.s.vmx.proc_ctls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
1608 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT);
1609 }
1610 else
1611 {
1612 /* Reenable cr3 read/write monitoring as our identity mapped page table is active. */
1613 pVCpu->hwaccm.s.vmx.proc_ctls |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
1614 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
1615 }
1616 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1617 AssertRC(rc);
1618 }
1619 else
1620 {
1621 /* Note: We must also set this as we rely on protecting various pages for which supervisor writes must be caught. */
1622 val |= X86_CR0_WP;
1623 }
1624
1625 /* Always enable caching. */
1626 val &= ~(X86_CR0_CD|X86_CR0_NW);
1627
1628 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_CR0, val);
1629 Log2(("Guest CR0 %08x\n", val));
1630 /* CR0 flags owned by the host; if the guests attempts to change them, then
1631 * the VM will exit.
1632 */
1633 val = X86_CR0_PE /* Must monitor this bit (assumptions are made for real mode emulation) */
1634 | X86_CR0_WP /* Must monitor this bit (it must always be enabled). */
1635 | X86_CR0_PG /* Must monitor this bit (assumptions are made for real mode & protected mode without paging emulation) */
1636 | X86_CR0_CD /* Bit not restored during VM-exit! */
1637 | X86_CR0_NW /* Bit not restored during VM-exit! */
1638 | X86_CR0_NE;
1639
1640 /* When the guest's FPU state is active, then we no longer care about
1641 * the FPU related bits.
1642 */
1643 if (CPUMIsGuestFPUStateActive(pVCpu) == false)
1644 val |= X86_CR0_TS | X86_CR0_ET | X86_CR0_MP;
1645
1646 pVCpu->hwaccm.s.vmx.cr0_mask = val;
1647
1648 rc |= VMXWriteVMCS(VMX_VMCS_CTRL_CR0_MASK, val);
1649 Log2(("Guest CR0-mask %08x\n", val));
1650 AssertRC(rc);
1651 }
1652 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_CR4)
1653 {
1654 /* CR4 */
1655 rc = VMXWriteVMCS(VMX_VMCS_CTRL_CR4_READ_SHADOW, pCtx->cr4);
1656 Log2(("Guest CR4-shadow %08x\n", pCtx->cr4));
1657 /* Set the required bits in cr4 too (currently X86_CR4_VMXE). */
1658 val = pCtx->cr4 | (uint32_t)pVM->hwaccm.s.vmx.msr.vmx_cr4_fixed0;
1659
1660 if (!pVM->hwaccm.s.fNestedPaging)
1661 {
1662 switch(pVCpu->hwaccm.s.enmShadowMode)
1663 {
1664 case PGMMODE_REAL: /* Real mode -> emulated using v86 mode */
1665 case PGMMODE_PROTECTED: /* Protected mode, no paging -> emulated using identity mapping. */
1666 case PGMMODE_32_BIT: /* 32-bit paging. */
1667 val &= ~X86_CR4_PAE;
1668 break;
1669
1670 case PGMMODE_PAE: /* PAE paging. */
1671 case PGMMODE_PAE_NX: /* PAE paging with NX enabled. */
1672 /** Must use PAE paging as we could use physical memory > 4 GB */
1673 val |= X86_CR4_PAE;
1674 break;
1675
1676 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
1677 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
1678#ifdef VBOX_ENABLE_64_BITS_GUESTS
1679 break;
1680#else
1681 AssertFailed();
1682 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1683#endif
1684 default: /* shut up gcc */
1685 AssertFailed();
1686 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1687 }
1688 }
1689 else
1690 if ( !CPUMIsGuestInPagedProtectedModeEx(pCtx)
1691 && !pVM->hwaccm.s.vmx.fUnrestrictedGuest)
1692 {
1693 /* We use 4 MB pages in our identity mapping page table for real and protected mode without paging. */
1694 val |= X86_CR4_PSE;
1695 /* Our identity mapping is a 32 bits page directory. */
1696 val &= ~X86_CR4_PAE;
1697 }
1698
1699 /* Turn off VME if we're in emulated real mode. */
1700 if ( CPUMIsGuestInRealModeEx(pCtx)
1701 && pVM->hwaccm.s.vmx.pRealModeTSS)
1702 val &= ~X86_CR4_VME;
1703
1704 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_CR4, val);
1705 Log2(("Guest CR4 %08x\n", val));
1706 /* CR4 flags owned by the host; if the guests attempts to change them, then
1707 * the VM will exit.
1708 */
1709 val = 0
1710 | X86_CR4_VME
1711 | X86_CR4_PAE
1712 | X86_CR4_PGE
1713 | X86_CR4_PSE
1714 | X86_CR4_VMXE;
1715 pVCpu->hwaccm.s.vmx.cr4_mask = val;
1716
1717 rc |= VMXWriteVMCS(VMX_VMCS_CTRL_CR4_MASK, val);
1718 Log2(("Guest CR4-mask %08x\n", val));
1719 AssertRC(rc);
1720 }
1721
1722 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_CR3)
1723 {
1724 if (pVM->hwaccm.s.fNestedPaging)
1725 {
1726 Assert(PGMGetHyperCR3(pVCpu));
1727 pVCpu->hwaccm.s.vmx.GCPhysEPTP = PGMGetHyperCR3(pVCpu);
1728
1729 Assert(!(pVCpu->hwaccm.s.vmx.GCPhysEPTP & 0xfff));
1730 /** @todo Check the IA32_VMX_EPT_VPID_CAP MSR for other supported memory types. */
1731 pVCpu->hwaccm.s.vmx.GCPhysEPTP |= VMX_EPT_MEMTYPE_WB
1732 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
1733
1734 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_EPTP_FULL, pVCpu->hwaccm.s.vmx.GCPhysEPTP);
1735 AssertRC(rc);
1736
1737 if ( !CPUMIsGuestInPagedProtectedModeEx(pCtx)
1738 && !pVM->hwaccm.s.vmx.fUnrestrictedGuest)
1739 {
1740 RTGCPHYS GCPhys;
1741
1742 /* We convert it here every time as pci regions could be reconfigured. */
1743 rc = PDMVMMDevHeapR3ToGCPhys(pVM, pVM->hwaccm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
1744 AssertMsgRC(rc, ("pNonPagingModeEPTPageTable = %RGv\n", pVM->hwaccm.s.vmx.pNonPagingModeEPTPageTable));
1745
1746 /* We use our identity mapping page table here as we need to map guest virtual to guest physical addresses; EPT will
1747 * take care of the translation to host physical addresses.
1748 */
1749 val = GCPhys;
1750 }
1751 else
1752 {
1753 /* Save the real guest CR3 in VMX_VMCS_GUEST_CR3 */
1754 val = pCtx->cr3;
1755 /* Prefetch the four PDPT entries in PAE mode. */
1756 rc = hmR0VmxPrefetchPAEPdptrs(pVM, pVCpu, pCtx);
1757 AssertRCReturn(rc, rc);
1758 }
1759 }
1760 else
1761 {
1762 val = PGMGetHyperCR3(pVCpu);
1763 Assert(val || VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL));
1764 }
1765
1766 /* Save our shadow CR3 register. */
1767 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_CR3, val);
1768 AssertRC(rc);
1769 }
1770
1771 /* Debug registers. */
1772 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_DEBUG)
1773 {
1774 pCtx->dr[6] |= X86_DR6_INIT_VAL; /* set all reserved bits to 1. */
1775 pCtx->dr[6] &= ~RT_BIT(12); /* must be zero. */
1776
1777 pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
1778 pCtx->dr[7] &= ~(RT_BIT(11) | RT_BIT(12) | RT_BIT(14) | RT_BIT(15)); /* must be zero */
1779 pCtx->dr[7] |= 0x400; /* must be one */
1780
1781 /* Resync DR7 */
1782 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_DR7, pCtx->dr[7]);
1783 AssertRC(rc);
1784
1785#ifdef DEBUG
1786 /* Sync the hypervisor debug state now if any breakpoint is armed. */
1787 if ( CPUMGetHyperDR7(pVCpu) & (X86_DR7_ENABLED_MASK|X86_DR7_GD)
1788 && !CPUMIsHyperDebugStateActive(pVCpu)
1789 && !DBGFIsStepping(pVCpu))
1790 {
1791 /* Save the host and load the hypervisor debug state. */
1792 rc = CPUMR0LoadHyperDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
1793 AssertRC(rc);
1794
1795 /* DRx intercepts remain enabled. */
1796
1797 /* Override dr7 with the hypervisor value. */
1798 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_DR7, CPUMGetHyperDR7(pVCpu));
1799 AssertRC(rc);
1800 }
1801 else
1802#endif
1803 /* Sync the debug state now if any breakpoint is armed. */
1804 if ( (pCtx->dr[7] & (X86_DR7_ENABLED_MASK|X86_DR7_GD))
1805 && !CPUMIsGuestDebugStateActive(pVCpu)
1806 && !DBGFIsStepping(pVCpu))
1807 {
1808 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatDRxArmed);
1809
1810 /* Disable drx move intercepts. */
1811 pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
1812 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1813 AssertRC(rc);
1814
1815 /* Save the host and load the guest debug state. */
1816 rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
1817 AssertRC(rc);
1818 }
1819
1820 /* IA32_DEBUGCTL MSR. */
1821 rc = VMXWriteVMCS64(VMX_VMCS_GUEST_DEBUGCTL_FULL, 0);
1822 AssertRC(rc);
1823
1824 /** @todo do we really ever need this? */
1825 rc |= VMXWriteVMCS(VMX_VMCS_GUEST_DEBUG_EXCEPTIONS, 0);
1826 AssertRC(rc);
1827 }
1828
1829 /* 64 bits guest mode? */
1830 if (CPUMIsGuestInLongModeEx(pCtx))
1831 {
1832#if !defined(VBOX_ENABLE_64_BITS_GUESTS)
1833 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1834#elif HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1835 pVCpu->hwaccm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
1836#else
1837# ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1838 if (!pVM->hwaccm.s.fAllow64BitGuests)
1839 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1840# endif
1841 pVCpu->hwaccm.s.vmx.pfnStartVM = VMXR0StartVM64;
1842#endif
1843 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_MSR)
1844 {
1845 /* Update these as wrmsr might have changed them. */
1846 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_FS_BASE, pCtx->fsHid.u64Base);
1847 AssertRC(rc);
1848 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_GS_BASE, pCtx->gsHid.u64Base);
1849 AssertRC(rc);
1850 }
1851 }
1852 else
1853 {
1854 pVCpu->hwaccm.s.vmx.pfnStartVM = VMXR0StartVM32;
1855 }
1856
1857 hmR0VmxUpdateExceptionBitmap(pVM, pVCpu, pCtx);
1858
1859#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
1860 /* Store all guest MSRs in the VM-Entry load area, so they will be loaded during the world switch. */
1861 PVMXMSR pMsr = (PVMXMSR)pVCpu->hwaccm.s.vmx.pGuestMSR;
1862 unsigned idxMsr = 0;
1863
1864 uint32_t ulEdx;
1865 uint32_t ulTemp;
1866 CPUMGetGuestCpuId(pVCpu, 0x80000001, &ulTemp, &ulTemp, &ulTemp, &ulEdx);
1867 /* EFER MSR present? */
1868 if (ulEdx & (X86_CPUID_AMD_FEATURE_EDX_NX|X86_CPUID_AMD_FEATURE_EDX_LONG_MODE))
1869 {
1870 pMsr->u32IndexMSR = MSR_K6_EFER;
1871 pMsr->u32Reserved = 0;
1872 pMsr->u64Value = pCtx->msrEFER;
1873 /* VT-x will complain if only MSR_K6_EFER_LME is set. */
1874 if (!CPUMIsGuestInLongModeEx(pCtx))
1875 pMsr->u64Value &= ~(MSR_K6_EFER_LMA|MSR_K6_EFER_LME);
1876 pMsr++; idxMsr++;
1877
1878 if (ulEdx & X86_CPUID_AMD_FEATURE_EDX_LONG_MODE)
1879 {
1880 pMsr->u32IndexMSR = MSR_K8_LSTAR;
1881 pMsr->u32Reserved = 0;
1882 pMsr->u64Value = pCtx->msrLSTAR; /* 64 bits mode syscall rip */
1883 pMsr++; idxMsr++;
1884 pMsr->u32IndexMSR = MSR_K6_STAR;
1885 pMsr->u32Reserved = 0;
1886 pMsr->u64Value = pCtx->msrSTAR; /* legacy syscall eip, cs & ss */
1887 pMsr++; idxMsr++;
1888 pMsr->u32IndexMSR = MSR_K8_SF_MASK;
1889 pMsr->u32Reserved = 0;
1890 pMsr->u64Value = pCtx->msrSFMASK; /* syscall flag mask */
1891 pMsr++; idxMsr++;
1892 pMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
1893 pMsr->u32Reserved = 0;
1894 pMsr->u64Value = pCtx->msrKERNELGSBASE; /* swapgs exchange value */
1895 pMsr++; idxMsr++;
1896 }
1897 }
1898 pVCpu->hwaccm.s.vmx.cCachedMSRs = idxMsr;
1899
1900 rc = VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_MSR_LOAD_COUNT, idxMsr);
1901 AssertRC(rc);
1902
1903 rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_MSR_STORE_COUNT, idxMsr);
1904 AssertRC(rc);
1905#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
1906
1907 bool fOffsettedTsc;
1908 if (pVM->hwaccm.s.vmx.fUsePreemptTimer)
1909 {
1910 uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &pVCpu->hwaccm.s.vmx.u64TSCOffset);
1911
1912 /* Make sure the returned values have sane upper and lower boundaries. */
1913 uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
1914
1915 cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64 of a second */
1916 cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
1917
1918 cTicksToDeadline >>= pVM->hwaccm.s.vmx.cPreemptTimerShift;
1919 uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
1920 rc = VMXWriteVMCS(VMX_VMCS32_GUEST_PREEMPTION_TIMER_VALUE, cPreemptionTickCount);
1921 AssertRC(rc);
1922 }
1923 else
1924 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hwaccm.s.vmx.u64TSCOffset);
1925 if (fOffsettedTsc)
1926 {
1927 uint64_t u64CurTSC = ASMReadTSC();
1928 if (u64CurTSC + pVCpu->hwaccm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
1929 {
1930 /* Note: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT takes precedence over TSC_OFFSET */
1931 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_TSC_OFFSET_FULL, pVCpu->hwaccm.s.vmx.u64TSCOffset);
1932 AssertRC(rc);
1933
1934 pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
1935 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1936 AssertRC(rc);
1937 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTSCOffset);
1938 }
1939 else
1940 {
1941 /* Fall back to rdtsc emulation as we would otherwise pass decreasing tsc values to the guest. */
1942 LogFlow(("TSC %RX64 offset %RX64 time=%RX64 last=%RX64 (diff=%RX64, virt_tsc=%RX64)\n", u64CurTSC, pVCpu->hwaccm.s.vmx.u64TSCOffset, u64CurTSC + pVCpu->hwaccm.s.vmx.u64TSCOffset, TMCpuTickGetLastSeen(pVCpu), TMCpuTickGetLastSeen(pVCpu) - u64CurTSC - pVCpu->hwaccm.s.vmx.u64TSCOffset, TMCpuTickGet(pVCpu)));
1943 pVCpu->hwaccm.s.vmx.proc_ctls |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
1944 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1945 AssertRC(rc);
1946 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTSCInterceptOverFlow);
1947 }
1948 }
1949 else
1950 {
1951 pVCpu->hwaccm.s.vmx.proc_ctls |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
1952 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1953 AssertRC(rc);
1954 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTSCIntercept);
1955 }
1956
1957 /* Done with the major changes */
1958 pVCpu->hwaccm.s.fContextUseFlags &= ~HWACCM_CHANGED_ALL_GUEST;
1959
1960 /* Minimal guest state update (esp, eip, eflags mostly) */
1961 VMXR0LoadMinimalGuestState(pVM, pVCpu, pCtx);
1962 return rc;
1963}
1964
1965/**
1966 * Syncs back the guest state
1967 *
1968 * @returns VBox status code.
1969 * @param pVM The VM to operate on.
1970 * @param pVCpu The VMCPU to operate on.
1971 * @param pCtx Guest context
1972 */
1973DECLINLINE(int) VMXR0SaveGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1974{
1975 RTGCUINTREG val, valShadow;
1976 RTGCUINTPTR uInterruptState;
1977 int rc;
1978
1979 /* Let's first sync back eip, esp, and eflags. */
1980 rc = VMXReadCachedVMCS(VMX_VMCS64_GUEST_RIP, &val);
1981 AssertRC(rc);
1982 pCtx->rip = val;
1983 rc = VMXReadCachedVMCS(VMX_VMCS64_GUEST_RSP, &val);
1984 AssertRC(rc);
1985 pCtx->rsp = val;
1986 rc = VMXReadCachedVMCS(VMX_VMCS_GUEST_RFLAGS, &val);
1987 AssertRC(rc);
1988 pCtx->eflags.u32 = val;
1989
1990 /* Take care of instruction fusing (sti, mov ss) */
1991 rc |= VMXReadCachedVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &val);
1992 uInterruptState = val;
1993 if (uInterruptState != 0)
1994 {
1995 Assert(uInterruptState <= 2); /* only sti & mov ss */
1996 Log(("uInterruptState %x eip=%RGv\n", (uint32_t)uInterruptState, pCtx->rip));
1997 EMSetInhibitInterruptsPC(pVCpu, pCtx->rip);
1998 }
1999 else
2000 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2001
2002 /* Control registers. */
2003 VMXReadCachedVMCS(VMX_VMCS_CTRL_CR0_READ_SHADOW, &valShadow);
2004 VMXReadCachedVMCS(VMX_VMCS64_GUEST_CR0, &val);
2005 val = (valShadow & pVCpu->hwaccm.s.vmx.cr0_mask) | (val & ~pVCpu->hwaccm.s.vmx.cr0_mask);
2006 CPUMSetGuestCR0(pVCpu, val);
2007
2008 VMXReadCachedVMCS(VMX_VMCS_CTRL_CR4_READ_SHADOW, &valShadow);
2009 VMXReadCachedVMCS(VMX_VMCS64_GUEST_CR4, &val);
2010 val = (valShadow & pVCpu->hwaccm.s.vmx.cr4_mask) | (val & ~pVCpu->hwaccm.s.vmx.cr4_mask);
2011 CPUMSetGuestCR4(pVCpu, val);
2012
2013 /* Note: no reason to sync back the CRx registers. They can't be changed by the guest. */
2014 /* Note: only in the nested paging case can CR3 & CR4 be changed by the guest. */
2015 if ( pVM->hwaccm.s.fNestedPaging
2016 && CPUMIsGuestInPagedProtectedModeEx(pCtx))
2017 {
2018 PVMCSCACHE pCache = &pVCpu->hwaccm.s.vmx.VMCSCache;
2019
2020 /* Can be updated behind our back in the nested paging case. */
2021 CPUMSetGuestCR2(pVCpu, pCache->cr2);
2022
2023 VMXReadCachedVMCS(VMX_VMCS64_GUEST_CR3, &val);
2024
2025 if (val != pCtx->cr3)
2026 {
2027 CPUMSetGuestCR3(pVCpu, val);
2028 PGMUpdateCR3(pVCpu, val);
2029 }
2030 /* Prefetch the four PDPT entries in PAE mode. */
2031 rc = hmR0VmxPrefetchPAEPdptrs(pVM, pVCpu, pCtx);
2032 AssertRCReturn(rc, rc);
2033 }
2034
2035 /* Sync back DR7 here. */
2036 VMXReadCachedVMCS(VMX_VMCS64_GUEST_DR7, &val);
2037 pCtx->dr[7] = val;
2038
2039 /* Guest CPU context: ES, CS, SS, DS, FS, GS. */
2040 VMX_READ_SELREG(ES, es);
2041 VMX_READ_SELREG(SS, ss);
2042 VMX_READ_SELREG(CS, cs);
2043 VMX_READ_SELREG(DS, ds);
2044 VMX_READ_SELREG(FS, fs);
2045 VMX_READ_SELREG(GS, gs);
2046
2047 /*
2048 * System MSRs
2049 */
2050 VMXReadCachedVMCS(VMX_VMCS32_GUEST_SYSENTER_CS, &val);
2051 pCtx->SysEnter.cs = val;
2052 VMXReadCachedVMCS(VMX_VMCS64_GUEST_SYSENTER_EIP, &val);
2053 pCtx->SysEnter.eip = val;
2054 VMXReadCachedVMCS(VMX_VMCS64_GUEST_SYSENTER_ESP, &val);
2055 pCtx->SysEnter.esp = val;
2056
2057 /* Misc. registers; must sync everything otherwise we can get out of sync when jumping to ring 3. */
2058 VMX_READ_SELREG(LDTR, ldtr);
2059
2060 VMXReadCachedVMCS(VMX_VMCS32_GUEST_GDTR_LIMIT, &val);
2061 pCtx->gdtr.cbGdt = val;
2062 VMXReadCachedVMCS(VMX_VMCS64_GUEST_GDTR_BASE, &val);
2063 pCtx->gdtr.pGdt = val;
2064
2065 VMXReadCachedVMCS(VMX_VMCS32_GUEST_IDTR_LIMIT, &val);
2066 pCtx->idtr.cbIdt = val;
2067 VMXReadCachedVMCS(VMX_VMCS64_GUEST_IDTR_BASE, &val);
2068 pCtx->idtr.pIdt = val;
2069
2070 /* Real mode emulation using v86 mode. */
2071 if ( CPUMIsGuestInRealModeEx(pCtx)
2072 && pVM->hwaccm.s.vmx.pRealModeTSS)
2073 {
2074 /* Hide our emulation flags */
2075 pCtx->eflags.Bits.u1VM = 0;
2076
2077 /* Restore original IOPL setting as we always use 0. */
2078 pCtx->eflags.Bits.u2IOPL = pVCpu->hwaccm.s.vmx.RealMode.eflags.Bits.u2IOPL;
2079
2080 /* Force a TR resync every time in case we switch modes. */
2081 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_TR;
2082 }
2083 else
2084 {
2085 /* In real mode we have a fake TSS, so only sync it back when it's supposed to be valid. */
2086 VMX_READ_SELREG(TR, tr);
2087 }
2088
2089#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
2090 /* Save the possibly changed MSRs that we automatically restore and save during a world switch. */
2091 for (unsigned i = 0; i < pVCpu->hwaccm.s.vmx.cCachedMSRs; i++)
2092 {
2093 PVMXMSR pMsr = (PVMXMSR)pVCpu->hwaccm.s.vmx.pGuestMSR;
2094 pMsr += i;
2095
2096 switch (pMsr->u32IndexMSR)
2097 {
2098 case MSR_K8_LSTAR:
2099 pCtx->msrLSTAR = pMsr->u64Value;
2100 break;
2101 case MSR_K6_STAR:
2102 pCtx->msrSTAR = pMsr->u64Value;
2103 break;
2104 case MSR_K8_SF_MASK:
2105 pCtx->msrSFMASK = pMsr->u64Value;
2106 break;
2107 case MSR_K8_KERNEL_GS_BASE:
2108 pCtx->msrKERNELGSBASE = pMsr->u64Value;
2109 break;
2110 case MSR_K6_EFER:
2111 /* EFER can't be changed without causing a VM-exit. */
2112// Assert(pCtx->msrEFER == pMsr->u64Value);
2113 break;
2114 default:
2115 AssertFailed();
2116 return VERR_INTERNAL_ERROR;
2117 }
2118 }
2119#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
2120 return VINF_SUCCESS;
2121}
2122
2123/**
2124 * Dummy placeholder
2125 *
2126 * @param pVM The VM to operate on.
2127 * @param pVCpu The VMCPU to operate on.
2128 */
2129static void hmR0VmxSetupTLBDummy(PVM pVM, PVMCPU pVCpu)
2130{
2131 NOREF(pVM);
2132 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
2133 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2134 pVCpu->hwaccm.s.TlbShootdown.cPages = 0;
2135 return;
2136}
2137
2138/**
2139 * Setup the tagged TLB for EPT
2140 *
2141 * @returns VBox status code.
2142 * @param pVM The VM to operate on.
2143 * @param pVCpu The VMCPU to operate on.
2144 */
2145static void hmR0VmxSetupTLBEPT(PVM pVM, PVMCPU pVCpu)
2146{
2147 PHMGLOBLCPUINFO pCpu;
2148
2149 Assert(pVM->hwaccm.s.fNestedPaging);
2150 Assert(!pVM->hwaccm.s.vmx.fVPID);
2151
2152 /* Deal with tagged TLBs if VPID or EPT is supported. */
2153 pCpu = HWACCMR0GetCurrentCpu();
2154 /* Force a TLB flush for the first world switch if the current cpu differs from the one we ran on last. */
2155 /* Note that this can happen both for start and resume due to long jumps back to ring 3. */
2156 if ( pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu
2157 /* if the tlb flush count has changed, another VM has flushed the TLB of this cpu, so we can't use our current ASID anymore. */
2158 || pVCpu->hwaccm.s.cTLBFlushes != pCpu->cTLBFlushes)
2159 {
2160 /* Force a TLB flush on VM entry. */
2161 pVCpu->hwaccm.s.fForceTLBFlush = true;
2162 }
2163 /* Disabled because this has triggered every time I have suspended my
2164 * laptop with a VM running for the past three months or more. */
2165 // else
2166 // Assert(!pCpu->fFlushTLB);
2167
2168 /* Check for tlb shootdown flushes. */
2169 if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2170 pVCpu->hwaccm.s.fForceTLBFlush = true;
2171
2172 pVCpu->hwaccm.s.idLastCpu = pCpu->idCpu;
2173 pCpu->fFlushTLB = false;
2174
2175 if (pVCpu->hwaccm.s.fForceTLBFlush)
2176 {
2177 hmR0VmxFlushEPT(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushContext, 0);
2178 }
2179 else
2180 if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2181 {
2182 /* Deal with pending TLB shootdown actions which were queued when we were not executing code. */
2183 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTlbShootdown);
2184
2185 for (unsigned i=0;i<pVCpu->hwaccm.s.TlbShootdown.cPages;i++)
2186 {
2187 /* aTlbShootdownPages contains physical addresses in this case. */
2188 hmR0VmxFlushEPT(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushPage, pVCpu->hwaccm.s.TlbShootdown.aPages[i]);
2189 }
2190 }
2191 pVCpu->hwaccm.s.TlbShootdown.cPages= 0;
2192 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2193
2194#ifdef VBOX_WITH_STATISTICS
2195 if (pVCpu->hwaccm.s.fForceTLBFlush)
2196 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushTLBWorldSwitch);
2197 else
2198 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatNoFlushTLBWorldSwitch);
2199#endif
2200}
2201
2202#ifdef HWACCM_VTX_WITH_VPID
2203/**
2204 * Setup the tagged TLB for VPID
2205 *
2206 * @returns VBox status code.
2207 * @param pVM The VM to operate on.
2208 * @param pVCpu The VMCPU to operate on.
2209 */
2210static void hmR0VmxSetupTLBVPID(PVM pVM, PVMCPU pVCpu)
2211{
2212 PHMGLOBLCPUINFO pCpu;
2213
2214 Assert(pVM->hwaccm.s.vmx.fVPID);
2215 Assert(!pVM->hwaccm.s.fNestedPaging);
2216
2217 /* Deal with tagged TLBs if VPID or EPT is supported. */
2218 pCpu = HWACCMR0GetCurrentCpu();
2219 /* Force a TLB flush for the first world switch if the current cpu differs from the one we ran on last. */
2220 /* Note that this can happen both for start and resume due to long jumps back to ring 3. */
2221 if ( pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu
2222 /* if the tlb flush count has changed, another VM has flushed the TLB of this cpu, so we can't use our current ASID anymore. */
2223 || pVCpu->hwaccm.s.cTLBFlushes != pCpu->cTLBFlushes)
2224 {
2225 /* Force a TLB flush on VM entry. */
2226 pVCpu->hwaccm.s.fForceTLBFlush = true;
2227 }
2228 else
2229 Assert(!pCpu->fFlushTLB);
2230
2231 pVCpu->hwaccm.s.idLastCpu = pCpu->idCpu;
2232
2233 /* Check for tlb shootdown flushes. */
2234 if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2235 pVCpu->hwaccm.s.fForceTLBFlush = true;
2236
2237 /* Make sure we flush the TLB when required. Switch ASID to achieve the same thing, but without actually flushing the whole TLB (which is expensive). */
2238 if (pVCpu->hwaccm.s.fForceTLBFlush)
2239 {
2240 if ( ++pCpu->uCurrentASID >= pVM->hwaccm.s.uMaxASID
2241 || pCpu->fFlushTLB)
2242 {
2243 pCpu->fFlushTLB = false;
2244 pCpu->uCurrentASID = 1; /* start at 1; host uses 0 */
2245 pCpu->cTLBFlushes++;
2246 hmR0VmxFlushVPID(pVM, pVCpu, VMX_FLUSH_ALL_CONTEXTS, 0);
2247 }
2248 else
2249 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushASID);
2250
2251 pVCpu->hwaccm.s.fForceTLBFlush = false;
2252 pVCpu->hwaccm.s.cTLBFlushes = pCpu->cTLBFlushes;
2253 pVCpu->hwaccm.s.uCurrentASID = pCpu->uCurrentASID;
2254 }
2255 else
2256 {
2257 Assert(!pCpu->fFlushTLB);
2258 Assert(pVCpu->hwaccm.s.uCurrentASID && pCpu->uCurrentASID);
2259
2260 if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2261 {
2262 /* Deal with pending TLB shootdown actions which were queued when we were not executing code. */
2263 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTlbShootdown);
2264 for (unsigned i = 0; i < pVCpu->hwaccm.s.TlbShootdown.cPages; i++)
2265 hmR0VmxFlushVPID(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushPage, pVCpu->hwaccm.s.TlbShootdown.aPages[i]);
2266 }
2267 }
2268 pVCpu->hwaccm.s.TlbShootdown.cPages = 0;
2269 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2270
2271 AssertMsg(pVCpu->hwaccm.s.cTLBFlushes == pCpu->cTLBFlushes, ("Flush count mismatch for cpu %d (%x vs %x)\n", pCpu->idCpu, pVCpu->hwaccm.s.cTLBFlushes, pCpu->cTLBFlushes));
2272 AssertMsg(pCpu->uCurrentASID >= 1 && pCpu->uCurrentASID < pVM->hwaccm.s.uMaxASID, ("cpu%d uCurrentASID = %x\n", pCpu->idCpu, pCpu->uCurrentASID));
2273 AssertMsg(pVCpu->hwaccm.s.uCurrentASID >= 1 && pVCpu->hwaccm.s.uCurrentASID < pVM->hwaccm.s.uMaxASID, ("cpu%d VM uCurrentASID = %x\n", pCpu->idCpu, pVCpu->hwaccm.s.uCurrentASID));
2274
2275 int rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hwaccm.s.uCurrentASID);
2276 AssertRC(rc);
2277
2278 if (pVCpu->hwaccm.s.fForceTLBFlush)
2279 hmR0VmxFlushVPID(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushContext, 0);
2280
2281# ifdef VBOX_WITH_STATISTICS
2282 if (pVCpu->hwaccm.s.fForceTLBFlush)
2283 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushTLBWorldSwitch);
2284 else
2285 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatNoFlushTLBWorldSwitch);
2286# endif
2287}
2288#endif /* HWACCM_VTX_WITH_VPID */
2289
2290/**
2291 * Runs guest code in a VT-x VM.
2292 *
2293 * @returns VBox status code.
2294 * @param pVM The VM to operate on.
2295 * @param pVCpu The VMCPU to operate on.
2296 * @param pCtx Guest context
2297 */
2298VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2299{
2300 STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatEntry, x);
2301 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hwaccm.s.StatExit1);
2302 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hwaccm.s.StatExit2);
2303
2304 VBOXSTRICTRC rc = VINF_SUCCESS;
2305 int rc2;
2306 RTGCUINTREG val;
2307 RTGCUINTREG exitReason = (RTGCUINTREG)VMX_EXIT_INVALID;
2308 RTGCUINTREG instrError, cbInstr;
2309 RTGCUINTPTR exitQualification = 0;
2310 RTGCUINTPTR intInfo = 0; /* shut up buggy gcc 4 */
2311 RTGCUINTPTR errCode, instrInfo;
2312 bool fSetupTPRCaching = false;
2313 uint64_t u64OldLSTAR = 0;
2314 uint8_t u8LastTPR = 0;
2315 RTCCUINTREG uOldEFlags = ~(RTCCUINTREG)0;
2316 unsigned cResume = 0;
2317#ifdef VBOX_STRICT
2318 RTCPUID idCpuCheck;
2319 bool fWasInLongMode = false;
2320#endif
2321#ifdef VBOX_HIGH_RES_TIMERS_HACK_IN_RING0
2322 uint64_t u64LastTime = RTTimeMilliTS();
2323#endif
2324
2325 Assert(!(pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC) || (pVCpu->hwaccm.s.vmx.pbVAPIC && pVM->hwaccm.s.vmx.pAPIC));
2326
2327 /* Check if we need to use TPR shadowing. */
2328 if ( CPUMIsGuestInLongModeEx(pCtx)
2329 || ( ((pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC) || pVM->hwaccm.s.fTRPPatchingAllowed)
2330 && pVM->hwaccm.s.fHasIoApic)
2331 )
2332 {
2333 fSetupTPRCaching = true;
2334 }
2335
2336 Log2(("\nE"));
2337
2338#ifdef VBOX_STRICT
2339 {
2340 RTCCUINTREG val2;
2341
2342 rc2 = VMXReadVMCS(VMX_VMCS_CTRL_PIN_EXEC_CONTROLS, &val2);
2343 AssertRC(rc2);
2344 Log2(("VMX_VMCS_CTRL_PIN_EXEC_CONTROLS = %08x\n", val2));
2345
2346 /* allowed zero */
2347 if ((val2 & pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.disallowed0)
2348 Log(("Invalid VMX_VMCS_CTRL_PIN_EXEC_CONTROLS: zero\n"));
2349
2350 /* allowed one */
2351 if ((val2 & ~pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.allowed1) != 0)
2352 Log(("Invalid VMX_VMCS_CTRL_PIN_EXEC_CONTROLS: one\n"));
2353
2354 rc2 = VMXReadVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, &val2);
2355 AssertRC(rc2);
2356 Log2(("VMX_VMCS_CTRL_PROC_EXEC_CONTROLS = %08x\n", val2));
2357
2358 /* Must be set according to the MSR, but can be cleared in case of EPT. */
2359 if (pVM->hwaccm.s.fNestedPaging)
2360 val2 |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT
2361 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
2362 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
2363
2364 /* allowed zero */
2365 if ((val2 & pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.disallowed0)
2366 Log(("Invalid VMX_VMCS_CTRL_PROC_EXEC_CONTROLS: zero\n"));
2367
2368 /* allowed one */
2369 if ((val2 & ~pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1) != 0)
2370 Log(("Invalid VMX_VMCS_CTRL_PROC_EXEC_CONTROLS: one\n"));
2371
2372 rc2 = VMXReadVMCS(VMX_VMCS_CTRL_ENTRY_CONTROLS, &val2);
2373 AssertRC(rc2);
2374 Log2(("VMX_VMCS_CTRL_ENTRY_CONTROLS = %08x\n", val2));
2375
2376 /* allowed zero */
2377 if ((val2 & pVM->hwaccm.s.vmx.msr.vmx_entry.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_entry.n.disallowed0)
2378 Log(("Invalid VMX_VMCS_CTRL_ENTRY_CONTROLS: zero\n"));
2379
2380 /* allowed one */
2381 if ((val2 & ~pVM->hwaccm.s.vmx.msr.vmx_entry.n.allowed1) != 0)
2382 Log(("Invalid VMX_VMCS_CTRL_ENTRY_CONTROLS: one\n"));
2383
2384 rc2 = VMXReadVMCS(VMX_VMCS_CTRL_EXIT_CONTROLS, &val2);
2385 AssertRC(rc2);
2386 Log2(("VMX_VMCS_CTRL_EXIT_CONTROLS = %08x\n", val2));
2387
2388 /* allowed zero */
2389 if ((val2 & pVM->hwaccm.s.vmx.msr.vmx_exit.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_exit.n.disallowed0)
2390 Log(("Invalid VMX_VMCS_CTRL_EXIT_CONTROLS: zero\n"));
2391
2392 /* allowed one */
2393 if ((val2 & ~pVM->hwaccm.s.vmx.msr.vmx_exit.n.allowed1) != 0)
2394 Log(("Invalid VMX_VMCS_CTRL_EXIT_CONTROLS: one\n"));
2395 }
2396 fWasInLongMode = CPUMIsGuestInLongModeEx(pCtx);
2397#endif /* VBOX_STRICT */
2398
2399#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2400 pVCpu->hwaccm.s.vmx.VMCSCache.u64TimeEntry = RTTimeNanoTS();
2401#endif
2402
2403 /* We can jump to this point to resume execution after determining that a VM-exit is innocent.
2404 */
2405ResumeExecution:
2406 if (!STAM_REL_PROFILE_ADV_IS_RUNNING(&pVCpu->hwaccm.s.StatEntry))
2407 STAM_REL_PROFILE_ADV_STOP_START(&pVCpu->hwaccm.s.StatExit2, &pVCpu->hwaccm.s.StatEntry, x);
2408 AssertMsg(pVCpu->hwaccm.s.idEnteredCpu == RTMpCpuId(),
2409 ("Expected %d, I'm %d; cResume=%d exitReason=%RGv exitQualification=%RGv\n",
2410 (int)pVCpu->hwaccm.s.idEnteredCpu, (int)RTMpCpuId(), cResume, exitReason, exitQualification));
2411 Assert(!HWACCMR0SuspendPending());
2412 /* Not allowed to switch modes without reloading the host state (32->64 switcher)!! */
2413 Assert(fWasInLongMode == CPUMIsGuestInLongModeEx(pCtx));
2414
2415 /* Safety precaution; looping for too long here can have a very bad effect on the host */
2416 if (RT_UNLIKELY(++cResume > pVM->hwaccm.s.cMaxResumeLoops))
2417 {
2418 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitMaxResume);
2419 rc = VINF_EM_RAW_INTERRUPT;
2420 goto end;
2421 }
2422
2423 /* Check for irq inhibition due to instruction fusing (sti, mov ss). */
2424 if (VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
2425 {
2426 Log(("VM_FF_INHIBIT_INTERRUPTS at %RGv successor %RGv\n", (RTGCPTR)pCtx->rip, EMGetInhibitInterruptsPC(pVCpu)));
2427 if (pCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
2428 {
2429 /* Note: we intentionally don't clear VM_FF_INHIBIT_INTERRUPTS here.
2430 * Before we are able to execute this instruction in raw mode (iret to guest code) an external interrupt might
2431 * force a world switch again. Possibly allowing a guest interrupt to be dispatched in the process. This could
2432 * break the guest. Sounds very unlikely, but such timing sensitive problems are not as rare as you might think.
2433 */
2434 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2435 /* Irq inhibition is no longer active; clear the corresponding VMX state. */
2436 rc2 = VMXWriteVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, 0);
2437 AssertRC(rc2);
2438 }
2439 }
2440 else
2441 {
2442 /* Irq inhibition is no longer active; clear the corresponding VMX state. */
2443 rc2 = VMXWriteVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, 0);
2444 AssertRC(rc2);
2445 }
2446
2447#ifdef VBOX_HIGH_RES_TIMERS_HACK_IN_RING0
2448 if (RT_UNLIKELY((cResume & 0xf) == 0))
2449 {
2450 uint64_t u64CurTime = RTTimeMilliTS();
2451
2452 if (RT_UNLIKELY(u64CurTime > u64LastTime))
2453 {
2454 u64LastTime = u64CurTime;
2455 TMTimerPollVoid(pVM, pVCpu);
2456 }
2457 }
2458#endif
2459
2460 /* Check for pending actions that force us to go back to ring 3. */
2461 if ( VM_FF_ISPENDING(pVM, VM_FF_HWACCM_TO_R3_MASK | VM_FF_REQUEST | VM_FF_PGM_POOL_FLUSH_PENDING | VM_FF_PDM_DMA)
2462 || VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_HWACCM_TO_R3_MASK | VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL | VMCPU_FF_REQUEST))
2463 {
2464 /* Check if a sync operation is pending. */
2465 if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
2466 {
2467 rc = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4, VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
2468 if (rc != VINF_SUCCESS)
2469 {
2470 AssertRC(VBOXSTRICTRC_VAL(rc));
2471 Log(("Pending pool sync is forcing us back to ring 3; rc=%d\n", VBOXSTRICTRC_VAL(rc)));
2472 goto end;
2473 }
2474 }
2475
2476#ifdef DEBUG
2477 /* Intercept X86_XCPT_DB if stepping is enabled */
2478 if (!DBGFIsStepping(pVCpu))
2479#endif
2480 {
2481 if ( VM_FF_ISPENDING(pVM, VM_FF_HWACCM_TO_R3_MASK)
2482 || VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_HWACCM_TO_R3_MASK))
2483 {
2484 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatSwitchToR3);
2485 rc = RT_UNLIKELY(VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
2486 goto end;
2487 }
2488 }
2489
2490 /* Pending request packets might contain actions that need immediate attention, such as pending hardware interrupts. */
2491 if ( VM_FF_ISPENDING(pVM, VM_FF_REQUEST)
2492 || VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_REQUEST))
2493 {
2494 rc = VINF_EM_PENDING_REQUEST;
2495 goto end;
2496 }
2497
2498 /* Check if a pgm pool flush is in progress. */
2499 if (VM_FF_ISPENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
2500 {
2501 rc = VINF_PGM_POOL_FLUSH_PENDING;
2502 goto end;
2503 }
2504
2505 /* Check if DMA work is pending (2nd+ run). */
2506 if (VM_FF_ISPENDING(pVM, VM_FF_PDM_DMA) && cResume > 1)
2507 {
2508 rc = VINF_EM_RAW_TO_R3;
2509 goto end;
2510 }
2511 }
2512
2513#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
2514 /*
2515 * Exit to ring-3 preemption/work is pending.
2516 *
2517 * Interrupts are disabled before the call to make sure we don't miss any interrupt
2518 * that would flag preemption (IPI, timer tick, ++). (Would've been nice to do this
2519 * further down, but hmR0VmxCheckPendingInterrupt makes that impossible.)
2520 *
2521 * Note! Interrupts must be disabled done *before* we check for TLB flushes; TLB
2522 * shootdowns rely on this.
2523 */
2524 uOldEFlags = ASMIntDisableFlags();
2525 if (RTThreadPreemptIsPending(NIL_RTTHREAD))
2526 {
2527 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitPreemptPending);
2528 rc = VINF_EM_RAW_INTERRUPT;
2529 goto end;
2530 }
2531 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
2532#endif
2533
2534 /* When external interrupts are pending, we should exit the VM when IF is set. */
2535 /* Note! *After* VM_FF_INHIBIT_INTERRUPTS check!!! */
2536 rc = hmR0VmxCheckPendingInterrupt(pVM, pVCpu, pCtx);
2537 if (RT_FAILURE(rc))
2538 goto end;
2539
2540 /** @todo check timers?? */
2541
2542 /* TPR caching using CR8 is only available in 64 bits mode */
2543 /* Note the 32 bits exception for AMD (X86_CPUID_AMD_FEATURE_ECX_CR8L), but that appears missing in Intel CPUs */
2544 /* Note: we can't do this in LoadGuestState as PDMApicGetTPR can jump back to ring 3 (lock)!!!!! (no longer true) */
2545 /**
2546 * @todo query and update the TPR only when it could have been changed (mmio access & wrmsr (x2apic))
2547 */
2548 if (fSetupTPRCaching)
2549 {
2550 /* TPR caching in CR8 */
2551 bool fPending;
2552
2553 rc2 = PDMApicGetTPR(pVCpu, &u8LastTPR, &fPending);
2554 AssertRC(rc2);
2555 /* The TPR can be found at offset 0x80 in the APIC mmio page. */
2556 pVCpu->hwaccm.s.vmx.pbVAPIC[0x80] = u8LastTPR;
2557
2558 /* Two options here:
2559 * - external interrupt pending, but masked by the TPR value.
2560 * -> a CR8 update that lower the current TPR value should cause an exit
2561 * - no pending interrupts
2562 * -> We don't need to be explicitely notified. There are enough world switches for detecting pending interrupts.
2563 */
2564 rc = VMXWriteVMCS(VMX_VMCS_CTRL_TPR_THRESHOLD, (fPending) ? (u8LastTPR >> 4) : 0); /* cr8 bits 3-0 correspond to bits 7-4 of the task priority mmio register. */
2565 AssertRC(VBOXSTRICTRC_VAL(rc));
2566
2567 if (pVM->hwaccm.s.fTPRPatchingActive)
2568 {
2569 Assert(!CPUMIsGuestInLongModeEx(pCtx));
2570 /* Our patch code uses LSTAR for TPR caching. */
2571 pCtx->msrLSTAR = u8LastTPR;
2572
2573 if (fPending)
2574 {
2575 /* A TPR change could activate a pending interrupt, so catch lstar writes. */
2576 hmR0VmxSetMSRPermission(pVCpu, MSR_K8_LSTAR, true, false);
2577 }
2578 else
2579 {
2580 /* No interrupts are pending, so we don't need to be explicitely notified.
2581 * There are enough world switches for detecting pending interrupts.
2582 */
2583 hmR0VmxSetMSRPermission(pVCpu, MSR_K8_LSTAR, true, true);
2584 }
2585 }
2586 }
2587
2588#if defined(HWACCM_VTX_WITH_EPT) && defined(LOG_ENABLED)
2589 if ( pVM->hwaccm.s.fNestedPaging
2590# ifdef HWACCM_VTX_WITH_VPID
2591 || pVM->hwaccm.s.vmx.fVPID
2592# endif /* HWACCM_VTX_WITH_VPID */
2593 )
2594 {
2595 PHMGLOBLCPUINFO pCpu;
2596
2597 pCpu = HWACCMR0GetCurrentCpu();
2598 if ( pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu
2599 || pVCpu->hwaccm.s.cTLBFlushes != pCpu->cTLBFlushes)
2600 {
2601 if (pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu)
2602 LogFlow(("Force TLB flush due to rescheduling to a different cpu (%d vs %d)\n", pVCpu->hwaccm.s.idLastCpu, pCpu->idCpu));
2603 else
2604 LogFlow(("Force TLB flush due to changed TLB flush count (%x vs %x)\n", pVCpu->hwaccm.s.cTLBFlushes, pCpu->cTLBFlushes));
2605 }
2606 if (pCpu->fFlushTLB)
2607 LogFlow(("Force TLB flush: first time cpu %d is used -> flush\n", pCpu->idCpu));
2608 else
2609 if (pVCpu->hwaccm.s.fForceTLBFlush)
2610 LogFlow(("Manual TLB flush\n"));
2611 }
2612#endif
2613#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
2614 PGMRZDynMapFlushAutoSet(pVCpu);
2615#endif
2616
2617 /*
2618 * NOTE: DO NOT DO ANYTHING AFTER THIS POINT THAT MIGHT JUMP BACK TO RING 3!
2619 * (until the actual world switch)
2620 */
2621#ifdef VBOX_STRICT
2622 idCpuCheck = RTMpCpuId();
2623#endif
2624#ifdef LOG_ENABLED
2625 VMMR0LogFlushDisable(pVCpu);
2626#endif
2627 /* Save the host state first. */
2628 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_HOST_CONTEXT)
2629 {
2630 rc = VMXR0SaveHostState(pVM, pVCpu);
2631 if (RT_UNLIKELY(rc != VINF_SUCCESS))
2632 {
2633 VMMR0LogFlushEnable(pVCpu);
2634 goto end;
2635 }
2636 }
2637
2638 /* Load the guest state */
2639 if (!pVCpu->hwaccm.s.fContextUseFlags)
2640 {
2641 VMXR0LoadMinimalGuestState(pVM, pVCpu, pCtx);
2642 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatLoadMinimal);
2643 }
2644 else
2645 {
2646 rc = VMXR0LoadGuestState(pVM, pVCpu, pCtx);
2647 if (RT_UNLIKELY(rc != VINF_SUCCESS))
2648 {
2649 VMMR0LogFlushEnable(pVCpu);
2650 goto end;
2651 }
2652 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatLoadFull);
2653 }
2654
2655#ifndef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
2656 /* Disable interrupts to make sure a poke will interrupt execution.
2657 * This must be done *before* we check for TLB flushes; TLB shootdowns rely on this.
2658 */
2659 uOldEFlags = ASMIntDisableFlags();
2660 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
2661#endif
2662
2663 /* Non-register state Guest Context */
2664 /** @todo change me according to cpu state */
2665 rc2 = VMXWriteVMCS(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_CMS_GUEST_ACTIVITY_ACTIVE);
2666 AssertRC(rc2);
2667
2668 /* Set TLB flush state as checked until we return from the world switch. */
2669 ASMAtomicWriteBool(&pVCpu->hwaccm.s.fCheckedTLBFlush, true);
2670 /* Deal with tagged TLB setup and invalidation. */
2671 pVM->hwaccm.s.vmx.pfnSetupTaggedTLB(pVM, pVCpu);
2672
2673 /* Manual save and restore:
2674 * - General purpose registers except RIP, RSP
2675 *
2676 * Trashed:
2677 * - CR2 (we don't care)
2678 * - LDTR (reset to 0)
2679 * - DRx (presumably not changed at all)
2680 * - DR7 (reset to 0x400)
2681 * - EFLAGS (reset to RT_BIT(1); not relevant)
2682 *
2683 */
2684
2685 /* All done! Let's start VM execution. */
2686 STAM_PROFILE_ADV_STOP_START(&pVCpu->hwaccm.s.StatEntry, &pVCpu->hwaccm.s.StatInGC, x);
2687 Assert(idCpuCheck == RTMpCpuId());
2688
2689#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2690 pVCpu->hwaccm.s.vmx.VMCSCache.cResume = cResume;
2691 pVCpu->hwaccm.s.vmx.VMCSCache.u64TimeSwitch = RTTimeNanoTS();
2692#endif
2693
2694 /* Save the current TPR value in the LSTAR msr so our patches can access it. */
2695 if (pVM->hwaccm.s.fTPRPatchingActive)
2696 {
2697 Assert(pVM->hwaccm.s.fTPRPatchingActive);
2698 u64OldLSTAR = ASMRdMsr(MSR_K8_LSTAR);
2699 ASMWrMsr(MSR_K8_LSTAR, u8LastTPR);
2700 }
2701
2702 TMNotifyStartOfExecution(pVCpu);
2703#ifdef VBOX_WITH_KERNEL_USING_XMM
2704 rc = hwaccmR0VMXStartVMWrapXMM(pVCpu->hwaccm.s.fResumeVM, pCtx, &pVCpu->hwaccm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hwaccm.s.vmx.pfnStartVM);
2705#else
2706 rc = pVCpu->hwaccm.s.vmx.pfnStartVM(pVCpu->hwaccm.s.fResumeVM, pCtx, &pVCpu->hwaccm.s.vmx.VMCSCache, pVM, pVCpu);
2707#endif
2708 ASMAtomicWriteBool(&pVCpu->hwaccm.s.fCheckedTLBFlush, false);
2709 ASMAtomicIncU32(&pVCpu->hwaccm.s.cWorldSwitchExits);
2710 /* Possibly the last TSC value seen by the guest (too high) (only when we're in tsc offset mode). */
2711 if (!(pVCpu->hwaccm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT))
2712 TMCpuTickSetLastSeen(pVCpu, ASMReadTSC() + pVCpu->hwaccm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
2713
2714 TMNotifyEndOfExecution(pVCpu);
2715 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED);
2716 Assert(!(ASMGetFlags() & X86_EFL_IF));
2717
2718 /* Restore the host LSTAR msr if the guest could have changed it. */
2719 if (pVM->hwaccm.s.fTPRPatchingActive)
2720 {
2721 Assert(pVM->hwaccm.s.fTPRPatchingActive);
2722 pVCpu->hwaccm.s.vmx.pbVAPIC[0x80] = pCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
2723 ASMWrMsr(MSR_K8_LSTAR, u64OldLSTAR);
2724 }
2725
2726 STAM_PROFILE_ADV_STOP_START(&pVCpu->hwaccm.s.StatInGC, &pVCpu->hwaccm.s.StatExit1, x);
2727 ASMSetFlags(uOldEFlags);
2728#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
2729 uOldEFlags = ~(RTCCUINTREG)0;
2730#endif
2731
2732 AssertMsg(!pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries, ("pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries=%d\n", pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries));
2733
2734 /* In case we execute a goto ResumeExecution later on. */
2735 pVCpu->hwaccm.s.fResumeVM = true;
2736 pVCpu->hwaccm.s.fForceTLBFlush = false;
2737
2738 /*
2739 * !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
2740 * IMPORTANT: WE CAN'T DO ANY LOGGING OR OPERATIONS THAT CAN DO A LONGJMP BACK TO RING 3 *BEFORE* WE'VE SYNCED BACK (MOST OF) THE GUEST STATE
2741 * !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
2742 */
2743
2744 if (RT_UNLIKELY(rc != VINF_SUCCESS))
2745 {
2746 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx);
2747 VMMR0LogFlushEnable(pVCpu);
2748 goto end;
2749 }
2750
2751 /* Success. Query the guest state and figure out what has happened. */
2752
2753 /* Investigate why there was a VM-exit. */
2754 rc2 = VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_REASON, &exitReason);
2755 STAM_COUNTER_INC(&pVCpu->hwaccm.s.paStatExitReasonR0[exitReason & MASK_EXITREASON_STAT]);
2756
2757 exitReason &= 0xffff; /* bit 0-15 contain the exit code. */
2758 rc2 |= VMXReadCachedVMCS(VMX_VMCS32_RO_VM_INSTR_ERROR, &instrError);
2759 rc2 |= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &cbInstr);
2760 rc2 |= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &intInfo);
2761 /* might not be valid; depends on VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID. */
2762 rc2 |= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERRCODE, &errCode);
2763 rc2 |= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INSTR_INFO, &instrInfo);
2764 rc2 |= VMXReadCachedVMCS(VMX_VMCS_RO_EXIT_QUALIFICATION, &exitQualification);
2765 AssertRC(rc2);
2766
2767 /* Sync back the guest state */
2768 rc2 = VMXR0SaveGuestState(pVM, pVCpu, pCtx);
2769 AssertRC(rc2);
2770
2771 /* Note! NOW IT'S SAFE FOR LOGGING! */
2772 VMMR0LogFlushEnable(pVCpu);
2773 Log2(("Raw exit reason %08x\n", exitReason));
2774
2775 /* Check if an injected event was interrupted prematurely. */
2776 rc2 = VMXReadCachedVMCS(VMX_VMCS32_RO_IDT_INFO, &val);
2777 AssertRC(rc2);
2778 pVCpu->hwaccm.s.Event.intInfo = VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(val);
2779 if ( VMX_EXIT_INTERRUPTION_INFO_VALID(pVCpu->hwaccm.s.Event.intInfo)
2780 /* Ignore 'int xx' as they'll be restarted anyway. */
2781 && VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hwaccm.s.Event.intInfo) != VMX_EXIT_INTERRUPTION_INFO_TYPE_SW
2782 /* Ignore software exceptions (such as int3) as they'll reoccur when we restart the instruction anyway. */
2783 && VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hwaccm.s.Event.intInfo) != VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT)
2784 {
2785 Assert(!pVCpu->hwaccm.s.Event.fPending);
2786 pVCpu->hwaccm.s.Event.fPending = true;
2787 /* Error code present? */
2788 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVCpu->hwaccm.s.Event.intInfo))
2789 {
2790 rc2 = VMXReadCachedVMCS(VMX_VMCS32_RO_IDT_ERRCODE, &val);
2791 AssertRC(rc2);
2792 pVCpu->hwaccm.s.Event.errCode = val;
2793 Log(("Pending inject %RX64 at %RGv exit=%08x intInfo=%08x exitQualification=%RGv pending error=%RX64\n", pVCpu->hwaccm.s.Event.intInfo, (RTGCPTR)pCtx->rip, exitReason, intInfo, exitQualification, val));
2794 }
2795 else
2796 {
2797 Log(("Pending inject %RX64 at %RGv exit=%08x intInfo=%08x exitQualification=%RGv\n", pVCpu->hwaccm.s.Event.intInfo, (RTGCPTR)pCtx->rip, exitReason, intInfo, exitQualification));
2798 pVCpu->hwaccm.s.Event.errCode = 0;
2799 }
2800 }
2801#ifdef VBOX_STRICT
2802 else
2803 if ( VMX_EXIT_INTERRUPTION_INFO_VALID(pVCpu->hwaccm.s.Event.intInfo)
2804 /* Ignore software exceptions (such as int3) as they're reoccur when we restart the instruction anyway. */
2805 && VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hwaccm.s.Event.intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT)
2806 {
2807 Log(("Ignore pending inject %RX64 at %RGv exit=%08x intInfo=%08x exitQualification=%RGv\n", pVCpu->hwaccm.s.Event.intInfo, (RTGCPTR)pCtx->rip, exitReason, intInfo, exitQualification));
2808 }
2809
2810 if (exitReason == VMX_EXIT_ERR_INVALID_GUEST_STATE)
2811 HWACCMDumpRegs(pVM, pVCpu, pCtx);
2812#endif
2813
2814 Log2(("E%d: New EIP=%x:%RGv\n", (uint32_t)exitReason, pCtx->cs, (RTGCPTR)pCtx->rip));
2815 Log2(("Exit reason %d, exitQualification %RGv\n", (uint32_t)exitReason, exitQualification));
2816 Log2(("instrInfo=%d instrError=%d instr length=%d\n", (uint32_t)instrInfo, (uint32_t)instrError, (uint32_t)cbInstr));
2817 Log2(("Interruption error code %d\n", (uint32_t)errCode));
2818 Log2(("IntInfo = %08x\n", (uint32_t)intInfo));
2819
2820 /* Sync back the TPR if it was changed. */
2821 if ( fSetupTPRCaching
2822 && u8LastTPR != pVCpu->hwaccm.s.vmx.pbVAPIC[0x80])
2823 {
2824 rc2 = PDMApicSetTPR(pVCpu, pVCpu->hwaccm.s.vmx.pbVAPIC[0x80]);
2825 AssertRC(rc2);
2826 }
2827
2828 STAM_PROFILE_ADV_STOP_START(&pVCpu->hwaccm.s.StatExit1, &pVCpu->hwaccm.s.StatExit2, x);
2829
2830 /* Some cases don't need a complete resync of the guest CPU state; handle them here. */
2831 Assert(rc == VINF_SUCCESS); /* might consider VERR_IPE_UNINITIALIZED_STATUS here later... */
2832 switch (exitReason)
2833 {
2834 case VMX_EXIT_EXCEPTION: /* 0 Exception or non-maskable interrupt (NMI). */
2835 case VMX_EXIT_EXTERNAL_IRQ: /* 1 External interrupt. */
2836 {
2837 uint32_t vector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(intInfo);
2838
2839 if (!VMX_EXIT_INTERRUPTION_INFO_VALID(intInfo))
2840 {
2841 Assert(exitReason == VMX_EXIT_EXTERNAL_IRQ);
2842#if 0 //def VBOX_WITH_VMMR0_DISABLE_PREEMPTION
2843 if ( RTThreadPreemptIsPendingTrusty()
2844 && !RTThreadPreemptIsPending(NIL_RTTHREAD))
2845 goto ResumeExecution;
2846#endif
2847 /* External interrupt; leave to allow it to be dispatched again. */
2848 rc = VINF_EM_RAW_INTERRUPT;
2849 break;
2850 }
2851 STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2852 switch (VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo))
2853 {
2854 case VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI: /* Non-maskable interrupt. */
2855 /* External interrupt; leave to allow it to be dispatched again. */
2856 rc = VINF_EM_RAW_INTERRUPT;
2857 break;
2858
2859 case VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT: /* External hardware interrupt. */
2860 AssertFailed(); /* can't come here; fails the first check. */
2861 break;
2862
2863 case VMX_EXIT_INTERRUPTION_INFO_TYPE_DBEXCPT: /* Unknown why we get this type for #DB */
2864 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT: /* Software exception. (#BP or #OF) */
2865 Assert(vector == 1 || vector == 3 || vector == 4);
2866 /* no break */
2867 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT: /* Hardware exception. */
2868 Log2(("Hardware/software interrupt %d\n", vector));
2869 switch (vector)
2870 {
2871 case X86_XCPT_NM:
2872 {
2873 Log(("#NM fault at %RGv error code %x\n", (RTGCPTR)pCtx->rip, errCode));
2874
2875 /** @todo don't intercept #NM exceptions anymore when we've activated the guest FPU state. */
2876 /* If we sync the FPU/XMM state on-demand, then we can continue execution as if nothing has happened. */
2877 rc = CPUMR0LoadGuestFPU(pVM, pVCpu, pCtx);
2878 if (rc == VINF_SUCCESS)
2879 {
2880 Assert(CPUMIsGuestFPUStateActive(pVCpu));
2881
2882 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitShadowNM);
2883
2884 /* Continue execution. */
2885 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_CR0;
2886
2887 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2888 goto ResumeExecution;
2889 }
2890
2891 Log(("Forward #NM fault to the guest\n"));
2892 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestNM);
2893 rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, 0);
2894 AssertRC(rc2);
2895 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2896 goto ResumeExecution;
2897 }
2898
2899 case X86_XCPT_PF: /* Page fault */
2900 {
2901#ifdef DEBUG
2902 if (pVM->hwaccm.s.fNestedPaging)
2903 { /* A genuine pagefault.
2904 * Forward the trap to the guest by injecting the exception and resuming execution.
2905 */
2906 Log(("Guest page fault at %RGv cr2=%RGv error code %RGv rsp=%RGv\n", (RTGCPTR)pCtx->rip, exitQualification, errCode, (RTGCPTR)pCtx->rsp));
2907
2908 Assert(CPUMIsGuestInPagedProtectedModeEx(pCtx));
2909
2910 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestPF);
2911
2912 /* Now we must update CR2. */
2913 pCtx->cr2 = exitQualification;
2914 rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
2915 AssertRC(rc2);
2916
2917 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2918 goto ResumeExecution;
2919 }
2920#endif
2921 Assert(!pVM->hwaccm.s.fNestedPaging);
2922
2923#ifdef VBOX_HWACCM_WITH_GUEST_PATCHING
2924 /* Shortcut for APIC TPR reads and writes; 32 bits guests only */
2925 if ( pVM->hwaccm.s.fTRPPatchingAllowed
2926 && pVM->hwaccm.s.pGuestPatchMem
2927 && (exitQualification & 0xfff) == 0x080
2928 && !(errCode & X86_TRAP_PF_P) /* not present */
2929 && CPUMGetGuestCPL(pVCpu, CPUMCTX2CORE(pCtx)) == 0
2930 && !CPUMIsGuestInLongModeEx(pCtx)
2931 && pVM->hwaccm.s.cPatches < RT_ELEMENTS(pVM->hwaccm.s.aPatches))
2932 {
2933 RTGCPHYS GCPhysApicBase, GCPhys;
2934 PDMApicGetBase(pVM, &GCPhysApicBase); /* @todo cache this */
2935 GCPhysApicBase &= PAGE_BASE_GC_MASK;
2936
2937 rc = PGMGstGetPage(pVCpu, (RTGCPTR)exitQualification, NULL, &GCPhys);
2938 if ( rc == VINF_SUCCESS
2939 && GCPhys == GCPhysApicBase)
2940 {
2941 /* Only attempt to patch the instruction once. */
2942 PHWACCMTPRPATCH pPatch = (PHWACCMTPRPATCH)RTAvloU32Get(&pVM->hwaccm.s.PatchTree, (AVLOU32KEY)pCtx->eip);
2943 if (!pPatch)
2944 {
2945 rc = VINF_EM_HWACCM_PATCH_TPR_INSTR;
2946 break;
2947 }
2948 }
2949 }
2950#endif
2951
2952 Log2(("Page fault at %RGv error code %x\n", exitQualification, errCode));
2953 /* Exit qualification contains the linear address of the page fault. */
2954 TRPMAssertTrap(pVCpu, X86_XCPT_PF, TRPM_TRAP);
2955 TRPMSetErrorCode(pVCpu, errCode);
2956 TRPMSetFaultAddress(pVCpu, exitQualification);
2957
2958 /* Shortcut for APIC TPR reads and writes. */
2959 if ( (exitQualification & 0xfff) == 0x080
2960 && !(errCode & X86_TRAP_PF_P) /* not present */
2961 && fSetupTPRCaching
2962 && (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
2963 {
2964 RTGCPHYS GCPhysApicBase, GCPhys;
2965 PDMApicGetBase(pVM, &GCPhysApicBase); /* @todo cache this */
2966 GCPhysApicBase &= PAGE_BASE_GC_MASK;
2967
2968 rc = PGMGstGetPage(pVCpu, (RTGCPTR)exitQualification, NULL, &GCPhys);
2969 if ( rc == VINF_SUCCESS
2970 && GCPhys == GCPhysApicBase)
2971 {
2972 Log(("Enable VT-x virtual APIC access filtering\n"));
2973 rc2 = IOMMMIOMapMMIOHCPage(pVM, GCPhysApicBase, pVM->hwaccm.s.vmx.pAPICPhys, X86_PTE_RW | X86_PTE_P);
2974 AssertRC(rc2);
2975 }
2976 }
2977
2978 /* Forward it to our trap handler first, in case our shadow pages are out of sync. */
2979 rc = PGMTrap0eHandler(pVCpu, errCode, CPUMCTX2CORE(pCtx), (RTGCPTR)exitQualification);
2980 Log2(("PGMTrap0eHandler %RGv returned %Rrc\n", (RTGCPTR)pCtx->rip, VBOXSTRICTRC_VAL(rc)));
2981
2982 if (rc == VINF_SUCCESS)
2983 { /* We've successfully synced our shadow pages, so let's just continue execution. */
2984 Log2(("Shadow page fault at %RGv cr2=%RGv error code %x\n", (RTGCPTR)pCtx->rip, exitQualification ,errCode));
2985 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitShadowPF);
2986
2987 TRPMResetTrap(pVCpu);
2988 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2989 goto ResumeExecution;
2990 }
2991 else
2992 if (rc == VINF_EM_RAW_GUEST_TRAP)
2993 { /* A genuine pagefault.
2994 * Forward the trap to the guest by injecting the exception and resuming execution.
2995 */
2996 Log2(("Forward page fault to the guest\n"));
2997
2998 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestPF);
2999 /* The error code might have been changed. */
3000 errCode = TRPMGetErrorCode(pVCpu);
3001
3002 TRPMResetTrap(pVCpu);
3003
3004 /* Now we must update CR2. */
3005 pCtx->cr2 = exitQualification;
3006 rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
3007 AssertRC(rc2);
3008
3009 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3010 goto ResumeExecution;
3011 }
3012#ifdef VBOX_STRICT
3013 if (rc != VINF_EM_RAW_EMULATE_INSTR && rc != VINF_EM_RAW_EMULATE_IO_BLOCK)
3014 Log2(("PGMTrap0eHandler failed with %d\n", VBOXSTRICTRC_VAL(rc)));
3015#endif
3016 /* Need to go back to the recompiler to emulate the instruction. */
3017 TRPMResetTrap(pVCpu);
3018 break;
3019 }
3020
3021 case X86_XCPT_MF: /* Floating point exception. */
3022 {
3023 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestMF);
3024 if (!(pCtx->cr0 & X86_CR0_NE))
3025 {
3026 /* old style FPU error reporting needs some extra work. */
3027 /** @todo don't fall back to the recompiler, but do it manually. */
3028 rc = VINF_EM_RAW_EMULATE_INSTR;
3029 break;
3030 }
3031 Log(("Trap %x at %04X:%RGv\n", vector, pCtx->cs, (RTGCPTR)pCtx->rip));
3032 rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
3033 AssertRC(rc2);
3034
3035 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3036 goto ResumeExecution;
3037 }
3038
3039 case X86_XCPT_DB: /* Debug exception. */
3040 {
3041 uint64_t uDR6;
3042
3043 /* DR6, DR7.GD and IA32_DEBUGCTL.LBR are not updated yet.
3044 *
3045 * Exit qualification bits:
3046 * 3:0 B0-B3 which breakpoint condition was met
3047 * 12:4 Reserved (0)
3048 * 13 BD - debug register access detected
3049 * 14 BS - single step execution or branch taken
3050 * 63:15 Reserved (0)
3051 */
3052 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestDB);
3053
3054 /* Note that we don't support guest and host-initiated debugging at the same time. */
3055
3056 uDR6 = X86_DR6_INIT_VAL;
3057 uDR6 |= (exitQualification & (X86_DR6_B0|X86_DR6_B1|X86_DR6_B2|X86_DR6_B3|X86_DR6_BD|X86_DR6_BS));
3058 rc = DBGFRZTrap01Handler(pVM, pVCpu, CPUMCTX2CORE(pCtx), uDR6);
3059 if (rc == VINF_EM_RAW_GUEST_TRAP)
3060 {
3061 /* Update DR6 here. */
3062 pCtx->dr[6] = uDR6;
3063
3064 /* Resync DR6 if the debug state is active. */
3065 if (CPUMIsGuestDebugStateActive(pVCpu))
3066 ASMSetDR6(pCtx->dr[6]);
3067
3068 /* X86_DR7_GD will be cleared if drx accesses should be trapped inside the guest. */
3069 pCtx->dr[7] &= ~X86_DR7_GD;
3070
3071 /* Paranoia. */
3072 pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
3073 pCtx->dr[7] &= ~(RT_BIT(11) | RT_BIT(12) | RT_BIT(14) | RT_BIT(15)); /* must be zero */
3074 pCtx->dr[7] |= 0x400; /* must be one */
3075
3076 /* Resync DR7 */
3077 rc2 = VMXWriteVMCS64(VMX_VMCS64_GUEST_DR7, pCtx->dr[7]);
3078 AssertRC(rc2);
3079
3080 Log(("Trap %x (debug) at %RGv exit qualification %RX64 dr6=%x dr7=%x\n", vector, (RTGCPTR)pCtx->rip, exitQualification, (uint32_t)pCtx->dr[6], (uint32_t)pCtx->dr[7]));
3081 rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
3082 AssertRC(rc2);
3083
3084 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3085 goto ResumeExecution;
3086 }
3087 /* Return to ring 3 to deal with the debug exit code. */
3088 Log(("Debugger hardware BP at %04x:%RGv (rc=%Rrc)\n", pCtx->cs, pCtx->rip, VBOXSTRICTRC_VAL(rc)));
3089 break;
3090 }
3091
3092 case X86_XCPT_BP: /* Breakpoint. */
3093 {
3094 rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pCtx));
3095 if (rc == VINF_EM_RAW_GUEST_TRAP)
3096 {
3097 Log(("Guest #BP at %04x:%RGv\n", pCtx->cs, pCtx->rip));
3098 rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
3099 AssertRC(rc2);
3100 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3101 goto ResumeExecution;
3102 }
3103 if (rc == VINF_SUCCESS)
3104 {
3105 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3106 goto ResumeExecution;
3107 }
3108 Log(("Debugger BP at %04x:%RGv (rc=%Rrc)\n", pCtx->cs, pCtx->rip, VBOXSTRICTRC_VAL(rc)));
3109 break;
3110 }
3111
3112 case X86_XCPT_GP: /* General protection failure exception.*/
3113 {
3114 uint32_t cbOp;
3115 uint32_t cbSize;
3116 PDISCPUSTATE pDis = &pVCpu->hwaccm.s.DisState;
3117
3118 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestGP);
3119#ifdef VBOX_STRICT
3120 if ( !CPUMIsGuestInRealModeEx(pCtx)
3121 || !pVM->hwaccm.s.vmx.pRealModeTSS)
3122 {
3123 Log(("Trap %x at %04X:%RGv errorCode=%RGv\n", vector, pCtx->cs, (RTGCPTR)pCtx->rip, errCode));
3124 rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
3125 AssertRC(rc2);
3126 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3127 goto ResumeExecution;
3128 }
3129#endif
3130 Assert(CPUMIsGuestInRealModeEx(pCtx));
3131
3132 LogFlow(("Real mode X86_XCPT_GP instruction emulation at %x:%RGv\n", pCtx->cs, (RTGCPTR)pCtx->rip));
3133
3134 rc2 = EMInterpretDisasOne(pVM, pVCpu, CPUMCTX2CORE(pCtx), pDis, &cbOp);
3135 if (RT_SUCCESS(rc2))
3136 {
3137 bool fUpdateRIP = true;
3138
3139 rc = VINF_SUCCESS;
3140 Assert(cbOp == pDis->opsize);
3141 switch (pDis->pCurInstr->opcode)
3142 {
3143 case OP_CLI:
3144 pCtx->eflags.Bits.u1IF = 0;
3145 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCli);
3146 break;
3147
3148 case OP_STI:
3149 pCtx->eflags.Bits.u1IF = 1;
3150 EMSetInhibitInterruptsPC(pVCpu, pCtx->rip + pDis->opsize);
3151 Assert(VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
3152 rc2 = VMXWriteVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
3153 AssertRC(rc2);
3154 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitSti);
3155 break;
3156
3157 case OP_HLT:
3158 fUpdateRIP = false;
3159 rc = VINF_EM_HALT;
3160 pCtx->rip += pDis->opsize;
3161 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitHlt);
3162 break;
3163
3164 case OP_POPF:
3165 {
3166 RTGCPTR GCPtrStack;
3167 uint32_t cbParm;
3168 uint32_t uMask;
3169 X86EFLAGS eflags;
3170
3171 if (pDis->prefix & PREFIX_OPSIZE)
3172 {
3173 cbParm = 4;
3174 uMask = 0xffffffff;
3175 }
3176 else
3177 {
3178 cbParm = 2;
3179 uMask = 0xffff;
3180 }
3181
3182 rc2 = SELMToFlatEx(pVM, DIS_SELREG_SS, CPUMCTX2CORE(pCtx), pCtx->esp & uMask, 0, &GCPtrStack);
3183 if (RT_FAILURE(rc2))
3184 {
3185 rc = VERR_EM_INTERPRETER;
3186 break;
3187 }
3188 eflags.u = 0;
3189 rc2 = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &eflags.u, cbParm);
3190 if (RT_FAILURE(rc2))
3191 {
3192 rc = VERR_EM_INTERPRETER;
3193 break;
3194 }
3195 LogFlow(("POPF %x -> %RGv mask=%x\n", eflags.u, pCtx->rsp, uMask));
3196 pCtx->eflags.u = (pCtx->eflags.u & ~(X86_EFL_POPF_BITS & uMask)) | (eflags.u & X86_EFL_POPF_BITS & uMask);
3197 /* RF cleared when popped in real mode; see pushf description in AMD manual. */
3198 pCtx->eflags.Bits.u1RF = 0;
3199 pCtx->esp += cbParm;
3200 pCtx->esp &= uMask;
3201
3202 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitPopf);
3203 break;
3204 }
3205
3206 case OP_PUSHF:
3207 {
3208 RTGCPTR GCPtrStack;
3209 uint32_t cbParm;
3210 uint32_t uMask;
3211 X86EFLAGS eflags;
3212
3213 if (pDis->prefix & PREFIX_OPSIZE)
3214 {
3215 cbParm = 4;
3216 uMask = 0xffffffff;
3217 }
3218 else
3219 {
3220 cbParm = 2;
3221 uMask = 0xffff;
3222 }
3223
3224 rc2 = SELMToFlatEx(pVM, DIS_SELREG_SS, CPUMCTX2CORE(pCtx), (pCtx->esp - cbParm) & uMask, 0, &GCPtrStack);
3225 if (RT_FAILURE(rc2))
3226 {
3227 rc = VERR_EM_INTERPRETER;
3228 break;
3229 }
3230 eflags = pCtx->eflags;
3231 /* RF & VM cleared when pushed in real mode; see pushf description in AMD manual. */
3232 eflags.Bits.u1RF = 0;
3233 eflags.Bits.u1VM = 0;
3234
3235 rc2 = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &eflags.u, cbParm);
3236 if (RT_FAILURE(rc2))
3237 {
3238 rc = VERR_EM_INTERPRETER;
3239 break;
3240 }
3241 LogFlow(("PUSHF %x -> %RGv\n", eflags.u, GCPtrStack));
3242 pCtx->esp -= cbParm;
3243 pCtx->esp &= uMask;
3244 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitPushf);
3245 break;
3246 }
3247
3248 case OP_IRET:
3249 {
3250 RTGCPTR GCPtrStack;
3251 uint32_t uMask = 0xffff;
3252 uint16_t aIretFrame[3];
3253
3254 if (pDis->prefix & (PREFIX_OPSIZE | PREFIX_ADDRSIZE))
3255 {
3256 rc = VERR_EM_INTERPRETER;
3257 break;
3258 }
3259
3260 rc2 = SELMToFlatEx(pVM, DIS_SELREG_SS, CPUMCTX2CORE(pCtx), pCtx->esp & uMask, 0, &GCPtrStack);
3261 if (RT_FAILURE(rc2))
3262 {
3263 rc = VERR_EM_INTERPRETER;
3264 break;
3265 }
3266 rc2 = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
3267 if (RT_FAILURE(rc2))
3268 {
3269 rc = VERR_EM_INTERPRETER;
3270 break;
3271 }
3272 pCtx->ip = aIretFrame[0];
3273 pCtx->cs = aIretFrame[1];
3274 pCtx->csHid.u64Base = pCtx->cs << 4;
3275 pCtx->eflags.u = (pCtx->eflags.u & ~(X86_EFL_POPF_BITS & uMask)) | (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
3276 pCtx->sp += sizeof(aIretFrame);
3277
3278 LogFlow(("iret to %04x:%x\n", pCtx->cs, pCtx->ip));
3279 fUpdateRIP = false;
3280 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIret);
3281 break;
3282 }
3283
3284 case OP_INT:
3285 {
3286 uint32_t intInfo2;
3287
3288 LogFlow(("Realmode: INT %x\n", pDis->param1.parval & 0xff));
3289 intInfo2 = pDis->param1.parval & 0xff;
3290 intInfo2 |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
3291 intInfo2 |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
3292
3293 rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo2, cbOp, 0);
3294 AssertRC(VBOXSTRICTRC_VAL(rc));
3295 fUpdateRIP = false;
3296 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInt);
3297 break;
3298 }
3299
3300 case OP_INTO:
3301 {
3302 if (pCtx->eflags.Bits.u1OF)
3303 {
3304 uint32_t intInfo2;
3305
3306 LogFlow(("Realmode: INTO\n"));
3307 intInfo2 = X86_XCPT_OF;
3308 intInfo2 |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
3309 intInfo2 |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
3310
3311 rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo2, cbOp, 0);
3312 AssertRC(VBOXSTRICTRC_VAL(rc));
3313 fUpdateRIP = false;
3314 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInt);
3315 }
3316 break;
3317 }
3318
3319 case OP_INT3:
3320 {
3321 uint32_t intInfo2;
3322
3323 LogFlow(("Realmode: INT 3\n"));
3324 intInfo2 = 3;
3325 intInfo2 |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
3326 intInfo2 |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
3327
3328 rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo2, cbOp, 0);
3329 AssertRC(VBOXSTRICTRC_VAL(rc));
3330 fUpdateRIP = false;
3331 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInt);
3332 break;
3333 }
3334
3335 default:
3336 rc = EMInterpretInstructionCPU(pVM, pVCpu, pDis, CPUMCTX2CORE(pCtx), 0, EMCODETYPE_SUPERVISOR, &cbSize);
3337 break;
3338 }
3339
3340 if (rc == VINF_SUCCESS)
3341 {
3342 if (fUpdateRIP)
3343 pCtx->rip += cbOp; /* Move on to the next instruction. */
3344
3345 /* lidt, lgdt can end up here. In the future crx changes as well. Just reload the whole context to be done with it. */
3346 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_ALL;
3347
3348 /* Only resume if successful. */
3349 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3350 goto ResumeExecution;
3351 }
3352 }
3353 else
3354 rc = VERR_EM_INTERPRETER;
3355
3356 AssertMsg(rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_EM_HALT, ("Unexpected rc=%Rrc\n", VBOXSTRICTRC_VAL(rc)));
3357 break;
3358 }
3359
3360#ifdef VBOX_STRICT
3361 case X86_XCPT_XF: /* SIMD exception. */
3362 case X86_XCPT_DE: /* Divide error. */
3363 case X86_XCPT_UD: /* Unknown opcode exception. */
3364 case X86_XCPT_SS: /* Stack segment exception. */
3365 case X86_XCPT_NP: /* Segment not present exception. */
3366 {
3367 switch(vector)
3368 {
3369 case X86_XCPT_DE:
3370 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestDE);
3371 break;
3372 case X86_XCPT_UD:
3373 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestUD);
3374 break;
3375 case X86_XCPT_SS:
3376 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestSS);
3377 break;
3378 case X86_XCPT_NP:
3379 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestNP);
3380 break;
3381 }
3382
3383 Log(("Trap %x at %04X:%RGv\n", vector, pCtx->cs, (RTGCPTR)pCtx->rip));
3384 rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
3385 AssertRC(rc2);
3386
3387 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3388 goto ResumeExecution;
3389 }
3390#endif
3391 default:
3392 if ( CPUMIsGuestInRealModeEx(pCtx)
3393 && pVM->hwaccm.s.vmx.pRealModeTSS)
3394 {
3395 Log(("Real Mode Trap %x at %04x:%04X error code %x\n", vector, pCtx->cs, pCtx->eip, errCode));
3396 rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
3397 AssertRC(VBOXSTRICTRC_VAL(rc)); /* Strict RC check below. */
3398
3399 /* Go back to ring 3 in case of a triple fault. */
3400 if ( vector == X86_XCPT_DF
3401 && rc == VINF_EM_RESET)
3402 break;
3403
3404 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3405 goto ResumeExecution;
3406 }
3407 AssertMsgFailed(("Unexpected vm-exit caused by exception %x\n", vector));
3408 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
3409 break;
3410 } /* switch (vector) */
3411
3412 break;
3413
3414 default:
3415 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_CODE;
3416 AssertMsgFailed(("Unexpected interruption code %x\n", intInfo));
3417 break;
3418 }
3419
3420 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3421 break;
3422 }
3423
3424 case VMX_EXIT_EPT_VIOLATION: /* 48 EPT violation. An attempt to access memory with a guest-physical address was disallowed by the configuration of the EPT paging structures. */
3425 {
3426 RTGCPHYS GCPhys;
3427
3428 Assert(pVM->hwaccm.s.fNestedPaging);
3429
3430 rc2 = VMXReadVMCS64(VMX_VMCS_EXIT_PHYS_ADDR_FULL, &GCPhys);
3431 AssertRC(rc2);
3432 Assert(((exitQualification >> 7) & 3) != 2);
3433
3434 /* Determine the kind of violation. */
3435 errCode = 0;
3436 if (exitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
3437 errCode |= X86_TRAP_PF_ID;
3438
3439 if (exitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
3440 errCode |= X86_TRAP_PF_RW;
3441
3442 /* If the page is present, then it's a page level protection fault. */
3443 if (exitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
3444 {
3445 errCode |= X86_TRAP_PF_P;
3446 }
3447 else
3448 {
3449 /* Shortcut for APIC TPR reads and writes. */
3450 if ( (GCPhys & 0xfff) == 0x080
3451 && GCPhys > 0x1000000 /* to skip VGA frame buffer accesses */
3452 && fSetupTPRCaching
3453 && (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
3454 {
3455 RTGCPHYS GCPhysApicBase;
3456 PDMApicGetBase(pVM, &GCPhysApicBase); /* @todo cache this */
3457 GCPhysApicBase &= PAGE_BASE_GC_MASK;
3458 if (GCPhys == GCPhysApicBase + 0x80)
3459 {
3460 Log(("Enable VT-x virtual APIC access filtering\n"));
3461 rc2 = IOMMMIOMapMMIOHCPage(pVM, GCPhysApicBase, pVM->hwaccm.s.vmx.pAPICPhys, X86_PTE_RW | X86_PTE_P);
3462 AssertRC(rc2);
3463 }
3464 }
3465 }
3466 Log(("EPT Page fault %x at %RGp error code %x\n", (uint32_t)exitQualification, GCPhys, errCode));
3467
3468 /* GCPhys contains the guest physical address of the page fault. */
3469 TRPMAssertTrap(pVCpu, X86_XCPT_PF, TRPM_TRAP);
3470 TRPMSetErrorCode(pVCpu, errCode);
3471 TRPMSetFaultAddress(pVCpu, GCPhys);
3472
3473 /* Handle the pagefault trap for the nested shadow table. */
3474 rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, errCode, CPUMCTX2CORE(pCtx), GCPhys);
3475 Log2(("PGMR0Trap0eHandlerNestedPaging %RGv returned %Rrc\n", (RTGCPTR)pCtx->rip, VBOXSTRICTRC_VAL(rc)));
3476 if (rc == VINF_SUCCESS)
3477 { /* We've successfully synced our shadow pages, so let's just continue execution. */
3478 Log2(("Shadow page fault at %RGv cr2=%RGp error code %x\n", (RTGCPTR)pCtx->rip, exitQualification , errCode));
3479 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitReasonNPF);
3480
3481 TRPMResetTrap(pVCpu);
3482 goto ResumeExecution;
3483 }
3484
3485#ifdef VBOX_STRICT
3486 if (rc != VINF_EM_RAW_EMULATE_INSTR)
3487 LogFlow(("PGMTrap0eHandlerNestedPaging failed with %d\n", VBOXSTRICTRC_VAL(rc)));
3488#endif
3489 /* Need to go back to the recompiler to emulate the instruction. */
3490 TRPMResetTrap(pVCpu);
3491 break;
3492 }
3493
3494 case VMX_EXIT_EPT_MISCONFIG:
3495 {
3496 RTGCPHYS GCPhys;
3497
3498 Assert(pVM->hwaccm.s.fNestedPaging);
3499
3500 rc2 = VMXReadVMCS64(VMX_VMCS_EXIT_PHYS_ADDR_FULL, &GCPhys);
3501 AssertRC(rc2);
3502 Log(("VMX_EXIT_EPT_MISCONFIG for %RGp\n", GCPhys));
3503
3504 /* Shortcut for APIC TPR reads and writes. */
3505 if ( (GCPhys & 0xfff) == 0x080
3506 && GCPhys > 0x1000000 /* to skip VGA frame buffer accesses */
3507 && fSetupTPRCaching
3508 && (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
3509 {
3510 RTGCPHYS GCPhysApicBase;
3511 PDMApicGetBase(pVM, &GCPhysApicBase); /* @todo cache this */
3512 GCPhysApicBase &= PAGE_BASE_GC_MASK;
3513 if (GCPhys == GCPhysApicBase + 0x80)
3514 {
3515 Log(("Enable VT-x virtual APIC access filtering\n"));
3516 rc2 = IOMMMIOMapMMIOHCPage(pVM, GCPhysApicBase, pVM->hwaccm.s.vmx.pAPICPhys, X86_PTE_RW | X86_PTE_P);
3517 AssertRC(rc2);
3518 }
3519 }
3520
3521 rc = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pCtx), GCPhys, UINT32_MAX);
3522 if (rc == VINF_SUCCESS)
3523 {
3524 Log2(("PGMR0Trap0eHandlerNPMisconfig(,,,%RGp) at %RGv -> resume\n", GCPhys, (RTGCPTR)pCtx->rip));
3525 goto ResumeExecution;
3526 }
3527
3528 Log2(("PGMR0Trap0eHandlerNPMisconfig(,,,%RGp) at %RGv -> %Rrc\n", GCPhys, (RTGCPTR)pCtx->rip, VBOXSTRICTRC_VAL(rc)));
3529 break;
3530 }
3531
3532 case VMX_EXIT_IRQ_WINDOW: /* 7 Interrupt window. */
3533 /* Clear VM-exit on IF=1 change. */
3534 LogFlow(("VMX_EXIT_IRQ_WINDOW %RGv pending=%d IF=%d\n", (RTGCPTR)pCtx->rip, VMCPU_FF_ISPENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC|VMCPU_FF_INTERRUPT_PIC)), pCtx->eflags.Bits.u1IF));
3535 pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_IRQ_WINDOW_EXIT;
3536 rc2 = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
3537 AssertRC(rc2);
3538 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIrqWindow);
3539 goto ResumeExecution; /* we check for pending guest interrupts there */
3540
3541 case VMX_EXIT_WBINVD: /* 54 Guest software attempted to execute WBINVD. (conditional) */
3542 case VMX_EXIT_INVD: /* 13 Guest software attempted to execute INVD. (unconditional) */
3543 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInvd);
3544 /* Skip instruction and continue directly. */
3545 pCtx->rip += cbInstr;
3546 /* Continue execution.*/
3547 goto ResumeExecution;
3548
3549 case VMX_EXIT_CPUID: /* 10 Guest software attempted to execute CPUID. */
3550 {
3551 Log2(("VMX: Cpuid %x\n", pCtx->eax));
3552 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCpuid);
3553 rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pCtx));
3554 if (rc == VINF_SUCCESS)
3555 {
3556 /* Update EIP and continue execution. */
3557 Assert(cbInstr == 2);
3558 pCtx->rip += cbInstr;
3559 goto ResumeExecution;
3560 }
3561 AssertMsgFailed(("EMU: cpuid failed with %Rrc\n", VBOXSTRICTRC_VAL(rc)));
3562 rc = VINF_EM_RAW_EMULATE_INSTR;
3563 break;
3564 }
3565
3566 case VMX_EXIT_RDPMC: /* 15 Guest software attempted to execute RDPMC. */
3567 {
3568 Log2(("VMX: Rdpmc %x\n", pCtx->ecx));
3569 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitRdpmc);
3570 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pCtx));
3571 if (rc == VINF_SUCCESS)
3572 {
3573 /* Update EIP and continue execution. */
3574 Assert(cbInstr == 2);
3575 pCtx->rip += cbInstr;
3576 goto ResumeExecution;
3577 }
3578 rc = VINF_EM_RAW_EMULATE_INSTR;
3579 break;
3580 }
3581
3582 case VMX_EXIT_RDTSC: /* 16 Guest software attempted to execute RDTSC. */
3583 {
3584 Log2(("VMX: Rdtsc\n"));
3585 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitRdtsc);
3586 rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pCtx));
3587 if (rc == VINF_SUCCESS)
3588 {
3589 /* Update EIP and continue execution. */
3590 Assert(cbInstr == 2);
3591 pCtx->rip += cbInstr;
3592 goto ResumeExecution;
3593 }
3594 rc = VINF_EM_RAW_EMULATE_INSTR;
3595 break;
3596 }
3597
3598 case VMX_EXIT_INVPG: /* 14 Guest software attempted to execute INVPG. */
3599 {
3600 Log2(("VMX: invlpg\n"));
3601 Assert(!pVM->hwaccm.s.fNestedPaging);
3602
3603 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInvpg);
3604 rc = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pCtx), exitQualification);
3605 if (rc == VINF_SUCCESS)
3606 {
3607 /* Update EIP and continue execution. */
3608 pCtx->rip += cbInstr;
3609 goto ResumeExecution;
3610 }
3611 AssertMsg(rc == VERR_EM_INTERPRETER, ("EMU: invlpg %RGv failed with %Rrc\n", exitQualification, VBOXSTRICTRC_VAL(rc)));
3612 break;
3613 }
3614
3615 case VMX_EXIT_MONITOR: /* 39 Guest software attempted to execute MONITOR. */
3616 {
3617 Log2(("VMX: monitor\n"));
3618
3619 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitMonitor);
3620 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pCtx));
3621 if (rc == VINF_SUCCESS)
3622 {
3623 /* Update EIP and continue execution. */
3624 pCtx->rip += cbInstr;
3625 goto ResumeExecution;
3626 }
3627 AssertMsg(rc == VERR_EM_INTERPRETER, ("EMU: monitor failed with %Rrc\n", VBOXSTRICTRC_VAL(rc)));
3628 break;
3629 }
3630
3631 case VMX_EXIT_WRMSR: /* 32 WRMSR. Guest software attempted to execute WRMSR. */
3632 /* When an interrupt is pending, we'll let MSR_K8_LSTAR writes fault in our TPR patch code. */
3633 if ( pVM->hwaccm.s.fTPRPatchingActive
3634 && pCtx->ecx == MSR_K8_LSTAR)
3635 {
3636 Assert(!CPUMIsGuestInLongModeEx(pCtx));
3637 if ((pCtx->eax & 0xff) != u8LastTPR)
3638 {
3639 Log(("VMX: Faulting MSR_K8_LSTAR write with new TPR value %x\n", pCtx->eax & 0xff));
3640
3641 /* Our patch code uses LSTAR for TPR caching. */
3642 rc2 = PDMApicSetTPR(pVCpu, pCtx->eax & 0xff);
3643 AssertRC(rc2);
3644 }
3645
3646 /* Skip the instruction and continue. */
3647 pCtx->rip += cbInstr; /* wrmsr = [0F 30] */
3648
3649 /* Only resume if successful. */
3650 goto ResumeExecution;
3651 }
3652 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_MSR;
3653 /* no break */
3654 case VMX_EXIT_RDMSR: /* 31 RDMSR. Guest software attempted to execute RDMSR. */
3655 {
3656 uint32_t cbSize;
3657
3658 STAM_COUNTER_INC((exitReason == VMX_EXIT_RDMSR) ? &pVCpu->hwaccm.s.StatExitRdmsr : &pVCpu->hwaccm.s.StatExitWrmsr);
3659
3660 /* Note: the intel manual claims there's a REX version of RDMSR that's slightly different, so we play safe by completely disassembling the instruction. */
3661 Log2(("VMX: %s\n", (exitReason == VMX_EXIT_RDMSR) ? "rdmsr" : "wrmsr"));
3662 rc = EMInterpretInstruction(pVM, pVCpu, CPUMCTX2CORE(pCtx), 0, &cbSize);
3663 if (rc == VINF_SUCCESS)
3664 {
3665 /* EIP has been updated already. */
3666
3667 /* Only resume if successful. */
3668 goto ResumeExecution;
3669 }
3670 AssertMsg(rc == VERR_EM_INTERPRETER, ("EMU: %s failed with %Rrc\n", (exitReason == VMX_EXIT_RDMSR) ? "rdmsr" : "wrmsr", VBOXSTRICTRC_VAL(rc)));
3671 break;
3672 }
3673
3674 case VMX_EXIT_CRX_MOVE: /* 28 Control-register accesses. */
3675 {
3676 STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatExit2Sub2, y2);
3677
3678 switch (VMX_EXIT_QUALIFICATION_CRX_ACCESS(exitQualification))
3679 {
3680 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE:
3681 Log2(("VMX: %RGv mov cr%d, x\n", (RTGCPTR)pCtx->rip, VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification)));
3682 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification)]);
3683 rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
3684 VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification),
3685 VMX_EXIT_QUALIFICATION_CRX_GENREG(exitQualification));
3686
3687 switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification))
3688 {
3689 case 0:
3690 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_CR0 | HWACCM_CHANGED_GUEST_CR3;
3691 break;
3692 case 2:
3693 break;
3694 case 3:
3695 Assert(!pVM->hwaccm.s.fNestedPaging || !CPUMIsGuestInPagedProtectedModeEx(pCtx));
3696 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_CR3;
3697 break;
3698 case 4:
3699 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_CR4;
3700 break;
3701 case 8:
3702 /* CR8 contains the APIC TPR */
3703 Assert(!(pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
3704 break;
3705
3706 default:
3707 AssertFailed();
3708 break;
3709 }
3710 break;
3711
3712 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ:
3713 Log2(("VMX: mov x, crx\n"));
3714 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification)]);
3715
3716 Assert(!pVM->hwaccm.s.fNestedPaging || !CPUMIsGuestInPagedProtectedModeEx(pCtx) || VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification) != USE_REG_CR3);
3717
3718 /* CR8 reads only cause an exit when the TPR shadow feature isn't present. */
3719 Assert(VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification) != 8 || !(pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
3720
3721 rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
3722 VMX_EXIT_QUALIFICATION_CRX_GENREG(exitQualification),
3723 VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification));
3724 break;
3725
3726 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS:
3727 Log2(("VMX: clts\n"));
3728 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCLTS);
3729 rc = EMInterpretCLTS(pVM, pVCpu);
3730 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_CR0;
3731 break;
3732
3733 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW:
3734 Log2(("VMX: lmsw %x\n", VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(exitQualification)));
3735 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitLMSW);
3736 rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(exitQualification));
3737 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_CR0;
3738 break;
3739 }
3740
3741 /* Update EIP if no error occurred. */
3742 if (RT_SUCCESS(rc))
3743 pCtx->rip += cbInstr;
3744
3745 if (rc == VINF_SUCCESS)
3746 {
3747 /* Only resume if successful. */
3748 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub2, y2);
3749 goto ResumeExecution;
3750 }
3751 Assert(rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_PGM_SYNC_CR3);
3752 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub2, y2);
3753 break;
3754 }
3755
3756 case VMX_EXIT_DRX_MOVE: /* 29 Debug-register accesses. */
3757 {
3758 if ( !DBGFIsStepping(pVCpu)
3759 && !CPUMIsHyperDebugStateActive(pVCpu))
3760 {
3761 /* Disable drx move intercepts. */
3762 pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
3763 rc2 = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
3764 AssertRC(rc2);
3765
3766 /* Save the host and load the guest debug state. */
3767 rc2 = CPUMR0LoadGuestDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
3768 AssertRC(rc2);
3769
3770#ifdef LOG_ENABLED
3771 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
3772 Log(("VMX_EXIT_DRX_MOVE: write DR%d genreg %d\n", VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification), VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification)));
3773 else
3774 Log(("VMX_EXIT_DRX_MOVE: read DR%d\n", VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification)));
3775#endif
3776
3777#ifdef VBOX_WITH_STATISTICS
3778 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatDRxContextSwitch);
3779 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
3780 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxWrite);
3781 else
3782 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxRead);
3783#endif
3784
3785 goto ResumeExecution;
3786 }
3787
3788 /** @todo clear VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT after the first time and restore drx registers afterwards */
3789 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
3790 {
3791 Log2(("VMX: mov drx%d, genreg%d\n", VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification), VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification)));
3792 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxWrite);
3793 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
3794 VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification),
3795 VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification));
3796 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_DEBUG;
3797 Log2(("DR7=%08x\n", pCtx->dr[7]));
3798 }
3799 else
3800 {
3801 Log2(("VMX: mov x, drx\n"));
3802 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxRead);
3803 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
3804 VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification),
3805 VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification));
3806 }
3807 /* Update EIP if no error occurred. */
3808 if (RT_SUCCESS(rc))
3809 pCtx->rip += cbInstr;
3810
3811 if (rc == VINF_SUCCESS)
3812 {
3813 /* Only resume if successful. */
3814 goto ResumeExecution;
3815 }
3816 Assert(rc == VERR_EM_INTERPRETER);
3817 break;
3818 }
3819
3820 /* Note: We'll get a #GP if the IO instruction isn't allowed (IOPL or TSS bitmap); no need to double check. */
3821 case VMX_EXIT_PORT_IO: /* 30 I/O instruction. */
3822 {
3823 STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3824 uint32_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(exitQualification);
3825 uint32_t uPort;
3826 bool fIOWrite = (VMX_EXIT_QUALIFICATION_IO_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
3827
3828 /** @todo necessary to make the distinction? */
3829 if (VMX_EXIT_QUALIFICATION_IO_ENCODING(exitQualification) == VMX_EXIT_QUALIFICATION_IO_ENCODING_DX)
3830 {
3831 uPort = pCtx->edx & 0xffff;
3832 }
3833 else
3834 uPort = VMX_EXIT_QUALIFICATION_IO_PORT(exitQualification); /* Immediate encoding. */
3835
3836 /* paranoia */
3837 if (RT_UNLIKELY(uIOWidth == 2 || uIOWidth >= 4))
3838 {
3839 rc = fIOWrite ? VINF_IOM_HC_IOPORT_WRITE : VINF_IOM_HC_IOPORT_READ;
3840 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3841 break;
3842 }
3843
3844 uint32_t cbSize = g_aIOSize[uIOWidth];
3845
3846 if (VMX_EXIT_QUALIFICATION_IO_STRING(exitQualification))
3847 {
3848 /* ins/outs */
3849 PDISCPUSTATE pDis = &pVCpu->hwaccm.s.DisState;
3850
3851 /* Disassemble manually to deal with segment prefixes. */
3852 /** @todo VMX_VMCS_EXIT_GUEST_LINEAR_ADDR contains the flat pointer operand of the instruction. */
3853 /** @todo VMX_VMCS32_RO_EXIT_INSTR_INFO also contains segment prefix info. */
3854 rc2 = EMInterpretDisasOne(pVM, pVCpu, CPUMCTX2CORE(pCtx), pDis, NULL);
3855 if (RT_SUCCESS(rc))
3856 {
3857 if (fIOWrite)
3858 {
3859 Log2(("IOMInterpretOUTSEx %RGv %x size=%d\n", (RTGCPTR)pCtx->rip, uPort, cbSize));
3860 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIOStringWrite);
3861 rc = IOMInterpretOUTSEx(pVM, CPUMCTX2CORE(pCtx), uPort, pDis->prefix, pDis->addrmode, cbSize);
3862 }
3863 else
3864 {
3865 Log2(("IOMInterpretINSEx %RGv %x size=%d\n", (RTGCPTR)pCtx->rip, uPort, cbSize));
3866 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIOStringRead);
3867 rc = IOMInterpretINSEx(pVM, CPUMCTX2CORE(pCtx), uPort, pDis->prefix, pDis->addrmode, cbSize);
3868 }
3869 }
3870 else
3871 rc = VINF_EM_RAW_EMULATE_INSTR;
3872 }
3873 else
3874 {
3875 /* normal in/out */
3876 uint32_t uAndVal = g_aIOOpAnd[uIOWidth];
3877
3878 Assert(!VMX_EXIT_QUALIFICATION_IO_REP(exitQualification));
3879
3880 if (fIOWrite)
3881 {
3882 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIOWrite);
3883 rc = IOMIOPortWrite(pVM, uPort, pCtx->eax & uAndVal, cbSize);
3884 if (rc == VINF_IOM_HC_IOPORT_WRITE)
3885 HWACCMR0SavePendingIOPortWrite(pVCpu, pCtx->rip, pCtx->rip + cbInstr, uPort, uAndVal, cbSize);
3886 }
3887 else
3888 {
3889 uint32_t u32Val = 0;
3890
3891 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIORead);
3892 rc = IOMIOPortRead(pVM, uPort, &u32Val, cbSize);
3893 if (IOM_SUCCESS(rc))
3894 {
3895 /* Write back to the EAX register. */
3896 pCtx->eax = (pCtx->eax & ~uAndVal) | (u32Val & uAndVal);
3897 }
3898 else
3899 if (rc == VINF_IOM_HC_IOPORT_READ)
3900 HWACCMR0SavePendingIOPortRead(pVCpu, pCtx->rip, pCtx->rip + cbInstr, uPort, uAndVal, cbSize);
3901 }
3902 }
3903 /*
3904 * Handled the I/O return codes.
3905 * (The unhandled cases end up with rc == VINF_EM_RAW_EMULATE_INSTR.)
3906 */
3907 if (IOM_SUCCESS(rc))
3908 {
3909 /* Update EIP and continue execution. */
3910 pCtx->rip += cbInstr;
3911 if (RT_LIKELY(rc == VINF_SUCCESS))
3912 {
3913 /* If any IO breakpoints are armed, then we should check if a debug trap needs to be generated. */
3914 if (pCtx->dr[7] & X86_DR7_ENABLED_MASK)
3915 {
3916 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatDRxIOCheck);
3917 for (unsigned i=0;i<4;i++)
3918 {
3919 unsigned uBPLen = g_aIOSize[X86_DR7_GET_LEN(pCtx->dr[7], i)];
3920
3921 if ( (uPort >= pCtx->dr[i] && uPort < pCtx->dr[i] + uBPLen)
3922 && (pCtx->dr[7] & (X86_DR7_L(i) | X86_DR7_G(i)))
3923 && (pCtx->dr[7] & X86_DR7_RW(i, X86_DR7_RW_IO)) == X86_DR7_RW(i, X86_DR7_RW_IO))
3924 {
3925 uint64_t uDR6;
3926
3927 Assert(CPUMIsGuestDebugStateActive(pVCpu));
3928
3929 uDR6 = ASMGetDR6();
3930
3931 /* Clear all breakpoint status flags and set the one we just hit. */
3932 uDR6 &= ~(X86_DR6_B0|X86_DR6_B1|X86_DR6_B2|X86_DR6_B3);
3933 uDR6 |= (uint64_t)RT_BIT(i);
3934
3935 /* Note: AMD64 Architecture Programmer's Manual 13.1:
3936 * Bits 15:13 of the DR6 register is never cleared by the processor and must be cleared by software after
3937 * the contents have been read.
3938 */
3939 ASMSetDR6(uDR6);
3940
3941 /* X86_DR7_GD will be cleared if drx accesses should be trapped inside the guest. */
3942 pCtx->dr[7] &= ~X86_DR7_GD;
3943
3944 /* Paranoia. */
3945 pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
3946 pCtx->dr[7] &= ~(RT_BIT(11) | RT_BIT(12) | RT_BIT(14) | RT_BIT(15)); /* must be zero */
3947 pCtx->dr[7] |= 0x400; /* must be one */
3948
3949 /* Resync DR7 */
3950 rc2 = VMXWriteVMCS64(VMX_VMCS64_GUEST_DR7, pCtx->dr[7]);
3951 AssertRC(rc2);
3952
3953 /* Construct inject info. */
3954 intInfo = X86_XCPT_DB;
3955 intInfo |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
3956 intInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
3957
3958 Log(("Inject IO debug trap at %RGv\n", (RTGCPTR)pCtx->rip));
3959 rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), 0, 0);
3960 AssertRC(rc2);
3961
3962 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3963 goto ResumeExecution;
3964 }
3965 }
3966 }
3967 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3968 goto ResumeExecution;
3969 }
3970 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3971 break;
3972 }
3973
3974#ifdef VBOX_STRICT
3975 if (rc == VINF_IOM_HC_IOPORT_READ)
3976 Assert(!fIOWrite);
3977 else if (rc == VINF_IOM_HC_IOPORT_WRITE)
3978 Assert(fIOWrite);
3979 else
3980 AssertMsg(RT_FAILURE(rc) || rc == VINF_EM_RAW_EMULATE_INSTR || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rc)));
3981#endif
3982 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3983 break;
3984 }
3985
3986 case VMX_EXIT_TPR: /* 43 TPR below threshold. Guest software executed MOV to CR8. */
3987 LogFlow(("VMX_EXIT_TPR\n"));
3988 /* RIP is already set to the next instruction and the TPR has been synced back. Just resume. */
3989 goto ResumeExecution;
3990
3991 case VMX_EXIT_APIC_ACCESS: /* 44 APIC access. Guest software attempted to access memory at a physical address on the APIC-access page. */
3992 {
3993 LogFlow(("VMX_EXIT_APIC_ACCESS\n"));
3994 unsigned uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(exitQualification);
3995
3996 switch(uAccessType)
3997 {
3998 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
3999 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
4000 {
4001 RTGCPHYS GCPhys;
4002 PDMApicGetBase(pVM, &GCPhys);
4003 GCPhys &= PAGE_BASE_GC_MASK;
4004 GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(exitQualification);
4005
4006 LogFlow(("Apic access at %RGp\n", GCPhys));
4007 rc = IOMMMIOPhysHandler(pVM, (uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW, CPUMCTX2CORE(pCtx), GCPhys);
4008 if (rc == VINF_SUCCESS)
4009 goto ResumeExecution; /* rip already updated */
4010 break;
4011 }
4012
4013 default:
4014 rc = VINF_EM_RAW_EMULATE_INSTR;
4015 break;
4016 }
4017 break;
4018 }
4019
4020 case VMX_EXIT_PREEMPTION_TIMER: /* 52 VMX-preemption timer expired. The preemption timer counted down to zero. */
4021 if (!TMTimerPollBool(pVM, pVCpu))
4022 goto ResumeExecution;
4023 rc = VINF_EM_RAW_TIMER_PENDING;
4024 break;
4025
4026 default:
4027 /* The rest is handled after syncing the entire CPU state. */
4028 break;
4029 }
4030
4031 /* Note: the guest state isn't entirely synced back at this stage. */
4032
4033 /* Investigate why there was a VM-exit. (part 2) */
4034 switch (exitReason)
4035 {
4036 case VMX_EXIT_EXCEPTION: /* 0 Exception or non-maskable interrupt (NMI). */
4037 case VMX_EXIT_EXTERNAL_IRQ: /* 1 External interrupt. */
4038 case VMX_EXIT_EPT_VIOLATION:
4039 case VMX_EXIT_EPT_MISCONFIG: /* 49 EPT misconfig is used by the PGM/MMIO optimizations. */
4040 case VMX_EXIT_PREEMPTION_TIMER: /* 52 VMX-preemption timer expired. The preemption timer counted down to zero. */
4041 /* Already handled above. */
4042 break;
4043
4044 case VMX_EXIT_TRIPLE_FAULT: /* 2 Triple fault. */
4045 rc = VINF_EM_RESET; /* Triple fault equals a reset. */
4046 break;
4047
4048 case VMX_EXIT_INIT_SIGNAL: /* 3 INIT signal. */
4049 case VMX_EXIT_SIPI: /* 4 Start-up IPI (SIPI). */
4050 rc = VINF_EM_RAW_INTERRUPT;
4051 AssertFailed(); /* Can't happen. Yet. */
4052 break;
4053
4054 case VMX_EXIT_IO_SMI_IRQ: /* 5 I/O system-management interrupt (SMI). */
4055 case VMX_EXIT_SMI_IRQ: /* 6 Other SMI. */
4056 rc = VINF_EM_RAW_INTERRUPT;
4057 AssertFailed(); /* Can't happen afaik. */
4058 break;
4059
4060 case VMX_EXIT_TASK_SWITCH: /* 9 Task switch: too complicated to emulate, so fall back to the recompiler */
4061 Log(("VMX_EXIT_TASK_SWITCH: exit=%RX64\n", exitQualification));
4062 if ( (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(exitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
4063 && pVCpu->hwaccm.s.Event.fPending)
4064 {
4065 /* Caused by an injected interrupt. */
4066 pVCpu->hwaccm.s.Event.fPending = false;
4067
4068 Log(("VMX_EXIT_TASK_SWITCH: reassert trap %d\n", VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVCpu->hwaccm.s.Event.intInfo)));
4069 Assert(!VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVCpu->hwaccm.s.Event.intInfo));
4070 rc2 = TRPMAssertTrap(pVCpu, VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVCpu->hwaccm.s.Event.intInfo), TRPM_HARDWARE_INT);
4071 AssertRC(rc2);
4072 }
4073 /* else Exceptions and software interrupts can just be restarted. */
4074 rc = VERR_EM_INTERPRETER;
4075 break;
4076
4077 case VMX_EXIT_HLT: /* 12 Guest software attempted to execute HLT. */
4078 /** Check if external interrupts are pending; if so, don't switch back. */
4079 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitHlt);
4080 pCtx->rip++; /* skip hlt */
4081 if (EMShouldContinueAfterHalt(pVCpu, pCtx))
4082 goto ResumeExecution;
4083
4084 rc = VINF_EM_HALT;
4085 break;
4086
4087 case VMX_EXIT_MWAIT: /* 36 Guest software executed MWAIT. */
4088 Log2(("VMX: mwait\n"));
4089 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitMwait);
4090 rc = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pCtx));
4091 if ( rc == VINF_EM_HALT
4092 || rc == VINF_SUCCESS)
4093 {
4094 /* Update EIP and continue execution. */
4095 pCtx->rip += cbInstr;
4096
4097 /** Check if external interrupts are pending; if so, don't switch back. */
4098 if ( rc == VINF_SUCCESS
4099 || ( rc == VINF_EM_HALT
4100 && EMShouldContinueAfterHalt(pVCpu, pCtx))
4101 )
4102 goto ResumeExecution;
4103 }
4104 AssertMsg(rc == VERR_EM_INTERPRETER || rc == VINF_EM_HALT, ("EMU: mwait failed with %Rrc\n", VBOXSTRICTRC_VAL(rc)));
4105 break;
4106
4107 case VMX_EXIT_RSM: /* 17 Guest software attempted to execute RSM in SMM. */
4108 AssertFailed(); /* can't happen. */
4109 rc = VERR_EM_INTERPRETER;
4110 break;
4111
4112 case VMX_EXIT_VMCALL: /* 18 Guest software executed VMCALL. */
4113 case VMX_EXIT_VMCLEAR: /* 19 Guest software executed VMCLEAR. */
4114 case VMX_EXIT_VMLAUNCH: /* 20 Guest software executed VMLAUNCH. */
4115 case VMX_EXIT_VMPTRLD: /* 21 Guest software executed VMPTRLD. */
4116 case VMX_EXIT_VMPTRST: /* 22 Guest software executed VMPTRST. */
4117 case VMX_EXIT_VMREAD: /* 23 Guest software executed VMREAD. */
4118 case VMX_EXIT_VMRESUME: /* 24 Guest software executed VMRESUME. */
4119 case VMX_EXIT_VMWRITE: /* 25 Guest software executed VMWRITE. */
4120 case VMX_EXIT_VMXOFF: /* 26 Guest software executed VMXOFF. */
4121 case VMX_EXIT_VMXON: /* 27 Guest software executed VMXON. */
4122 /** @todo inject #UD immediately */
4123 rc = VERR_EM_INTERPRETER;
4124 break;
4125
4126 case VMX_EXIT_CPUID: /* 10 Guest software attempted to execute CPUID. */
4127 case VMX_EXIT_RDTSC: /* 16 Guest software attempted to execute RDTSC. */
4128 case VMX_EXIT_INVPG: /* 14 Guest software attempted to execute INVPG. */
4129 case VMX_EXIT_CRX_MOVE: /* 28 Control-register accesses. */
4130 case VMX_EXIT_DRX_MOVE: /* 29 Debug-register accesses. */
4131 case VMX_EXIT_PORT_IO: /* 30 I/O instruction. */
4132 case VMX_EXIT_RDPMC: /* 15 Guest software attempted to execute RDPMC. */
4133 /* already handled above */
4134 AssertMsg( rc == VINF_PGM_CHANGE_MODE
4135 || rc == VINF_EM_RAW_INTERRUPT
4136 || rc == VERR_EM_INTERPRETER
4137 || rc == VINF_EM_RAW_EMULATE_INSTR
4138 || rc == VINF_PGM_SYNC_CR3
4139 || rc == VINF_IOM_HC_IOPORT_READ
4140 || rc == VINF_IOM_HC_IOPORT_WRITE
4141 || rc == VINF_EM_RAW_GUEST_TRAP
4142 || rc == VINF_TRPM_XCPT_DISPATCHED
4143 || rc == VINF_EM_RESCHEDULE_REM,
4144 ("rc = %d\n", VBOXSTRICTRC_VAL(rc)));
4145 break;
4146
4147 case VMX_EXIT_TPR: /* 43 TPR below threshold. Guest software executed MOV to CR8. */
4148 case VMX_EXIT_APIC_ACCESS: /* 44 APIC access. Guest software attempted to access memory at a physical address on the APIC-access page. */
4149 case VMX_EXIT_RDMSR: /* 31 RDMSR. Guest software attempted to execute RDMSR. */
4150 case VMX_EXIT_WRMSR: /* 32 WRMSR. Guest software attempted to execute WRMSR. */
4151 case VMX_EXIT_PAUSE: /* 40 Guest software attempted to execute PAUSE. */
4152 case VMX_EXIT_MONITOR: /* 39 Guest software attempted to execute MONITOR. */
4153 /* Note: If we decide to emulate them here, then we must sync the MSRs that could have been changed (sysenter, fs/gs base)!!! */
4154 rc = VERR_EM_INTERPRETER;
4155 break;
4156
4157 case VMX_EXIT_IRQ_WINDOW: /* 7 Interrupt window. */
4158 Assert(rc == VINF_EM_RAW_INTERRUPT);
4159 break;
4160
4161 case VMX_EXIT_ERR_INVALID_GUEST_STATE: /* 33 VM-entry failure due to invalid guest state. */
4162 {
4163#ifdef VBOX_STRICT
4164 RTCCUINTREG val2 = 0;
4165
4166 Log(("VMX_EXIT_ERR_INVALID_GUEST_STATE\n"));
4167
4168 VMXReadVMCS(VMX_VMCS64_GUEST_RIP, &val2);
4169 Log(("Old eip %RGv new %RGv\n", (RTGCPTR)pCtx->rip, (RTGCPTR)val2));
4170
4171 VMXReadVMCS(VMX_VMCS64_GUEST_CR0, &val2);
4172 Log(("VMX_VMCS_GUEST_CR0 %RX64\n", (uint64_t)val2));
4173
4174 VMXReadVMCS(VMX_VMCS64_GUEST_CR3, &val2);
4175 Log(("VMX_VMCS_GUEST_CR3 %RX64\n", (uint64_t)val2));
4176
4177 VMXReadVMCS(VMX_VMCS64_GUEST_CR4, &val2);
4178 Log(("VMX_VMCS_GUEST_CR4 %RX64\n", (uint64_t)val2));
4179
4180 VMXReadVMCS(VMX_VMCS_GUEST_RFLAGS, &val2);
4181 Log(("VMX_VMCS_GUEST_RFLAGS %08x\n", val2));
4182
4183 VMX_LOG_SELREG(CS, "CS", val2);
4184 VMX_LOG_SELREG(DS, "DS", val2);
4185 VMX_LOG_SELREG(ES, "ES", val2);
4186 VMX_LOG_SELREG(FS, "FS", val2);
4187 VMX_LOG_SELREG(GS, "GS", val2);
4188 VMX_LOG_SELREG(SS, "SS", val2);
4189 VMX_LOG_SELREG(TR, "TR", val2);
4190 VMX_LOG_SELREG(LDTR, "LDTR", val2);
4191
4192 VMXReadVMCS(VMX_VMCS64_GUEST_GDTR_BASE, &val2);
4193 Log(("VMX_VMCS_GUEST_GDTR_BASE %RX64\n", (uint64_t)val2));
4194 VMXReadVMCS(VMX_VMCS64_GUEST_IDTR_BASE, &val2);
4195 Log(("VMX_VMCS_GUEST_IDTR_BASE %RX64\n", (uint64_t)val2));
4196#endif /* VBOX_STRICT */
4197 rc = VERR_VMX_INVALID_GUEST_STATE;
4198 break;
4199 }
4200
4201 case VMX_EXIT_ERR_MSR_LOAD: /* 34 VM-entry failure due to MSR loading. */
4202 case VMX_EXIT_ERR_MACHINE_CHECK: /* 41 VM-entry failure due to machine-check. */
4203 default:
4204 rc = VERR_VMX_UNEXPECTED_EXIT_CODE;
4205 AssertMsgFailed(("Unexpected exit code %d\n", exitReason)); /* Can't happen. */
4206 break;
4207
4208 }
4209end:
4210
4211 /* We now going back to ring-3, so clear the action flag. */
4212 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
4213
4214 /* Signal changes for the recompiler. */
4215 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR | CPUM_CHANGED_LDTR | CPUM_CHANGED_GDTR | CPUM_CHANGED_IDTR | CPUM_CHANGED_TR | CPUM_CHANGED_HIDDEN_SEL_REGS);
4216
4217 /* If we executed vmlaunch/vmresume and an external irq was pending, then we don't have to do a full sync the next time. */
4218 if ( exitReason == VMX_EXIT_EXTERNAL_IRQ
4219 && !VMX_EXIT_INTERRUPTION_INFO_VALID(intInfo))
4220 {
4221 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatPendingHostIrq);
4222 /* On the next entry we'll only sync the host context. */
4223 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_HOST_CONTEXT;
4224 }
4225 else
4226 {
4227 /* On the next entry we'll sync everything. */
4228 /** @todo we can do better than this */
4229 /* Not in the VINF_PGM_CHANGE_MODE though! */
4230 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_ALL;
4231 }
4232
4233 /* translate into a less severe return code */
4234 if (rc == VERR_EM_INTERPRETER)
4235 rc = VINF_EM_RAW_EMULATE_INSTR;
4236 else
4237 /* Try to extract more information about what might have gone wrong here. */
4238 if (rc == VERR_VMX_INVALID_VMCS_PTR)
4239 {
4240 VMXGetActivateVMCS(&pVCpu->hwaccm.s.vmx.lasterror.u64VMCSPhys);
4241 pVCpu->hwaccm.s.vmx.lasterror.ulVMCSRevision = *(uint32_t *)pVCpu->hwaccm.s.vmx.pvVMCS;
4242 pVCpu->hwaccm.s.vmx.lasterror.idEnteredCpu = pVCpu->hwaccm.s.idEnteredCpu;
4243 pVCpu->hwaccm.s.vmx.lasterror.idCurrentCpu = RTMpCpuId();
4244 }
4245
4246 /* Just set the correct state here instead of trying to catch every goto above. */
4247 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED, VMCPUSTATE_STARTED_EXEC);
4248
4249#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
4250 /* Restore interrupts if we exitted after disabling them. */
4251 if (uOldEFlags != ~(RTCCUINTREG)0)
4252 ASMSetFlags(uOldEFlags);
4253#endif
4254
4255 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2, x);
4256 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit1, x);
4257 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatEntry, x);
4258 Log2(("X"));
4259 return VBOXSTRICTRC_TODO(rc);
4260}
4261
4262
4263/**
4264 * Enters the VT-x session
4265 *
4266 * @returns VBox status code.
4267 * @param pVM The VM to operate on.
4268 * @param pVCpu The VMCPU to operate on.
4269 * @param pCpu CPU info struct
4270 */
4271VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBLCPUINFO pCpu)
4272{
4273 Assert(pVM->hwaccm.s.vmx.fSupported);
4274
4275 unsigned cr4 = ASMGetCR4();
4276 if (!(cr4 & X86_CR4_VMXE))
4277 {
4278 AssertMsgFailed(("X86_CR4_VMXE should be set!\n"));
4279 return VERR_VMX_X86_CR4_VMXE_CLEARED;
4280 }
4281
4282 /* Activate the VM Control Structure. */
4283 int rc = VMXActivateVMCS(pVCpu->hwaccm.s.vmx.HCPhysVMCS);
4284 if (RT_FAILURE(rc))
4285 return rc;
4286
4287 pVCpu->hwaccm.s.fResumeVM = false;
4288 return VINF_SUCCESS;
4289}
4290
4291
4292/**
4293 * Leaves the VT-x session
4294 *
4295 * @returns VBox status code.
4296 * @param pVM The VM to operate on.
4297 * @param pVCpu The VMCPU to operate on.
4298 * @param pCtx CPU context
4299 */
4300VMMR0DECL(int) VMXR0Leave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4301{
4302 Assert(pVM->hwaccm.s.vmx.fSupported);
4303
4304#ifdef DEBUG
4305 if (CPUMIsHyperDebugStateActive(pVCpu))
4306 {
4307 CPUMR0LoadHostDebugState(pVM, pVCpu);
4308 Assert(pVCpu->hwaccm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT);
4309 }
4310 else
4311#endif
4312 /* Save the guest debug state if necessary. */
4313 if (CPUMIsGuestDebugStateActive(pVCpu))
4314 {
4315 CPUMR0SaveGuestDebugState(pVM, pVCpu, pCtx, true /* save DR6 */);
4316
4317 /* Enable drx move intercepts again. */
4318 pVCpu->hwaccm.s.vmx.proc_ctls |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
4319 int rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
4320 AssertRC(rc);
4321
4322 /* Resync the debug registers the next time. */
4323 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_DEBUG;
4324 }
4325 else
4326 Assert(pVCpu->hwaccm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT);
4327
4328 /* Clear VM Control Structure. Marking it inactive, clearing implementation specific data and writing back VMCS data to memory. */
4329 int rc = VMXClearVMCS(pVCpu->hwaccm.s.vmx.HCPhysVMCS);
4330 AssertRC(rc);
4331
4332 return VINF_SUCCESS;
4333}
4334
4335/**
4336 * Flush the TLB (EPT)
4337 *
4338 * @returns VBox status code.
4339 * @param pVM The VM to operate on.
4340 * @param pVCpu The VM CPU to operate on.
4341 * @param enmFlush Type of flush
4342 * @param GCPhys Physical address of the page to flush
4343 */
4344static void hmR0VmxFlushEPT(PVM pVM, PVMCPU pVCpu, VMX_FLUSH enmFlush, RTGCPHYS GCPhys)
4345{
4346 uint64_t descriptor[2];
4347
4348 LogFlow(("hmR0VmxFlushEPT %d %RGv\n", enmFlush, GCPhys));
4349 Assert(pVM->hwaccm.s.fNestedPaging);
4350 descriptor[0] = pVCpu->hwaccm.s.vmx.GCPhysEPTP;
4351 descriptor[1] = GCPhys;
4352 int rc = VMXR0InvEPT(enmFlush, &descriptor[0]);
4353 AssertRC(rc);
4354}
4355
4356#ifdef HWACCM_VTX_WITH_VPID
4357/**
4358 * Flush the TLB (EPT)
4359 *
4360 * @returns VBox status code.
4361 * @param pVM The VM to operate on.
4362 * @param pVCpu The VM CPU to operate on.
4363 * @param enmFlush Type of flush
4364 * @param GCPtr Virtual address of the page to flush
4365 */
4366static void hmR0VmxFlushVPID(PVM pVM, PVMCPU pVCpu, VMX_FLUSH enmFlush, RTGCPTR GCPtr)
4367{
4368#if HC_ARCH_BITS == 32
4369 /* If we get a flush in 64 bits guest mode, then force a full TLB flush. Invvpid probably takes only 32 bits addresses. (@todo) */
4370 if ( CPUMIsGuestInLongMode(pVCpu)
4371 && !VMX_IS_64BIT_HOST_MODE())
4372 {
4373 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
4374 }
4375 else
4376#endif
4377 {
4378 uint64_t descriptor[2];
4379
4380 Assert(pVM->hwaccm.s.vmx.fVPID);
4381 descriptor[0] = pVCpu->hwaccm.s.uCurrentASID;
4382 descriptor[1] = GCPtr;
4383 int rc = VMXR0InvVPID(enmFlush, &descriptor[0]);
4384 AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvVPID %x %x %RGv failed with %d\n", enmFlush, pVCpu->hwaccm.s.uCurrentASID, GCPtr, rc));
4385 }
4386}
4387#endif /* HWACCM_VTX_WITH_VPID */
4388
4389/**
4390 * Invalidates a guest page
4391 *
4392 * @returns VBox status code.
4393 * @param pVM The VM to operate on.
4394 * @param pVCpu The VM CPU to operate on.
4395 * @param GCVirt Page to invalidate
4396 */
4397VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
4398{
4399 bool fFlushPending = VMCPU_FF_ISSET(pVCpu, VMCPU_FF_TLB_FLUSH);
4400
4401 Log2(("VMXR0InvalidatePage %RGv\n", GCVirt));
4402
4403 /* Only relevant if we want to use VPID.
4404 * In the nested paging case we still see such calls, but
4405 * can safely ignore them. (e.g. after cr3 updates)
4406 */
4407#ifdef HWACCM_VTX_WITH_VPID
4408 /* Skip it if a TLB flush is already pending. */
4409 if ( !fFlushPending
4410 && pVM->hwaccm.s.vmx.fVPID)
4411 hmR0VmxFlushVPID(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushPage, GCVirt);
4412#endif /* HWACCM_VTX_WITH_VPID */
4413
4414 return VINF_SUCCESS;
4415}
4416
4417/**
4418 * Invalidates a guest page by physical address
4419 *
4420 * NOTE: Assumes the current instruction references this physical page though a virtual address!!
4421 *
4422 * @returns VBox status code.
4423 * @param pVM The VM to operate on.
4424 * @param pVCpu The VM CPU to operate on.
4425 * @param GCPhys Page to invalidate
4426 */
4427VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
4428{
4429 bool fFlushPending = VMCPU_FF_ISSET(pVCpu, VMCPU_FF_TLB_FLUSH);
4430
4431 Assert(pVM->hwaccm.s.fNestedPaging);
4432
4433 LogFlow(("VMXR0InvalidatePhysPage %RGp\n", GCPhys));
4434
4435 /* Skip it if a TLB flush is already pending. */
4436 if (!fFlushPending)
4437 hmR0VmxFlushEPT(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushPage, GCPhys);
4438
4439 return VINF_SUCCESS;
4440}
4441
4442/**
4443 * Report world switch error and dump some useful debug info
4444 *
4445 * @param pVM The VM to operate on.
4446 * @param pVCpu The VMCPU to operate on.
4447 * @param rc Return code
4448 * @param pCtx Current CPU context (not updated)
4449 */
4450static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, VBOXSTRICTRC rc, PCPUMCTX pCtx)
4451{
4452 switch (VBOXSTRICTRC_VAL(rc))
4453 {
4454 case VERR_VMX_INVALID_VMXON_PTR:
4455 AssertFailed();
4456 break;
4457
4458 case VERR_VMX_UNABLE_TO_START_VM:
4459 case VERR_VMX_UNABLE_TO_RESUME_VM:
4460 {
4461 int rc2;
4462 RTCCUINTREG exitReason, instrError;
4463
4464 rc2 = VMXReadVMCS(VMX_VMCS32_RO_EXIT_REASON, &exitReason);
4465 rc2 |= VMXReadVMCS(VMX_VMCS32_RO_VM_INSTR_ERROR, &instrError);
4466 AssertRC(rc2);
4467 if (rc2 == VINF_SUCCESS)
4468 {
4469 Log(("Unable to start/resume VM for reason: %x. Instruction error %x\n", (uint32_t)exitReason, (uint32_t)instrError));
4470 Log(("Current stack %08x\n", &rc2));
4471
4472 pVCpu->hwaccm.s.vmx.lasterror.ulInstrError = instrError;
4473 pVCpu->hwaccm.s.vmx.lasterror.ulExitReason = exitReason;
4474
4475#ifdef VBOX_STRICT
4476 RTGDTR gdtr;
4477 PCX86DESCHC pDesc;
4478 RTCCUINTREG val;
4479
4480 ASMGetGDTR(&gdtr);
4481
4482 VMXReadVMCS(VMX_VMCS64_GUEST_RIP, &val);
4483 Log(("Old eip %RGv new %RGv\n", (RTGCPTR)pCtx->rip, (RTGCPTR)val));
4484 VMXReadVMCS(VMX_VMCS_CTRL_PIN_EXEC_CONTROLS, &val);
4485 Log(("VMX_VMCS_CTRL_PIN_EXEC_CONTROLS %08x\n", val));
4486 VMXReadVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, &val);
4487 Log(("VMX_VMCS_CTRL_PROC_EXEC_CONTROLS %08x\n", val));
4488 VMXReadVMCS(VMX_VMCS_CTRL_ENTRY_CONTROLS, &val);
4489 Log(("VMX_VMCS_CTRL_ENTRY_CONTROLS %08x\n", val));
4490 VMXReadVMCS(VMX_VMCS_CTRL_EXIT_CONTROLS, &val);
4491 Log(("VMX_VMCS_CTRL_EXIT_CONTROLS %08x\n", val));
4492
4493 VMXReadVMCS(VMX_VMCS_HOST_CR0, &val);
4494 Log(("VMX_VMCS_HOST_CR0 %08x\n", val));
4495
4496 VMXReadVMCS(VMX_VMCS_HOST_CR3, &val);
4497 Log(("VMX_VMCS_HOST_CR3 %08x\n", val));
4498
4499 VMXReadVMCS(VMX_VMCS_HOST_CR4, &val);
4500 Log(("VMX_VMCS_HOST_CR4 %08x\n", val));
4501
4502 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_CS, &val);
4503 Log(("VMX_VMCS_HOST_FIELD_CS %08x\n", val));
4504
4505 VMXReadVMCS(VMX_VMCS_GUEST_RFLAGS, &val);
4506 Log(("VMX_VMCS_GUEST_RFLAGS %08x\n", val));
4507
4508 if (val < gdtr.cbGdt)
4509 {
4510 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
4511 HWACCMR0DumpDescriptor(pDesc, val, "CS: ");
4512 }
4513
4514 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_DS, &val);
4515 Log(("VMX_VMCS_HOST_FIELD_DS %08x\n", val));
4516 if (val < gdtr.cbGdt)
4517 {
4518 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
4519 HWACCMR0DumpDescriptor(pDesc, val, "DS: ");
4520 }
4521
4522 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_ES, &val);
4523 Log(("VMX_VMCS_HOST_FIELD_ES %08x\n", val));
4524 if (val < gdtr.cbGdt)
4525 {
4526 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
4527 HWACCMR0DumpDescriptor(pDesc, val, "ES: ");
4528 }
4529
4530 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_FS, &val);
4531 Log(("VMX_VMCS16_HOST_FIELD_FS %08x\n", val));
4532 if (val < gdtr.cbGdt)
4533 {
4534 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
4535 HWACCMR0DumpDescriptor(pDesc, val, "FS: ");
4536 }
4537
4538 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_GS, &val);
4539 Log(("VMX_VMCS16_HOST_FIELD_GS %08x\n", val));
4540 if (val < gdtr.cbGdt)
4541 {
4542 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
4543 HWACCMR0DumpDescriptor(pDesc, val, "GS: ");
4544 }
4545
4546 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_SS, &val);
4547 Log(("VMX_VMCS16_HOST_FIELD_SS %08x\n", val));
4548 if (val < gdtr.cbGdt)
4549 {
4550 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
4551 HWACCMR0DumpDescriptor(pDesc, val, "SS: ");
4552 }
4553
4554 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_TR, &val);
4555 Log(("VMX_VMCS16_HOST_FIELD_TR %08x\n", val));
4556 if (val < gdtr.cbGdt)
4557 {
4558 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
4559 HWACCMR0DumpDescriptor(pDesc, val, "TR: ");
4560 }
4561
4562 VMXReadVMCS(VMX_VMCS_HOST_TR_BASE, &val);
4563 Log(("VMX_VMCS_HOST_TR_BASE %RHv\n", val));
4564
4565 VMXReadVMCS(VMX_VMCS_HOST_GDTR_BASE, &val);
4566 Log(("VMX_VMCS_HOST_GDTR_BASE %RHv\n", val));
4567 VMXReadVMCS(VMX_VMCS_HOST_IDTR_BASE, &val);
4568 Log(("VMX_VMCS_HOST_IDTR_BASE %RHv\n", val));
4569
4570 VMXReadVMCS(VMX_VMCS32_HOST_SYSENTER_CS, &val);
4571 Log(("VMX_VMCS_HOST_SYSENTER_CS %08x\n", val));
4572
4573 VMXReadVMCS(VMX_VMCS_HOST_SYSENTER_EIP, &val);
4574 Log(("VMX_VMCS_HOST_SYSENTER_EIP %RHv\n", val));
4575
4576 VMXReadVMCS(VMX_VMCS_HOST_SYSENTER_ESP, &val);
4577 Log(("VMX_VMCS_HOST_SYSENTER_ESP %RHv\n", val));
4578
4579 VMXReadVMCS(VMX_VMCS_HOST_RSP, &val);
4580 Log(("VMX_VMCS_HOST_RSP %RHv\n", val));
4581 VMXReadVMCS(VMX_VMCS_HOST_RIP, &val);
4582 Log(("VMX_VMCS_HOST_RIP %RHv\n", val));
4583
4584# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4585 if (VMX_IS_64BIT_HOST_MODE())
4586 {
4587 Log(("MSR_K6_EFER = %RX64\n", ASMRdMsr(MSR_K6_EFER)));
4588 Log(("MSR_K6_STAR = %RX64\n", ASMRdMsr(MSR_K6_STAR)));
4589 Log(("MSR_K8_LSTAR = %RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
4590 Log(("MSR_K8_CSTAR = %RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
4591 Log(("MSR_K8_SF_MASK = %RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
4592 }
4593# endif
4594#endif /* VBOX_STRICT */
4595 }
4596 break;
4597 }
4598
4599 default:
4600 /* impossible */
4601 AssertMsgFailed(("%Rrc (%#x)\n", VBOXSTRICTRC_VAL(rc), VBOXSTRICTRC_VAL(rc)));
4602 break;
4603 }
4604}
4605
4606#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4607/**
4608 * Prepares for and executes VMLAUNCH (64 bits guest mode)
4609 *
4610 * @returns VBox status code
4611 * @param fResume vmlauch/vmresume
4612 * @param pCtx Guest context
4613 * @param pCache VMCS cache
4614 * @param pVM The VM to operate on.
4615 * @param pVCpu The VMCPU to operate on.
4616 */
4617DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
4618{
4619 uint32_t aParam[6];
4620 PHMGLOBLCPUINFO pCpu;
4621 RTHCPHYS HCPhysCpuPage;
4622 int rc;
4623
4624 pCpu = HWACCMR0GetCurrentCpu();
4625 HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4626
4627#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4628 pCache->uPos = 1;
4629 pCache->interPD = PGMGetInterPaeCR3(pVM);
4630 pCache->pSwitcher = (uint64_t)pVM->hwaccm.s.pfnHost32ToGuest64R0;
4631#endif
4632
4633#ifdef DEBUG
4634 pCache->TestIn.HCPhysCpuPage= 0;
4635 pCache->TestIn.HCPhysVMCS = 0;
4636 pCache->TestIn.pCache = 0;
4637 pCache->TestOut.HCPhysVMCS = 0;
4638 pCache->TestOut.pCache = 0;
4639 pCache->TestOut.pCtx = 0;
4640 pCache->TestOut.eflags = 0;
4641#endif
4642
4643 aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
4644 aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
4645 aParam[2] = (uint32_t)(pVCpu->hwaccm.s.vmx.HCPhysVMCS); /* Param 2: VMCS physical address - Lo. */
4646 aParam[3] = (uint32_t)(pVCpu->hwaccm.s.vmx.HCPhysVMCS >> 32); /* Param 2: VMCS physical address - Hi. */
4647 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hwaccm.s.vmx.VMCSCache);
4648 aParam[5] = 0;
4649
4650#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4651 pCtx->dr[4] = pVM->hwaccm.s.vmx.pScratchPhys + 16 + 8;
4652 *(uint32_t *)(pVM->hwaccm.s.vmx.pScratch + 16 + 8) = 1;
4653#endif
4654 rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, pVM->hwaccm.s.pfnVMXGCStartVM64, 6, &aParam[0]);
4655
4656#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4657 Assert(*(uint32_t *)(pVM->hwaccm.s.vmx.pScratch + 16 + 8) == 5);
4658 Assert(pCtx->dr[4] == 10);
4659 *(uint32_t *)(pVM->hwaccm.s.vmx.pScratch + 16 + 8) = 0xff;
4660#endif
4661
4662#ifdef DEBUG
4663 AssertMsg(pCache->TestIn.HCPhysCpuPage== HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
4664 AssertMsg(pCache->TestIn.HCPhysVMCS == pVCpu->hwaccm.s.vmx.HCPhysVMCS, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVMCS, pVCpu->hwaccm.s.vmx.HCPhysVMCS));
4665 AssertMsg(pCache->TestIn.HCPhysVMCS == pCache->TestOut.HCPhysVMCS, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVMCS, pCache->TestOut.HCPhysVMCS));
4666 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache, pCache->TestOut.pCache));
4667 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hwaccm.s.vmx.VMCSCache), ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hwaccm.s.vmx.VMCSCache)));
4668 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx, pCache->TestOut.pCtx));
4669 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
4670#endif
4671 return rc;
4672}
4673
4674/**
4675 * Executes the specified handler in 64 mode
4676 *
4677 * @returns VBox status code.
4678 * @param pVM The VM to operate on.
4679 * @param pVCpu The VMCPU to operate on.
4680 * @param pCtx Guest context
4681 * @param pfnHandler RC handler
4682 * @param cbParam Number of parameters
4683 * @param paParam Array of 32 bits parameters
4684 */
4685VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, RTRCPTR pfnHandler, uint32_t cbParam, uint32_t *paParam)
4686{
4687 int rc, rc2;
4688 PHMGLOBLCPUINFO pCpu;
4689 RTHCPHYS HCPhysCpuPage;
4690 RTHCUINTREG uOldEFlags;
4691
4692 AssertReturn(pVM->hwaccm.s.pfnHost32ToGuest64R0, VERR_INTERNAL_ERROR);
4693 Assert(pfnHandler);
4694 Assert(pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hwaccm.s.vmx.VMCSCache.Write.aField));
4695 Assert(pVCpu->hwaccm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hwaccm.s.vmx.VMCSCache.Read.aField));
4696
4697#ifdef VBOX_STRICT
4698 for (unsigned i=0;i<pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries;i++)
4699 Assert(hmR0VmxIsValidWriteField(pVCpu->hwaccm.s.vmx.VMCSCache.Write.aField[i]));
4700
4701 for (unsigned i=0;i<pVCpu->hwaccm.s.vmx.VMCSCache.Read.cValidEntries;i++)
4702 Assert(hmR0VmxIsValidReadField(pVCpu->hwaccm.s.vmx.VMCSCache.Read.aField[i]));
4703#endif
4704
4705 /* Disable interrupts. */
4706 uOldEFlags = ASMIntDisableFlags();
4707
4708 pCpu = HWACCMR0GetCurrentCpu();
4709 HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4710
4711 /* Clear VM Control Structure. Marking it inactive, clearing implementation specific data and writing back VMCS data to memory. */
4712 VMXClearVMCS(pVCpu->hwaccm.s.vmx.HCPhysVMCS);
4713
4714 /* Leave VMX Root Mode. */
4715 VMXDisable();
4716
4717 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4718
4719 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
4720 CPUMSetHyperEIP(pVCpu, pfnHandler);
4721 for (int i=(int)cbParam-1;i>=0;i--)
4722 CPUMPushHyper(pVCpu, paParam[i]);
4723
4724 STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatWorldSwitch3264, z);
4725 /* Call switcher. */
4726 rc = pVM->hwaccm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
4727 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatWorldSwitch3264, z);
4728
4729 /* Make sure the VMX instructions don't cause #UD faults. */
4730 ASMSetCR4(ASMGetCR4() | X86_CR4_VMXE);
4731
4732 /* Enter VMX Root Mode */
4733 rc2 = VMXEnable(HCPhysCpuPage);
4734 if (RT_FAILURE(rc2))
4735 {
4736 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4737 ASMSetFlags(uOldEFlags);
4738 return VERR_VMX_VMXON_FAILED;
4739 }
4740
4741 rc2 = VMXActivateVMCS(pVCpu->hwaccm.s.vmx.HCPhysVMCS);
4742 AssertRC(rc2);
4743 Assert(!(ASMGetFlags() & X86_EFL_IF));
4744 ASMSetFlags(uOldEFlags);
4745 return rc;
4746}
4747
4748#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
4749
4750
4751#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
4752/**
4753 * Executes VMWRITE
4754 *
4755 * @returns VBox status code
4756 * @param pVCpu The VMCPU to operate on.
4757 * @param idxField VMCS index
4758 * @param u64Val 16, 32 or 64 bits value
4759 */
4760VMMR0DECL(int) VMXWriteVMCS64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4761{
4762 int rc;
4763
4764 switch (idxField)
4765 {
4766 case VMX_VMCS_CTRL_TSC_OFFSET_FULL:
4767 case VMX_VMCS_CTRL_IO_BITMAP_A_FULL:
4768 case VMX_VMCS_CTRL_IO_BITMAP_B_FULL:
4769 case VMX_VMCS_CTRL_MSR_BITMAP_FULL:
4770 case VMX_VMCS_CTRL_VMEXIT_MSR_STORE_FULL:
4771 case VMX_VMCS_CTRL_VMEXIT_MSR_LOAD_FULL:
4772 case VMX_VMCS_CTRL_VMENTRY_MSR_LOAD_FULL:
4773 case VMX_VMCS_CTRL_VAPIC_PAGEADDR_FULL:
4774 case VMX_VMCS_CTRL_APIC_ACCESSADDR_FULL:
4775 case VMX_VMCS_GUEST_LINK_PTR_FULL:
4776 case VMX_VMCS_GUEST_PDPTR0_FULL:
4777 case VMX_VMCS_GUEST_PDPTR1_FULL:
4778 case VMX_VMCS_GUEST_PDPTR2_FULL:
4779 case VMX_VMCS_GUEST_PDPTR3_FULL:
4780 case VMX_VMCS_GUEST_DEBUGCTL_FULL:
4781 case VMX_VMCS_GUEST_EFER_FULL:
4782 case VMX_VMCS_CTRL_EPTP_FULL:
4783 /* These fields consist of two parts, which are both writable in 32 bits mode. */
4784 rc = VMXWriteVMCS32(idxField, u64Val);
4785 rc |= VMXWriteVMCS32(idxField + 1, (uint32_t)(u64Val >> 32ULL));
4786 AssertRC(rc);
4787 return rc;
4788
4789 case VMX_VMCS64_GUEST_LDTR_BASE:
4790 case VMX_VMCS64_GUEST_TR_BASE:
4791 case VMX_VMCS64_GUEST_GDTR_BASE:
4792 case VMX_VMCS64_GUEST_IDTR_BASE:
4793 case VMX_VMCS64_GUEST_SYSENTER_EIP:
4794 case VMX_VMCS64_GUEST_SYSENTER_ESP:
4795 case VMX_VMCS64_GUEST_CR0:
4796 case VMX_VMCS64_GUEST_CR4:
4797 case VMX_VMCS64_GUEST_CR3:
4798 case VMX_VMCS64_GUEST_DR7:
4799 case VMX_VMCS64_GUEST_RIP:
4800 case VMX_VMCS64_GUEST_RSP:
4801 case VMX_VMCS64_GUEST_CS_BASE:
4802 case VMX_VMCS64_GUEST_DS_BASE:
4803 case VMX_VMCS64_GUEST_ES_BASE:
4804 case VMX_VMCS64_GUEST_FS_BASE:
4805 case VMX_VMCS64_GUEST_GS_BASE:
4806 case VMX_VMCS64_GUEST_SS_BASE:
4807 /* Queue a 64 bits value as we can't set it in 32 bits host mode. */
4808 if (u64Val >> 32ULL)
4809 rc = VMXWriteCachedVMCSEx(pVCpu, idxField, u64Val);
4810 else
4811 rc = VMXWriteVMCS32(idxField, (uint32_t)u64Val);
4812
4813 return rc;
4814
4815 default:
4816 AssertMsgFailed(("Unexpected field %x\n", idxField));
4817 return VERR_INVALID_PARAMETER;
4818 }
4819}
4820
4821/**
4822 * Cache VMCS writes for performance reasons (Darwin) and for running 64 bits guests on 32 bits hosts.
4823 *
4824 * @param pVCpu The VMCPU to operate on.
4825 * @param idxField VMCS field
4826 * @param u64Val Value
4827 */
4828VMMR0DECL(int) VMXWriteCachedVMCSEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4829{
4830 PVMCSCACHE pCache = &pVCpu->hwaccm.s.vmx.VMCSCache;
4831
4832 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1, ("entries=%x\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
4833
4834 /* Make sure there are no duplicates. */
4835 for (unsigned i=0;i<pCache->Write.cValidEntries;i++)
4836 {
4837 if (pCache->Write.aField[i] == idxField)
4838 {
4839 pCache->Write.aFieldVal[i] = u64Val;
4840 return VINF_SUCCESS;
4841 }
4842 }
4843
4844 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
4845 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
4846 pCache->Write.cValidEntries++;
4847 return VINF_SUCCESS;
4848}
4849
4850#endif /* HC_ARCH_BITS == 32 && !VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0 */
4851
4852#ifdef VBOX_STRICT
4853static bool hmR0VmxIsValidReadField(uint32_t idxField)
4854{
4855 switch(idxField)
4856 {
4857 case VMX_VMCS64_GUEST_RIP:
4858 case VMX_VMCS64_GUEST_RSP:
4859 case VMX_VMCS_GUEST_RFLAGS:
4860 case VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE:
4861 case VMX_VMCS_CTRL_CR0_READ_SHADOW:
4862 case VMX_VMCS64_GUEST_CR0:
4863 case VMX_VMCS_CTRL_CR4_READ_SHADOW:
4864 case VMX_VMCS64_GUEST_CR4:
4865 case VMX_VMCS64_GUEST_DR7:
4866 case VMX_VMCS32_GUEST_SYSENTER_CS:
4867 case VMX_VMCS64_GUEST_SYSENTER_EIP:
4868 case VMX_VMCS64_GUEST_SYSENTER_ESP:
4869 case VMX_VMCS32_GUEST_GDTR_LIMIT:
4870 case VMX_VMCS64_GUEST_GDTR_BASE:
4871 case VMX_VMCS32_GUEST_IDTR_LIMIT:
4872 case VMX_VMCS64_GUEST_IDTR_BASE:
4873 case VMX_VMCS16_GUEST_FIELD_CS:
4874 case VMX_VMCS32_GUEST_CS_LIMIT:
4875 case VMX_VMCS64_GUEST_CS_BASE:
4876 case VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS:
4877 case VMX_VMCS16_GUEST_FIELD_DS:
4878 case VMX_VMCS32_GUEST_DS_LIMIT:
4879 case VMX_VMCS64_GUEST_DS_BASE:
4880 case VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS:
4881 case VMX_VMCS16_GUEST_FIELD_ES:
4882 case VMX_VMCS32_GUEST_ES_LIMIT:
4883 case VMX_VMCS64_GUEST_ES_BASE:
4884 case VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS:
4885 case VMX_VMCS16_GUEST_FIELD_FS:
4886 case VMX_VMCS32_GUEST_FS_LIMIT:
4887 case VMX_VMCS64_GUEST_FS_BASE:
4888 case VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS:
4889 case VMX_VMCS16_GUEST_FIELD_GS:
4890 case VMX_VMCS32_GUEST_GS_LIMIT:
4891 case VMX_VMCS64_GUEST_GS_BASE:
4892 case VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS:
4893 case VMX_VMCS16_GUEST_FIELD_SS:
4894 case VMX_VMCS32_GUEST_SS_LIMIT:
4895 case VMX_VMCS64_GUEST_SS_BASE:
4896 case VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS:
4897 case VMX_VMCS16_GUEST_FIELD_LDTR:
4898 case VMX_VMCS32_GUEST_LDTR_LIMIT:
4899 case VMX_VMCS64_GUEST_LDTR_BASE:
4900 case VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS:
4901 case VMX_VMCS16_GUEST_FIELD_TR:
4902 case VMX_VMCS32_GUEST_TR_LIMIT:
4903 case VMX_VMCS64_GUEST_TR_BASE:
4904 case VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS:
4905 case VMX_VMCS32_RO_EXIT_REASON:
4906 case VMX_VMCS32_RO_VM_INSTR_ERROR:
4907 case VMX_VMCS32_RO_EXIT_INSTR_LENGTH:
4908 case VMX_VMCS32_RO_EXIT_INTERRUPTION_ERRCODE:
4909 case VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO:
4910 case VMX_VMCS32_RO_EXIT_INSTR_INFO:
4911 case VMX_VMCS_RO_EXIT_QUALIFICATION:
4912 case VMX_VMCS32_RO_IDT_INFO:
4913 case VMX_VMCS32_RO_IDT_ERRCODE:
4914 case VMX_VMCS64_GUEST_CR3:
4915 case VMX_VMCS_EXIT_PHYS_ADDR_FULL:
4916 return true;
4917 }
4918 return false;
4919}
4920
4921static bool hmR0VmxIsValidWriteField(uint32_t idxField)
4922{
4923 switch(idxField)
4924 {
4925 case VMX_VMCS64_GUEST_LDTR_BASE:
4926 case VMX_VMCS64_GUEST_TR_BASE:
4927 case VMX_VMCS64_GUEST_GDTR_BASE:
4928 case VMX_VMCS64_GUEST_IDTR_BASE:
4929 case VMX_VMCS64_GUEST_SYSENTER_EIP:
4930 case VMX_VMCS64_GUEST_SYSENTER_ESP:
4931 case VMX_VMCS64_GUEST_CR0:
4932 case VMX_VMCS64_GUEST_CR4:
4933 case VMX_VMCS64_GUEST_CR3:
4934 case VMX_VMCS64_GUEST_DR7:
4935 case VMX_VMCS64_GUEST_RIP:
4936 case VMX_VMCS64_GUEST_RSP:
4937 case VMX_VMCS64_GUEST_CS_BASE:
4938 case VMX_VMCS64_GUEST_DS_BASE:
4939 case VMX_VMCS64_GUEST_ES_BASE:
4940 case VMX_VMCS64_GUEST_FS_BASE:
4941 case VMX_VMCS64_GUEST_GS_BASE:
4942 case VMX_VMCS64_GUEST_SS_BASE:
4943 return true;
4944 }
4945 return false;
4946}
4947
4948#endif
4949
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette