VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HWVMXR0.cpp@ 28853

Last change on this file since 28853 was 28800, checked in by vboxsync, 15 years ago

Automated rebranding to Oracle copyright/license strings via filemuncher
  • Property svn:eol-style set to native
  • Property svn:keywords set to Id
File size: 194.6 KB
Line 
1/* $Id: HWVMXR0.cpp 28800 2010-04-27 08:22:32Z vboxsync $ */
2/** @file
3 * HWACCM VMX - Host Context Ring 0.
4 */
5
6/*
7 * Copyright (C) 2006-2007 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18
19/*******************************************************************************
20* Header Files *
21*******************************************************************************/
22#define LOG_GROUP LOG_GROUP_HWACCM
23#include <VBox/hwaccm.h>
24#include <VBox/pgm.h>
25#include <VBox/dbgf.h>
26#include <VBox/selm.h>
27#include <VBox/iom.h>
28#include <VBox/rem.h>
29#include <VBox/tm.h>
30#include "HWACCMInternal.h"
31#include <VBox/vm.h>
32#include <VBox/x86.h>
33#include <VBox/pdmapi.h>
34#include <VBox/err.h>
35#include <VBox/log.h>
36#include <iprt/asm.h>
37#include <iprt/assert.h>
38#include <iprt/param.h>
39#include <iprt/string.h>
40#include <iprt/time.h>
41#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
42# include <iprt/thread.h>
43#endif
44#include "HWVMXR0.h"
45
46/*******************************************************************************
47* Defined Constants And Macros *
48*******************************************************************************/
49#if defined(RT_ARCH_AMD64)
50# define VMX_IS_64BIT_HOST_MODE() (true)
51#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
52# define VMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
53#else
54# define VMX_IS_64BIT_HOST_MODE() (false)
55#endif
56
57/*******************************************************************************
58* Global Variables *
59*******************************************************************************/
60/* IO operation lookup arrays. */
61static uint32_t const g_aIOSize[4] = {1, 2, 0, 4};
62static uint32_t const g_aIOOpAnd[4] = {0xff, 0xffff, 0, 0xffffffff};
63
64#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
65/** See HWACCMR0A.asm. */
66extern "C" uint32_t g_fVMXIs64bitHost;
67#endif
68
69/*******************************************************************************
70* Local Functions *
71*******************************************************************************/
72static void VMXR0ReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rc, PCPUMCTX pCtx);
73static void vmxR0SetupTLBEPT(PVM pVM, PVMCPU pVCpu);
74static void vmxR0SetupTLBVPID(PVM pVM, PVMCPU pVCpu);
75static void vmxR0SetupTLBDummy(PVM pVM, PVMCPU pVCpu);
76static void vmxR0FlushEPT(PVM pVM, PVMCPU pVCpu, VMX_FLUSH enmFlush, RTGCPHYS GCPhys);
77static void vmxR0FlushVPID(PVM pVM, PVMCPU pVCpu, VMX_FLUSH enmFlush, RTGCPTR GCPtr);
78static void vmxR0UpdateExceptionBitmap(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
79#ifdef VBOX_STRICT
80static bool vmxR0IsValidReadField(uint32_t idxField);
81static bool vmxR0IsValidWriteField(uint32_t idxField);
82#endif
83static void vmxR0SetMSRPermission(PVMCPU pVCpu, unsigned ulMSR, bool fRead, bool fWrite);
84
85static void VMXR0CheckError(PVM pVM, PVMCPU pVCpu, int rc)
86{
87 if (rc == VERR_VMX_GENERIC)
88 {
89 RTCCUINTREG instrError;
90
91 VMXReadVMCS(VMX_VMCS32_RO_VM_INSTR_ERROR, &instrError);
92 pVCpu->hwaccm.s.vmx.lasterror.ulInstrError = instrError;
93 }
94 pVM->hwaccm.s.lLastError = rc;
95}
96
97/**
98 * Sets up and activates VT-x on the current CPU
99 *
100 * @returns VBox status code.
101 * @param pCpu CPU info struct
102 * @param pVM The VM to operate on. (can be NULL after a resume!!)
103 * @param pvPageCpu Pointer to the global cpu page
104 * @param pPageCpuPhys Physical address of the global cpu page
105 */
106VMMR0DECL(int) VMXR0EnableCpu(PHWACCM_CPUINFO pCpu, PVM pVM, void *pvPageCpu, RTHCPHYS pPageCpuPhys)
107{
108 AssertReturn(pPageCpuPhys, VERR_INVALID_PARAMETER);
109 AssertReturn(pvPageCpu, VERR_INVALID_PARAMETER);
110
111 if (pVM)
112 {
113 /* Set revision dword at the beginning of the VMXON structure. */
114 *(uint32_t *)pvPageCpu = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hwaccm.s.vmx.msr.vmx_basic_info);
115 }
116
117 /** @todo we should unmap the two pages from the virtual address space in order to prevent accidental corruption.
118 * (which can have very bad consequences!!!)
119 */
120
121 if (ASMGetCR4() & X86_CR4_VMXE)
122 return VERR_VMX_IN_VMX_ROOT_MODE;
123
124 /* Make sure the VMX instructions don't cause #UD faults. */
125 ASMSetCR4(ASMGetCR4() | X86_CR4_VMXE);
126
127 /* Enter VMX Root Mode */
128 int rc = VMXEnable(pPageCpuPhys);
129 if (RT_FAILURE(rc))
130 {
131 if (pVM)
132 VMXR0CheckError(pVM, &pVM->aCpus[0], rc);
133 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
134 return VERR_VMX_VMXON_FAILED;
135 }
136 return VINF_SUCCESS;
137}
138
139/**
140 * Deactivates VT-x on the current CPU
141 *
142 * @returns VBox status code.
143 * @param pCpu CPU info struct
144 * @param pvPageCpu Pointer to the global cpu page
145 * @param pPageCpuPhys Physical address of the global cpu page
146 */
147VMMR0DECL(int) VMXR0DisableCpu(PHWACCM_CPUINFO pCpu, void *pvPageCpu, RTHCPHYS pPageCpuPhys)
148{
149 AssertReturn(pPageCpuPhys, VERR_INVALID_PARAMETER);
150 AssertReturn(pvPageCpu, VERR_INVALID_PARAMETER);
151
152 /* Leave VMX Root Mode. */
153 VMXDisable();
154
155 /* And clear the X86_CR4_VMXE bit */
156 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
157 return VINF_SUCCESS;
158}
159
160/**
161 * Does Ring-0 per VM VT-x init.
162 *
163 * @returns VBox status code.
164 * @param pVM The VM to operate on.
165 */
166VMMR0DECL(int) VMXR0InitVM(PVM pVM)
167{
168 int rc;
169
170#ifdef LOG_ENABLED
171 SUPR0Printf("VMXR0InitVM %x\n", pVM);
172#endif
173
174 pVM->hwaccm.s.vmx.pMemObjAPIC = NIL_RTR0MEMOBJ;
175
176 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
177 {
178 /* Allocate one page for the APIC physical page (serves for filtering accesses). */
179 rc = RTR0MemObjAllocCont(&pVM->hwaccm.s.vmx.pMemObjAPIC, 1 << PAGE_SHIFT, true /* executable R0 mapping */);
180 AssertRC(rc);
181 if (RT_FAILURE(rc))
182 return rc;
183
184 pVM->hwaccm.s.vmx.pAPIC = (uint8_t *)RTR0MemObjAddress(pVM->hwaccm.s.vmx.pMemObjAPIC);
185 pVM->hwaccm.s.vmx.pAPICPhys = RTR0MemObjGetPagePhysAddr(pVM->hwaccm.s.vmx.pMemObjAPIC, 0);
186 ASMMemZero32(pVM->hwaccm.s.vmx.pAPIC, PAGE_SIZE);
187 }
188 else
189 {
190 pVM->hwaccm.s.vmx.pMemObjAPIC = 0;
191 pVM->hwaccm.s.vmx.pAPIC = 0;
192 pVM->hwaccm.s.vmx.pAPICPhys = 0;
193 }
194
195#ifdef VBOX_WITH_CRASHDUMP_MAGIC
196 {
197 rc = RTR0MemObjAllocCont(&pVM->hwaccm.s.vmx.pMemObjScratch, 1 << PAGE_SHIFT, true /* executable R0 mapping */);
198 AssertRC(rc);
199 if (RT_FAILURE(rc))
200 return rc;
201
202 pVM->hwaccm.s.vmx.pScratch = (uint8_t *)RTR0MemObjAddress(pVM->hwaccm.s.vmx.pMemObjScratch);
203 pVM->hwaccm.s.vmx.pScratchPhys = RTR0MemObjGetPagePhysAddr(pVM->hwaccm.s.vmx.pMemObjScratch, 0);
204
205 ASMMemZero32(pVM->hwaccm.s.vmx.pScratch, PAGE_SIZE);
206 strcpy((char *)pVM->hwaccm.s.vmx.pScratch, "SCRATCH Magic");
207 *(uint64_t *)(pVM->hwaccm.s.vmx.pScratch + 16) = UINT64_C(0xDEADBEEFDEADBEEF);
208 }
209#endif
210
211 /* Allocate VMCBs for all guest CPUs. */
212 for (VMCPUID i = 0; i < pVM->cCpus; i++)
213 {
214 PVMCPU pVCpu = &pVM->aCpus[i];
215
216 pVCpu->hwaccm.s.vmx.pMemObjVMCS = NIL_RTR0MEMOBJ;
217
218 /* Allocate one page for the VM control structure (VMCS). */
219 rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.pMemObjVMCS, 1 << PAGE_SHIFT, true /* executable R0 mapping */);
220 AssertRC(rc);
221 if (RT_FAILURE(rc))
222 return rc;
223
224 pVCpu->hwaccm.s.vmx.pVMCS = RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.pMemObjVMCS);
225 pVCpu->hwaccm.s.vmx.pVMCSPhys = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.pMemObjVMCS, 0);
226 ASMMemZero32(pVCpu->hwaccm.s.vmx.pVMCS, PAGE_SIZE);
227
228 pVCpu->hwaccm.s.vmx.cr0_mask = 0;
229 pVCpu->hwaccm.s.vmx.cr4_mask = 0;
230
231 /* Allocate one page for the virtual APIC page for TPR caching. */
232 rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.pMemObjVAPIC, 1 << PAGE_SHIFT, true /* executable R0 mapping */);
233 AssertRC(rc);
234 if (RT_FAILURE(rc))
235 return rc;
236
237 pVCpu->hwaccm.s.vmx.pVAPIC = (uint8_t *)RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.pMemObjVAPIC);
238 pVCpu->hwaccm.s.vmx.pVAPICPhys = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.pMemObjVAPIC, 0);
239 ASMMemZero32(pVCpu->hwaccm.s.vmx.pVAPIC, PAGE_SIZE);
240
241 /* Allocate the MSR bitmap if this feature is supported. */
242 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
243 {
244 rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap, 1 << PAGE_SHIFT, true /* executable R0 mapping */);
245 AssertRC(rc);
246 if (RT_FAILURE(rc))
247 return rc;
248
249 pVCpu->hwaccm.s.vmx.pMSRBitmap = (uint8_t *)RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap);
250 pVCpu->hwaccm.s.vmx.pMSRBitmapPhys = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap, 0);
251 memset(pVCpu->hwaccm.s.vmx.pMSRBitmap, 0xff, PAGE_SIZE);
252 }
253
254#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
255 /* Allocate one page for the guest MSR load area (for preloading guest MSRs during the world switch). */
256 rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.pMemObjGuestMSR, 1 << PAGE_SHIFT, true /* executable R0 mapping */);
257 AssertRC(rc);
258 if (RT_FAILURE(rc))
259 return rc;
260
261 pVCpu->hwaccm.s.vmx.pGuestMSR = (uint8_t *)RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.pMemObjGuestMSR);
262 pVCpu->hwaccm.s.vmx.pGuestMSRPhys = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.pMemObjGuestMSR, 0);
263 memset(pVCpu->hwaccm.s.vmx.pGuestMSR, 0, PAGE_SIZE);
264
265 /* Allocate one page for the host MSR load area (for restoring host MSRs after the world switch back). */
266 rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.pMemObjHostMSR, 1 << PAGE_SHIFT, true /* executable R0 mapping */);
267 AssertRC(rc);
268 if (RT_FAILURE(rc))
269 return rc;
270
271 pVCpu->hwaccm.s.vmx.pHostMSR = (uint8_t *)RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.pMemObjHostMSR);
272 pVCpu->hwaccm.s.vmx.pHostMSRPhys = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.pMemObjHostMSR, 0);
273 memset(pVCpu->hwaccm.s.vmx.pHostMSR, 0, PAGE_SIZE);
274#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
275
276 /* Current guest paging mode. */
277 pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode = PGMMODE_REAL;
278
279#ifdef LOG_ENABLED
280 SUPR0Printf("VMXR0InitVM %x VMCS=%x (%x)\n", pVM, pVCpu->hwaccm.s.vmx.pVMCS, (uint32_t)pVCpu->hwaccm.s.vmx.pVMCSPhys);
281#endif
282 }
283
284 return VINF_SUCCESS;
285}
286
287/**
288 * Does Ring-0 per VM VT-x termination.
289 *
290 * @returns VBox status code.
291 * @param pVM The VM to operate on.
292 */
293VMMR0DECL(int) VMXR0TermVM(PVM pVM)
294{
295 for (VMCPUID i = 0; i < pVM->cCpus; i++)
296 {
297 PVMCPU pVCpu = &pVM->aCpus[i];
298
299 if (pVCpu->hwaccm.s.vmx.pMemObjVMCS != NIL_RTR0MEMOBJ)
300 {
301 RTR0MemObjFree(pVCpu->hwaccm.s.vmx.pMemObjVMCS, false);
302 pVCpu->hwaccm.s.vmx.pMemObjVMCS = NIL_RTR0MEMOBJ;
303 pVCpu->hwaccm.s.vmx.pVMCS = 0;
304 pVCpu->hwaccm.s.vmx.pVMCSPhys = 0;
305 }
306 if (pVCpu->hwaccm.s.vmx.pMemObjVAPIC != NIL_RTR0MEMOBJ)
307 {
308 RTR0MemObjFree(pVCpu->hwaccm.s.vmx.pMemObjVAPIC, false);
309 pVCpu->hwaccm.s.vmx.pMemObjVAPIC = NIL_RTR0MEMOBJ;
310 pVCpu->hwaccm.s.vmx.pVAPIC = 0;
311 pVCpu->hwaccm.s.vmx.pVAPICPhys = 0;
312 }
313 if (pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap != NIL_RTR0MEMOBJ)
314 {
315 RTR0MemObjFree(pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap, false);
316 pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap = NIL_RTR0MEMOBJ;
317 pVCpu->hwaccm.s.vmx.pMSRBitmap = 0;
318 pVCpu->hwaccm.s.vmx.pMSRBitmapPhys = 0;
319 }
320#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
321 if (pVCpu->hwaccm.s.vmx.pMemObjHostMSR != NIL_RTR0MEMOBJ)
322 {
323 RTR0MemObjFree(pVCpu->hwaccm.s.vmx.pMemObjHostMSR, false);
324 pVCpu->hwaccm.s.vmx.pMemObjHostMSR = NIL_RTR0MEMOBJ;
325 pVCpu->hwaccm.s.vmx.pHostMSR = 0;
326 pVCpu->hwaccm.s.vmx.pHostMSRPhys = 0;
327 }
328 if (pVCpu->hwaccm.s.vmx.pMemObjGuestMSR != NIL_RTR0MEMOBJ)
329 {
330 RTR0MemObjFree(pVCpu->hwaccm.s.vmx.pMemObjGuestMSR, false);
331 pVCpu->hwaccm.s.vmx.pMemObjGuestMSR = NIL_RTR0MEMOBJ;
332 pVCpu->hwaccm.s.vmx.pGuestMSR = 0;
333 pVCpu->hwaccm.s.vmx.pGuestMSRPhys = 0;
334 }
335#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
336 }
337 if (pVM->hwaccm.s.vmx.pMemObjAPIC != NIL_RTR0MEMOBJ)
338 {
339 RTR0MemObjFree(pVM->hwaccm.s.vmx.pMemObjAPIC, false);
340 pVM->hwaccm.s.vmx.pMemObjAPIC = NIL_RTR0MEMOBJ;
341 pVM->hwaccm.s.vmx.pAPIC = 0;
342 pVM->hwaccm.s.vmx.pAPICPhys = 0;
343 }
344#ifdef VBOX_WITH_CRASHDUMP_MAGIC
345 if (pVM->hwaccm.s.vmx.pMemObjScratch != NIL_RTR0MEMOBJ)
346 {
347 ASMMemZero32(pVM->hwaccm.s.vmx.pScratch, PAGE_SIZE);
348 RTR0MemObjFree(pVM->hwaccm.s.vmx.pMemObjScratch, false);
349 pVM->hwaccm.s.vmx.pMemObjScratch = NIL_RTR0MEMOBJ;
350 pVM->hwaccm.s.vmx.pScratch = 0;
351 pVM->hwaccm.s.vmx.pScratchPhys = 0;
352 }
353#endif
354 return VINF_SUCCESS;
355}
356
357/**
358 * Sets up VT-x for the specified VM
359 *
360 * @returns VBox status code.
361 * @param pVM The VM to operate on.
362 */
363VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
364{
365 int rc = VINF_SUCCESS;
366 uint32_t val;
367
368 AssertReturn(pVM, VERR_INVALID_PARAMETER);
369
370 for (VMCPUID i = 0; i < pVM->cCpus; i++)
371 {
372 PVMCPU pVCpu = &pVM->aCpus[i];
373
374 Assert(pVCpu->hwaccm.s.vmx.pVMCS);
375
376 /* Set revision dword at the beginning of the VMCS structure. */
377 *(uint32_t *)pVCpu->hwaccm.s.vmx.pVMCS = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hwaccm.s.vmx.msr.vmx_basic_info);
378
379 /* Clear VM Control Structure. */
380 Log(("pVMCSPhys = %RHp\n", pVCpu->hwaccm.s.vmx.pVMCSPhys));
381 rc = VMXClearVMCS(pVCpu->hwaccm.s.vmx.pVMCSPhys);
382 if (RT_FAILURE(rc))
383 goto vmx_end;
384
385 /* Activate the VM Control Structure. */
386 rc = VMXActivateVMCS(pVCpu->hwaccm.s.vmx.pVMCSPhys);
387 if (RT_FAILURE(rc))
388 goto vmx_end;
389
390 /* VMX_VMCS_CTRL_PIN_EXEC_CONTROLS
391 * Set required bits to one and zero according to the MSR capabilities.
392 */
393 val = pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.disallowed0;
394 /* External and non-maskable interrupts cause VM-exits. */
395 val = val | VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_EXT_INT_EXIT | VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_NMI_EXIT;
396 val &= pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.allowed1;
397
398 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PIN_EXEC_CONTROLS, val);
399 AssertRC(rc);
400
401 /* VMX_VMCS_CTRL_PROC_EXEC_CONTROLS
402 * Set required bits to one and zero according to the MSR capabilities.
403 */
404 val = pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.disallowed0;
405 /* Program which event cause VM-exits and which features we want to use. */
406 val = val | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_HLT_EXIT
407 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_TSC_OFFSET
408 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT
409 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_UNCOND_IO_EXIT
410 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDPMC_EXIT
411 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_EXIT
412 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MWAIT_EXIT; /* don't execute mwait or else we'll idle inside the guest (host thinks the cpu load is high) */
413
414 /* Without nested paging we should intercept invlpg and cr3 mov instructions. */
415 if (!pVM->hwaccm.s.fNestedPaging)
416 val |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT
417 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
418 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
419
420 /* Note: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MWAIT_EXIT might cause a vmlaunch failure with an invalid control fields error. (combined with some other exit reasons) */
421 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
422 {
423 /* CR8 reads from the APIC shadow page; writes cause an exit is they lower the TPR below the threshold */
424 val |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW;
425 Assert(pVM->hwaccm.s.vmx.pAPIC);
426 }
427 else
428 /* Exit on CR8 reads & writes in case the TPR shadow feature isn't present. */
429 val |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_STORE_EXIT | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_LOAD_EXIT;
430
431 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
432 {
433 Assert(pVCpu->hwaccm.s.vmx.pMSRBitmapPhys);
434 val |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS;
435 }
436
437 /* We will use the secondary control if it's present. */
438 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
439
440 /* Mask away the bits that the CPU doesn't support */
441 /** @todo make sure they don't conflict with the above requirements. */
442 val &= pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1;
443 pVCpu->hwaccm.s.vmx.proc_ctls = val;
444
445 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, val);
446 AssertRC(rc);
447
448 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
449 {
450 /* VMX_VMCS_CTRL_PROC_EXEC_CONTROLS2
451 * Set required bits to one and zero according to the MSR capabilities.
452 */
453 val = pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0;
454 val |= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT;
455
456#ifdef HWACCM_VTX_WITH_EPT
457 if (pVM->hwaccm.s.fNestedPaging)
458 val |= VMX_VMCS_CTRL_PROC_EXEC2_EPT;
459#endif /* HWACCM_VTX_WITH_EPT */
460#ifdef HWACCM_VTX_WITH_VPID
461 else
462 if (pVM->hwaccm.s.vmx.fVPID)
463 val |= VMX_VMCS_CTRL_PROC_EXEC2_VPID;
464#endif /* HWACCM_VTX_WITH_VPID */
465
466 if (pVM->hwaccm.s.fHasIoApic)
467 val |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC;
468
469 if (pVM->hwaccm.s.vmx.fUnrestrictedGuest)
470 val |= VMX_VMCS_CTRL_PROC_EXEC2_REAL_MODE;
471
472 /* Mask away the bits that the CPU doesn't support */
473 /** @todo make sure they don't conflict with the above requirements. */
474 val &= pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1;
475 pVCpu->hwaccm.s.vmx.proc_ctls2 = val;
476 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS2, val);
477 AssertRC(rc);
478 }
479
480 /* VMX_VMCS_CTRL_CR3_TARGET_COUNT
481 * Set required bits to one and zero according to the MSR capabilities.
482 */
483 rc = VMXWriteVMCS(VMX_VMCS_CTRL_CR3_TARGET_COUNT, 0);
484 AssertRC(rc);
485
486 /* Forward all exception except #NM & #PF to the guest.
487 * We always need to check pagefaults since our shadow page table can be out of sync.
488 * And we always lazily sync the FPU & XMM state.
489 */
490
491 /** @todo Possible optimization:
492 * Keep the FPU and XMM state current in the EM thread. That way there's no need to
493 * lazily sync anything, but the downside is that we can't use the FPU stack or XMM
494 * registers ourselves of course.
495 *
496 * Note: only possible if the current state is actually ours (X86_CR0_TS flag)
497 */
498
499 /* Don't filter page faults; all of them should cause a switch. */
500 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PAGEFAULT_ERROR_MASK, 0);
501 rc |= VMXWriteVMCS(VMX_VMCS_CTRL_PAGEFAULT_ERROR_MATCH, 0);
502 AssertRC(rc);
503
504 /* Init TSC offset to zero. */
505 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_TSC_OFFSET_FULL, 0);
506 AssertRC(rc);
507
508 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_IO_BITMAP_A_FULL, 0);
509 AssertRC(rc);
510
511 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_IO_BITMAP_B_FULL, 0);
512 AssertRC(rc);
513
514 /* Set the MSR bitmap address. */
515 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
516 {
517 Assert(pVCpu->hwaccm.s.vmx.pMSRBitmapPhys);
518
519 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_MSR_BITMAP_FULL, pVCpu->hwaccm.s.vmx.pMSRBitmapPhys);
520 AssertRC(rc);
521
522 /* Allow the guest to directly modify these MSRs; they are restored and saved automatically. */
523 vmxR0SetMSRPermission(pVCpu, MSR_IA32_SYSENTER_CS, true, true);
524 vmxR0SetMSRPermission(pVCpu, MSR_IA32_SYSENTER_ESP, true, true);
525 vmxR0SetMSRPermission(pVCpu, MSR_IA32_SYSENTER_EIP, true, true);
526 vmxR0SetMSRPermission(pVCpu, MSR_K8_LSTAR, true, true);
527 vmxR0SetMSRPermission(pVCpu, MSR_K6_STAR, true, true);
528 vmxR0SetMSRPermission(pVCpu, MSR_K8_SF_MASK, true, true);
529 vmxR0SetMSRPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, true, true);
530 vmxR0SetMSRPermission(pVCpu, MSR_K8_GS_BASE, true, true);
531 vmxR0SetMSRPermission(pVCpu, MSR_K8_FS_BASE, true, true);
532 }
533
534#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
535 /* Set the guest & host MSR load/store physical addresses. */
536 Assert(pVCpu->hwaccm.s.vmx.pGuestMSRPhys);
537 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_VMENTRY_MSR_LOAD_FULL, pVCpu->hwaccm.s.vmx.pGuestMSRPhys);
538 AssertRC(rc);
539 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_VMEXIT_MSR_STORE_FULL, pVCpu->hwaccm.s.vmx.pGuestMSRPhys);
540 AssertRC(rc);
541
542 Assert(pVCpu->hwaccm.s.vmx.pHostMSRPhys);
543 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_VMEXIT_MSR_LOAD_FULL, pVCpu->hwaccm.s.vmx.pHostMSRPhys);
544 AssertRC(rc);
545#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
546
547 rc = VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
548 AssertRC(rc);
549
550 rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_MSR_STORE_COUNT, 0);
551 AssertRC(rc);
552
553 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
554 {
555 Assert(pVM->hwaccm.s.vmx.pMemObjAPIC);
556 /* Optional */
557 rc = VMXWriteVMCS(VMX_VMCS_CTRL_TPR_THRESHOLD, 0);
558 rc |= VMXWriteVMCS64(VMX_VMCS_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hwaccm.s.vmx.pVAPICPhys);
559
560 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
561 rc |= VMXWriteVMCS64(VMX_VMCS_CTRL_APIC_ACCESSADDR_FULL, pVM->hwaccm.s.vmx.pAPICPhys);
562
563 AssertRC(rc);
564 }
565
566 /* Set link pointer to -1. Not currently used. */
567 rc = VMXWriteVMCS64(VMX_VMCS_GUEST_LINK_PTR_FULL, 0xFFFFFFFFFFFFFFFFULL);
568 AssertRC(rc);
569
570 /* Clear VM Control Structure. Marking it inactive, clearing implementation specific data and writing back VMCS data to memory. */
571 rc = VMXClearVMCS(pVCpu->hwaccm.s.vmx.pVMCSPhys);
572 AssertRC(rc);
573
574 /* Configure the VMCS read cache. */
575 PVMCSCACHE pCache = &pVCpu->hwaccm.s.vmx.VMCSCache;
576
577 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_RIP);
578 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_RSP);
579 VMXSetupCachedReadVMCS(pCache, VMX_VMCS_GUEST_RFLAGS);
580 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE);
581 VMXSetupCachedReadVMCS(pCache, VMX_VMCS_CTRL_CR0_READ_SHADOW);
582 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_CR0);
583 VMXSetupCachedReadVMCS(pCache, VMX_VMCS_CTRL_CR4_READ_SHADOW);
584 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_CR4);
585 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_DR7);
586 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_SYSENTER_CS);
587 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_SYSENTER_EIP);
588 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_SYSENTER_ESP);
589 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_GDTR_LIMIT);
590 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_GDTR_BASE);
591 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_IDTR_LIMIT);
592 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_IDTR_BASE);
593
594 VMX_SETUP_SELREG(ES, pCache);
595 VMX_SETUP_SELREG(SS, pCache);
596 VMX_SETUP_SELREG(CS, pCache);
597 VMX_SETUP_SELREG(DS, pCache);
598 VMX_SETUP_SELREG(FS, pCache);
599 VMX_SETUP_SELREG(GS, pCache);
600 VMX_SETUP_SELREG(LDTR, pCache);
601 VMX_SETUP_SELREG(TR, pCache);
602
603 /* Status code VMCS reads. */
604 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_REASON);
605 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_VM_INSTR_ERROR);
606 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INSTR_LENGTH);
607 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INTERRUPTION_ERRCODE);
608 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO);
609 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INSTR_INFO);
610 VMXSetupCachedReadVMCS(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
611 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_IDT_INFO);
612 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_IDT_ERRCODE);
613
614 if (pVM->hwaccm.s.fNestedPaging)
615 {
616 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_CR3);
617 VMXSetupCachedReadVMCS(pCache, VMX_VMCS_EXIT_PHYS_ADDR_FULL);
618 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
619 }
620 else
621 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
622 } /* for each VMCPU */
623
624 /* Choose the right TLB setup function. */
625 if (pVM->hwaccm.s.fNestedPaging)
626 {
627 pVM->hwaccm.s.vmx.pfnSetupTaggedTLB = vmxR0SetupTLBEPT;
628
629 /* Default values for flushing. */
630 pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_ALL_CONTEXTS;
631 pVM->hwaccm.s.vmx.enmFlushContext = VMX_FLUSH_ALL_CONTEXTS;
632
633 /* If the capabilities specify we can do more, then make use of it. */
634 if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVEPT_CAPS_INDIV)
635 pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_PAGE;
636 else
637 if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVEPT_CAPS_CONTEXT)
638 pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_SINGLE_CONTEXT;
639
640 if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVEPT_CAPS_CONTEXT)
641 pVM->hwaccm.s.vmx.enmFlushContext = VMX_FLUSH_SINGLE_CONTEXT;
642 }
643#ifdef HWACCM_VTX_WITH_VPID
644 else
645 if (pVM->hwaccm.s.vmx.fVPID)
646 {
647 pVM->hwaccm.s.vmx.pfnSetupTaggedTLB = vmxR0SetupTLBVPID;
648
649 /* Default values for flushing. */
650 pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_ALL_CONTEXTS;
651 pVM->hwaccm.s.vmx.enmFlushContext = VMX_FLUSH_ALL_CONTEXTS;
652
653 /* If the capabilities specify we can do more, then make use of it. */
654 if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_INDIV)
655 pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_PAGE;
656 else
657 if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_CONTEXT)
658 pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_SINGLE_CONTEXT;
659
660 if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_CONTEXT)
661 pVM->hwaccm.s.vmx.enmFlushContext = VMX_FLUSH_SINGLE_CONTEXT;
662 }
663#endif /* HWACCM_VTX_WITH_VPID */
664 else
665 pVM->hwaccm.s.vmx.pfnSetupTaggedTLB = vmxR0SetupTLBDummy;
666
667vmx_end:
668 VMXR0CheckError(pVM, &pVM->aCpus[0], rc);
669 return rc;
670}
671
672/**
673 * Sets the permission bits for the specified MSR
674 *
675 * @param pVCpu The VMCPU to operate on.
676 * @param ulMSR MSR value
677 * @param fRead Reading allowed/disallowed
678 * @param fWrite Writing allowed/disallowed
679 */
680static void vmxR0SetMSRPermission(PVMCPU pVCpu, unsigned ulMSR, bool fRead, bool fWrite)
681{
682 unsigned ulBit;
683 uint8_t *pMSRBitmap = (uint8_t *)pVCpu->hwaccm.s.vmx.pMSRBitmap;
684
685 /* Layout:
686 * 0x000 - 0x3ff - Low MSR read bits
687 * 0x400 - 0x7ff - High MSR read bits
688 * 0x800 - 0xbff - Low MSR write bits
689 * 0xc00 - 0xfff - High MSR write bits
690 */
691 if (ulMSR <= 0x00001FFF)
692 {
693 /* Pentium-compatible MSRs */
694 ulBit = ulMSR;
695 }
696 else
697 if ( ulMSR >= 0xC0000000
698 && ulMSR <= 0xC0001FFF)
699 {
700 /* AMD Sixth Generation x86 Processor MSRs */
701 ulBit = (ulMSR - 0xC0000000);
702 pMSRBitmap += 0x400;
703 }
704 else
705 {
706 AssertFailed();
707 return;
708 }
709
710 Assert(ulBit <= 0x1fff);
711 if (fRead)
712 ASMBitClear(pMSRBitmap, ulBit);
713 else
714 ASMBitSet(pMSRBitmap, ulBit);
715
716 if (fWrite)
717 ASMBitClear(pMSRBitmap + 0x800, ulBit);
718 else
719 ASMBitSet(pMSRBitmap + 0x800, ulBit);
720}
721
722
723/**
724 * Injects an event (trap or external interrupt)
725 *
726 * @returns VBox status code.
727 * @param pVM The VM to operate on.
728 * @param pVCpu The VMCPU to operate on.
729 * @param pCtx CPU Context
730 * @param intInfo VMX interrupt info
731 * @param cbInstr Opcode length of faulting instruction
732 * @param errCode Error code (optional)
733 */
734static int VMXR0InjectEvent(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, uint32_t intInfo, uint32_t cbInstr, uint32_t errCode)
735{
736 int rc;
737 uint32_t iGate = VMX_EXIT_INTERRUPTION_INFO_VECTOR(intInfo);
738
739#ifdef VBOX_WITH_STATISTICS
740 STAM_COUNTER_INC(&pVCpu->hwaccm.s.paStatInjectedIrqsR0[iGate & MASK_INJECT_IRQ_STAT]);
741#endif
742
743#ifdef VBOX_STRICT
744 if (iGate == 0xE)
745 LogFlow(("VMXR0InjectEvent: Injecting interrupt %d at %RGv error code=%08x CR2=%RGv intInfo=%08x\n", iGate, (RTGCPTR)pCtx->rip, errCode, pCtx->cr2, intInfo));
746 else
747 if (iGate < 0x20)
748 LogFlow(("VMXR0InjectEvent: Injecting interrupt %d at %RGv error code=%08x\n", iGate, (RTGCPTR)pCtx->rip, errCode));
749 else
750 {
751 LogFlow(("INJ-EI: %x at %RGv\n", iGate, (RTGCPTR)pCtx->rip));
752 Assert(VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW || !VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
753 Assert(VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW || pCtx->eflags.u32 & X86_EFL_IF);
754 }
755#endif
756
757#ifdef HWACCM_VMX_EMULATE_REALMODE
758 if ( CPUMIsGuestInRealModeEx(pCtx)
759 && pVM->hwaccm.s.vmx.pRealModeTSS)
760 {
761 RTGCPHYS GCPhysHandler;
762 uint16_t offset, ip;
763 RTSEL sel;
764
765 /* Injecting events doesn't work right with real mode emulation.
766 * (#GP if we try to inject external hardware interrupts)
767 * Inject the interrupt or trap directly instead.
768 *
769 * ASSUMES no access handlers for the bits we read or write below (should be safe).
770 */
771 Log(("Manual interrupt/trap '%x' inject (real mode)\n", iGate));
772
773 /* Check if the interrupt handler is present. */
774 if (iGate * 4 + 3 > pCtx->idtr.cbIdt)
775 {
776 Log(("IDT cbIdt violation\n"));
777 if (iGate != X86_XCPT_DF)
778 {
779 uint32_t intInfo2;
780
781 intInfo2 = (iGate == X86_XCPT_GP) ? (uint32_t)X86_XCPT_DF : iGate;
782 intInfo2 |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
783 intInfo2 |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
784 intInfo2 |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
785
786 return VMXR0InjectEvent(pVM, pVCpu, pCtx, intInfo2, 0, 0 /* no error code according to the Intel docs */);
787 }
788 Log(("Triple fault -> reset the VM!\n"));
789 return VINF_EM_RESET;
790 }
791 if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW
792 || iGate == 3 /* Both #BP and #OF point to the instruction after. */
793 || iGate == 4)
794 {
795 ip = pCtx->ip + cbInstr;
796 }
797 else
798 ip = pCtx->ip;
799
800 /* Read the selector:offset pair of the interrupt handler. */
801 GCPhysHandler = (RTGCPHYS)pCtx->idtr.pIdt + iGate * 4;
802 rc = PGMPhysSimpleReadGCPhys(pVM, &offset, GCPhysHandler, sizeof(offset)); AssertRC(rc);
803 rc = PGMPhysSimpleReadGCPhys(pVM, &sel, GCPhysHandler + 2, sizeof(sel)); AssertRC(rc);
804
805 LogFlow(("IDT handler %04X:%04X\n", sel, offset));
806
807 /* Construct the stack frame. */
808 /** @todo should check stack limit. */
809 pCtx->sp -= 2;
810 LogFlow(("ss:sp %04X:%04X eflags=%x\n", pCtx->ss, pCtx->sp, pCtx->eflags.u));
811 rc = PGMPhysSimpleWriteGCPhys(pVM, pCtx->ssHid.u64Base + pCtx->sp, &pCtx->eflags, sizeof(uint16_t)); AssertRC(rc);
812 pCtx->sp -= 2;
813 LogFlow(("ss:sp %04X:%04X cs=%x\n", pCtx->ss, pCtx->sp, pCtx->cs));
814 rc = PGMPhysSimpleWriteGCPhys(pVM, pCtx->ssHid.u64Base + pCtx->sp, &pCtx->cs, sizeof(uint16_t)); AssertRC(rc);
815 pCtx->sp -= 2;
816 LogFlow(("ss:sp %04X:%04X ip=%x\n", pCtx->ss, pCtx->sp, ip));
817 rc = PGMPhysSimpleWriteGCPhys(pVM, pCtx->ssHid.u64Base + pCtx->sp, &ip, sizeof(ip)); AssertRC(rc);
818
819 /* Update the CPU state for executing the handler. */
820 pCtx->rip = offset;
821 pCtx->cs = sel;
822 pCtx->csHid.u64Base = sel << 4;
823 pCtx->eflags.u &= ~(X86_EFL_IF|X86_EFL_TF|X86_EFL_RF|X86_EFL_AC);
824
825 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_SEGMENT_REGS;
826 return VINF_SUCCESS;
827 }
828#endif /* HWACCM_VMX_EMULATE_REALMODE */
829
830 /* Set event injection state. */
831 rc = VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_IRQ_INFO, intInfo | (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT));
832
833 rc |= VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
834 rc |= VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_EXCEPTION_ERRCODE, errCode);
835
836 AssertRC(rc);
837 return rc;
838}
839
840
841/**
842 * Checks for pending guest interrupts and injects them
843 *
844 * @returns VBox status code.
845 * @param pVM The VM to operate on.
846 * @param pVCpu The VMCPU to operate on.
847 * @param pCtx CPU Context
848 */
849static int VMXR0CheckPendingInterrupt(PVM pVM, PVMCPU pVCpu, CPUMCTX *pCtx)
850{
851 int rc;
852
853 /* Dispatch any pending interrupts. (injected before, but a VM exit occurred prematurely) */
854 if (pVCpu->hwaccm.s.Event.fPending)
855 {
856 Log(("CPU%d: Reinjecting event %RX64 %08x at %RGv cr2=%RX64\n", pVCpu->idCpu, pVCpu->hwaccm.s.Event.intInfo, pVCpu->hwaccm.s.Event.errCode, (RTGCPTR)pCtx->rip, pCtx->cr2));
857 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatIntReinject);
858 rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, pVCpu->hwaccm.s.Event.intInfo, 0, pVCpu->hwaccm.s.Event.errCode);
859 AssertRC(rc);
860
861 pVCpu->hwaccm.s.Event.fPending = false;
862 return VINF_SUCCESS;
863 }
864
865 /* If an active trap is already pending, then we must forward it first! */
866 if (!TRPMHasTrap(pVCpu))
867 {
868 if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI))
869 {
870 RTGCUINTPTR intInfo;
871
872 Log(("CPU%d: injecting #NMI\n", pVCpu->idCpu));
873
874 intInfo = X86_XCPT_NMI;
875 intInfo |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
876 intInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
877
878 rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, intInfo, 0, 0);
879 AssertRC(rc);
880
881 return VINF_SUCCESS;
882 }
883
884 /* @todo SMI interrupts. */
885
886 /* When external interrupts are pending, we should exit the VM when IF is set. */
887 if (VMCPU_FF_ISPENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC|VMCPU_FF_INTERRUPT_PIC)))
888 {
889 if (!(pCtx->eflags.u32 & X86_EFL_IF))
890 {
891 if (!(pVCpu->hwaccm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_IRQ_WINDOW_EXIT))
892 {
893 LogFlow(("Enable irq window exit!\n"));
894 pVCpu->hwaccm.s.vmx.proc_ctls |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_IRQ_WINDOW_EXIT;
895 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
896 AssertRC(rc);
897 }
898 /* else nothing to do but wait */
899 }
900 else
901 if (!VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
902 {
903 uint8_t u8Interrupt;
904
905 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
906 Log(("CPU%d: Dispatch interrupt: u8Interrupt=%x (%d) rc=%Rrc cs:rip=%04X:%RGv\n", pVCpu->idCpu, u8Interrupt, u8Interrupt, rc, pCtx->cs, (RTGCPTR)pCtx->rip));
907 if (RT_SUCCESS(rc))
908 {
909 rc = TRPMAssertTrap(pVCpu, u8Interrupt, TRPM_HARDWARE_INT);
910 AssertRC(rc);
911 }
912 else
913 {
914 /* Can only happen in rare cases where a pending interrupt is cleared behind our back */
915 Assert(!VMCPU_FF_ISPENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC|VMCPU_FF_INTERRUPT_PIC)));
916 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatSwitchGuestIrq);
917 /* Just continue */
918 }
919 }
920 else
921 Log(("Pending interrupt blocked at %RGv by VM_FF_INHIBIT_INTERRUPTS!!\n", (RTGCPTR)pCtx->rip));
922 }
923 }
924
925#ifdef VBOX_STRICT
926 if (TRPMHasTrap(pVCpu))
927 {
928 uint8_t u8Vector;
929 rc = TRPMQueryTrapAll(pVCpu, &u8Vector, 0, 0, 0);
930 AssertRC(rc);
931 }
932#endif
933
934 if ( (pCtx->eflags.u32 & X86_EFL_IF)
935 && (!VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
936 && TRPMHasTrap(pVCpu)
937 )
938 {
939 uint8_t u8Vector;
940 TRPMEVENT enmType;
941 RTGCUINTPTR intInfo;
942 RTGCUINT errCode;
943
944 /* If a new event is pending, then dispatch it now. */
945 rc = TRPMQueryTrapAll(pVCpu, &u8Vector, &enmType, &errCode, 0);
946 AssertRC(rc);
947 Assert(pCtx->eflags.Bits.u1IF == 1 || enmType == TRPM_TRAP);
948 Assert(enmType != TRPM_SOFTWARE_INT);
949
950 /* Clear the pending trap. */
951 rc = TRPMResetTrap(pVCpu);
952 AssertRC(rc);
953
954 intInfo = u8Vector;
955 intInfo |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
956
957 if (enmType == TRPM_TRAP)
958 {
959 switch (u8Vector) {
960 case 8:
961 case 10:
962 case 11:
963 case 12:
964 case 13:
965 case 14:
966 case 17:
967 /* Valid error codes. */
968 intInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
969 break;
970 default:
971 break;
972 }
973 if (u8Vector == X86_XCPT_BP || u8Vector == X86_XCPT_OF)
974 intInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
975 else
976 intInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
977 }
978 else
979 intInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
980
981 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatIntInject);
982 rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, intInfo, 0, errCode);
983 AssertRC(rc);
984 } /* if (interrupts can be dispatched) */
985
986 return VINF_SUCCESS;
987}
988
989/**
990 * Save the host state
991 *
992 * @returns VBox status code.
993 * @param pVM The VM to operate on.
994 * @param pVCpu The VMCPU to operate on.
995 */
996VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
997{
998 int rc = VINF_SUCCESS;
999
1000 /*
1001 * Host CPU Context
1002 */
1003 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_HOST_CONTEXT)
1004 {
1005 RTIDTR idtr;
1006 RTGDTR gdtr;
1007 RTSEL SelTR;
1008 PCX86DESCHC pDesc;
1009 uintptr_t trBase;
1010 RTSEL cs;
1011 RTSEL ss;
1012 uint64_t cr3;
1013
1014 /* Control registers */
1015 rc = VMXWriteVMCS(VMX_VMCS_HOST_CR0, ASMGetCR0());
1016#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1017 if (VMX_IS_64BIT_HOST_MODE())
1018 {
1019 cr3 = hwaccmR0Get64bitCR3();
1020 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_CR3, cr3);
1021 }
1022 else
1023#endif
1024 {
1025 cr3 = ASMGetCR3();
1026 rc |= VMXWriteVMCS(VMX_VMCS_HOST_CR3, cr3);
1027 }
1028 rc |= VMXWriteVMCS(VMX_VMCS_HOST_CR4, ASMGetCR4());
1029 AssertRC(rc);
1030 Log2(("VMX_VMCS_HOST_CR0 %08x\n", ASMGetCR0()));
1031 Log2(("VMX_VMCS_HOST_CR3 %08RX64\n", cr3));
1032 Log2(("VMX_VMCS_HOST_CR4 %08x\n", ASMGetCR4()));
1033
1034 /* Selector registers. */
1035#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1036 if (VMX_IS_64BIT_HOST_MODE())
1037 {
1038 cs = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
1039 ss = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
1040 }
1041 else
1042 {
1043 /* sysenter loads LDT cs & ss, VMX doesn't like this. Load the GDT ones (safe). */
1044 cs = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
1045 ss = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
1046 }
1047#else
1048 cs = ASMGetCS();
1049 ss = ASMGetSS();
1050#endif
1051 Assert(!(cs & X86_SEL_LDT)); Assert((cs & X86_SEL_RPL) == 0);
1052 Assert(!(ss & X86_SEL_LDT)); Assert((ss & X86_SEL_RPL) == 0);
1053 rc = VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_CS, cs);
1054 /* Note: VMX is (again) very picky about the RPL of the selectors here; we'll restore them manually. */
1055 rc |= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_DS, 0);
1056 rc |= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_ES, 0);
1057#if HC_ARCH_BITS == 32
1058 if (!VMX_IS_64BIT_HOST_MODE())
1059 {
1060 rc |= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_FS, 0);
1061 rc |= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_GS, 0);
1062 }
1063#endif
1064 rc |= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_SS, ss);
1065 SelTR = ASMGetTR();
1066 rc |= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_TR, SelTR);
1067 AssertRC(rc);
1068 Log2(("VMX_VMCS_HOST_FIELD_CS %08x (%08x)\n", cs, ASMGetSS()));
1069 Log2(("VMX_VMCS_HOST_FIELD_DS 00000000 (%08x)\n", ASMGetDS()));
1070 Log2(("VMX_VMCS_HOST_FIELD_ES 00000000 (%08x)\n", ASMGetES()));
1071 Log2(("VMX_VMCS_HOST_FIELD_FS 00000000 (%08x)\n", ASMGetFS()));
1072 Log2(("VMX_VMCS_HOST_FIELD_GS 00000000 (%08x)\n", ASMGetGS()));
1073 Log2(("VMX_VMCS_HOST_FIELD_SS %08x (%08x)\n", ss, ASMGetSS()));
1074 Log2(("VMX_VMCS_HOST_FIELD_TR %08x\n", ASMGetTR()));
1075
1076 /* GDTR & IDTR */
1077#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1078 if (VMX_IS_64BIT_HOST_MODE())
1079 {
1080 X86XDTR64 gdtr64, idtr64;
1081 hwaccmR0Get64bitGDTRandIDTR(&gdtr64, &idtr64);
1082 rc = VMXWriteVMCS64(VMX_VMCS_HOST_GDTR_BASE, gdtr64.uAddr);
1083 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_IDTR_BASE, gdtr64.uAddr);
1084 AssertRC(rc);
1085 Log2(("VMX_VMCS_HOST_GDTR_BASE %RX64\n", gdtr64.uAddr));
1086 Log2(("VMX_VMCS_HOST_IDTR_BASE %RX64\n", idtr64.uAddr));
1087 gdtr.cbGdt = gdtr64.cb;
1088 gdtr.pGdt = (uintptr_t)gdtr64.uAddr;
1089 }
1090 else
1091#endif
1092 {
1093 ASMGetGDTR(&gdtr);
1094 rc = VMXWriteVMCS(VMX_VMCS_HOST_GDTR_BASE, gdtr.pGdt);
1095 ASMGetIDTR(&idtr);
1096 rc |= VMXWriteVMCS(VMX_VMCS_HOST_IDTR_BASE, idtr.pIdt);
1097 AssertRC(rc);
1098 Log2(("VMX_VMCS_HOST_GDTR_BASE %RHv\n", gdtr.pGdt));
1099 Log2(("VMX_VMCS_HOST_IDTR_BASE %RHv\n", idtr.pIdt));
1100 }
1101
1102 /* Save the base address of the TR selector. */
1103 if (SelTR > gdtr.cbGdt)
1104 {
1105 AssertMsgFailed(("Invalid TR selector %x. GDTR.cbGdt=%x\n", SelTR, gdtr.cbGdt));
1106 return VERR_VMX_INVALID_HOST_STATE;
1107 }
1108
1109 pDesc = (PCX86DESCHC)(gdtr.pGdt + (SelTR & X86_SEL_MASK));
1110#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1111 if (VMX_IS_64BIT_HOST_MODE())
1112 {
1113 uint64_t trBase64 = X86DESC64_BASE(*(PX86DESC64)pDesc);
1114 rc = VMXWriteVMCS64(VMX_VMCS_HOST_TR_BASE, trBase64);
1115 Log2(("VMX_VMCS_HOST_TR_BASE %RX64\n", trBase64));
1116 AssertRC(rc);
1117 }
1118 else
1119#endif
1120 {
1121#if HC_ARCH_BITS == 64
1122 trBase = X86DESC64_BASE(*pDesc);
1123#else
1124 trBase = X86DESC_BASE(*pDesc);
1125#endif
1126 rc = VMXWriteVMCS(VMX_VMCS_HOST_TR_BASE, trBase);
1127 AssertRC(rc);
1128 Log2(("VMX_VMCS_HOST_TR_BASE %RHv\n", trBase));
1129 }
1130
1131 /* FS and GS base. */
1132#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1133 if (VMX_IS_64BIT_HOST_MODE())
1134 {
1135 Log2(("MSR_K8_FS_BASE = %RX64\n", ASMRdMsr(MSR_K8_FS_BASE)));
1136 Log2(("MSR_K8_GS_BASE = %RX64\n", ASMRdMsr(MSR_K8_GS_BASE)));
1137 rc = VMXWriteVMCS64(VMX_VMCS_HOST_FS_BASE, ASMRdMsr(MSR_K8_FS_BASE));
1138 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_GS_BASE, ASMRdMsr(MSR_K8_GS_BASE));
1139 }
1140#endif
1141 AssertRC(rc);
1142
1143 /* Sysenter MSRs. */
1144 /** @todo expensive!! */
1145 rc = VMXWriteVMCS(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
1146 Log2(("VMX_VMCS_HOST_SYSENTER_CS %08x\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_CS)));
1147#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1148 if (VMX_IS_64BIT_HOST_MODE())
1149 {
1150 Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_EIP)));
1151 Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_ESP)));
1152 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
1153 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
1154 }
1155 else
1156 {
1157 rc |= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
1158 rc |= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
1159 Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP)));
1160 Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP)));
1161 }
1162#elif HC_ARCH_BITS == 32
1163 rc |= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
1164 rc |= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
1165 Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP)));
1166 Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP)));
1167#else
1168 Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_EIP)));
1169 Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_ESP)));
1170 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
1171 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
1172#endif
1173 AssertRC(rc);
1174
1175#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
1176 /* Store all host MSRs in the VM-Exit load area, so they will be reloaded after the world switch back to the host. */
1177 PVMXMSR pMsr = (PVMXMSR)pVCpu->hwaccm.s.vmx.pHostMSR;
1178 unsigned idxMsr = 0;
1179
1180 /* EFER MSR present? */
1181 if (ASMCpuId_EDX(0x80000001) & (X86_CPUID_AMD_FEATURE_EDX_NX|X86_CPUID_AMD_FEATURE_EDX_LONG_MODE))
1182 {
1183 if (ASMCpuId_EDX(0x80000001) & X86_CPUID_AMD_FEATURE_EDX_SEP)
1184 {
1185 pMsr->u32IndexMSR = MSR_K6_STAR;
1186 pMsr->u32Reserved = 0;
1187 pMsr->u64Value = ASMRdMsr(MSR_K6_STAR); /* legacy syscall eip, cs & ss */
1188 pMsr++; idxMsr++;
1189 }
1190
1191 pMsr->u32IndexMSR = MSR_K6_EFER;
1192 pMsr->u32Reserved = 0;
1193# if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1194 if (CPUMIsGuestInLongMode(pVCpu))
1195 {
1196 /* Must match the efer value in our 64 bits switcher. */
1197 pMsr->u64Value = ASMRdMsr(MSR_K6_EFER) | MSR_K6_EFER_LME | MSR_K6_EFER_SCE | MSR_K6_EFER_NXE;
1198 }
1199 else
1200# endif
1201 pMsr->u64Value = ASMRdMsr(MSR_K6_EFER);
1202 pMsr++; idxMsr++;
1203 }
1204
1205# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1206 if (VMX_IS_64BIT_HOST_MODE())
1207 {
1208 pMsr->u32IndexMSR = MSR_K8_LSTAR;
1209 pMsr->u32Reserved = 0;
1210 pMsr->u64Value = ASMRdMsr(MSR_K8_LSTAR); /* 64 bits mode syscall rip */
1211 pMsr++; idxMsr++;
1212 pMsr->u32IndexMSR = MSR_K8_SF_MASK;
1213 pMsr->u32Reserved = 0;
1214 pMsr->u64Value = ASMRdMsr(MSR_K8_SF_MASK); /* syscall flag mask */
1215 pMsr++; idxMsr++;
1216 pMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
1217 pMsr->u32Reserved = 0;
1218 pMsr->u64Value = ASMRdMsr(MSR_K8_KERNEL_GS_BASE); /* swapgs exchange value */
1219 pMsr++; idxMsr++;
1220 }
1221# endif
1222 rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_MSR_LOAD_COUNT, idxMsr);
1223 AssertRC(rc);
1224#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
1225
1226 pVCpu->hwaccm.s.fContextUseFlags &= ~HWACCM_CHANGED_HOST_CONTEXT;
1227 }
1228 return rc;
1229}
1230
1231/**
1232 * Prefetch the 4 PDPT pointers (PAE and nested paging only)
1233 *
1234 * @param pVM The VM to operate on.
1235 * @param pVCpu The VMCPU to operate on.
1236 * @param pCtx Guest context
1237 */
1238static void vmxR0PrefetchPAEPdptrs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1239{
1240 if (CPUMIsGuestInPAEModeEx(pCtx))
1241 {
1242 X86PDPE Pdpe;
1243
1244 for (unsigned i=0;i<4;i++)
1245 {
1246 Pdpe = PGMGstGetPaePDPtr(pVCpu, i);
1247 int rc = VMXWriteVMCS64(VMX_VMCS_GUEST_PDPTR0_FULL + i*2, Pdpe.u);
1248 AssertRC(rc);
1249 }
1250 }
1251}
1252
1253/**
1254 * Update the exception bitmap according to the current CPU state
1255 *
1256 * @param pVM The VM to operate on.
1257 * @param pVCpu The VMCPU to operate on.
1258 * @param pCtx Guest context
1259 */
1260static void vmxR0UpdateExceptionBitmap(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1261{
1262 uint32_t u32TrapMask;
1263 Assert(pCtx);
1264
1265 u32TrapMask = HWACCM_VMX_TRAP_MASK;
1266#ifndef DEBUG
1267 if (pVM->hwaccm.s.fNestedPaging)
1268 u32TrapMask &= ~RT_BIT(X86_XCPT_PF); /* no longer need to intercept #PF. */
1269#endif
1270
1271 /* Also catch floating point exceptions as we need to report them to the guest in a different way. */
1272 if ( CPUMIsGuestFPUStateActive(pVCpu) == true
1273 && !(pCtx->cr0 & X86_CR0_NE)
1274 && !pVCpu->hwaccm.s.fFPUOldStyleOverride)
1275 {
1276 u32TrapMask |= RT_BIT(X86_XCPT_MF);
1277 pVCpu->hwaccm.s.fFPUOldStyleOverride = true;
1278 }
1279
1280#ifdef VBOX_STRICT
1281 Assert(u32TrapMask & RT_BIT(X86_XCPT_GP));
1282#endif
1283
1284# ifdef HWACCM_VMX_EMULATE_REALMODE
1285 /* Intercept all exceptions in real mode as none of them can be injected directly (#GP otherwise). */
1286 if ( CPUMIsGuestInRealModeEx(pCtx)
1287 && pVM->hwaccm.s.vmx.pRealModeTSS)
1288 u32TrapMask |= HWACCM_VMX_TRAP_MASK_REALMODE;
1289# endif /* HWACCM_VMX_EMULATE_REALMODE */
1290
1291 int rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXCEPTION_BITMAP, u32TrapMask);
1292 AssertRC(rc);
1293}
1294
1295/**
1296 * Loads the guest state
1297 *
1298 * NOTE: Don't do anything here that can cause a jump back to ring 3!!!!!
1299 *
1300 * @returns VBox status code.
1301 * @param pVM The VM to operate on.
1302 * @param pVCpu The VMCPU to operate on.
1303 * @param pCtx Guest context
1304 */
1305VMMR0DECL(int) VMXR0LoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1306{
1307 int rc = VINF_SUCCESS;
1308 RTGCUINTPTR val;
1309 X86EFLAGS eflags;
1310
1311 /* VMX_VMCS_CTRL_ENTRY_CONTROLS
1312 * Set required bits to one and zero according to the MSR capabilities.
1313 */
1314 val = pVM->hwaccm.s.vmx.msr.vmx_entry.n.disallowed0;
1315 /* Load guest debug controls (dr7 & IA32_DEBUGCTL_MSR) (forced to 1 on the 'first' VT-x capable CPUs; this actually includes the newest Nehalem CPUs) */
1316 val |= VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_DEBUG;
1317 /* 64 bits guest mode? */
1318 if (CPUMIsGuestInLongModeEx(pCtx))
1319 val |= VMX_VMCS_CTRL_ENTRY_CONTROLS_IA64_MODE;
1320 /* else Must be zero when AMD64 is not available. */
1321
1322 /* Mask away the bits that the CPU doesn't support */
1323 val &= pVM->hwaccm.s.vmx.msr.vmx_entry.n.allowed1;
1324 rc = VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_CONTROLS, val);
1325 AssertRC(rc);
1326
1327 /* VMX_VMCS_CTRL_EXIT_CONTROLS
1328 * Set required bits to one and zero according to the MSR capabilities.
1329 */
1330 val = pVM->hwaccm.s.vmx.msr.vmx_exit.n.disallowed0;
1331
1332 /* Save debug controls (dr7 & IA32_DEBUGCTL_MSR) (forced to 1 on the 'first' VT-x capable CPUs; this actually includes the newest Nehalem CPUs) */
1333 val |= VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_DEBUG;
1334
1335#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1336 if (VMX_IS_64BIT_HOST_MODE())
1337 val |= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_AMD64;
1338 /* else: Must be zero when AMD64 is not available. */
1339#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
1340 if (CPUMIsGuestInLongModeEx(pCtx))
1341 val |= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_AMD64; /* our switcher goes to long mode */
1342 else
1343 Assert(!(val & VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_AMD64));
1344#endif
1345 val &= pVM->hwaccm.s.vmx.msr.vmx_exit.n.allowed1;
1346 /* Don't acknowledge external interrupts on VM-exit. */
1347 rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_CONTROLS, val);
1348 AssertRC(rc);
1349
1350 /* Guest CPU context: ES, CS, SS, DS, FS, GS. */
1351 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_SEGMENT_REGS)
1352 {
1353#ifdef HWACCM_VMX_EMULATE_REALMODE
1354 if (pVM->hwaccm.s.vmx.pRealModeTSS)
1355 {
1356 PGMMODE enmGuestMode = PGMGetGuestMode(pVCpu);
1357 if (pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode != enmGuestMode)
1358 {
1359 /* Correct weird requirements for switching to protected mode. */
1360 if ( pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode == PGMMODE_REAL
1361 && enmGuestMode >= PGMMODE_PROTECTED)
1362 {
1363 /* Flush the recompiler code cache as it's not unlikely
1364 * the guest will rewrite code it will later execute in real
1365 * mode (OpenBSD 4.0 is one such example)
1366 */
1367 REMFlushTBs(pVM);
1368
1369 /* DPL of all hidden selector registers must match the current CPL (0). */
1370 pCtx->csHid.Attr.n.u2Dpl = 0;
1371 pCtx->csHid.Attr.n.u4Type = X86_SEL_TYPE_CODE | X86_SEL_TYPE_RW_ACC;
1372
1373 pCtx->dsHid.Attr.n.u2Dpl = 0;
1374 pCtx->esHid.Attr.n.u2Dpl = 0;
1375 pCtx->fsHid.Attr.n.u2Dpl = 0;
1376 pCtx->gsHid.Attr.n.u2Dpl = 0;
1377 pCtx->ssHid.Attr.n.u2Dpl = 0;
1378
1379 /* The limit must correspond to the 32 bits setting. */
1380 if (!pCtx->csHid.Attr.n.u1DefBig)
1381 pCtx->csHid.u32Limit &= 0xffff;
1382 if (!pCtx->dsHid.Attr.n.u1DefBig)
1383 pCtx->dsHid.u32Limit &= 0xffff;
1384 if (!pCtx->esHid.Attr.n.u1DefBig)
1385 pCtx->esHid.u32Limit &= 0xffff;
1386 if (!pCtx->fsHid.Attr.n.u1DefBig)
1387 pCtx->fsHid.u32Limit &= 0xffff;
1388 if (!pCtx->gsHid.Attr.n.u1DefBig)
1389 pCtx->gsHid.u32Limit &= 0xffff;
1390 if (!pCtx->ssHid.Attr.n.u1DefBig)
1391 pCtx->ssHid.u32Limit &= 0xffff;
1392 }
1393 else
1394 /* Switching from protected mode to real mode. */
1395 if ( pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode >= PGMMODE_PROTECTED
1396 && enmGuestMode == PGMMODE_REAL)
1397 {
1398 /* The limit must also be set to 0xffff. */
1399 pCtx->csHid.u32Limit = 0xffff;
1400 pCtx->dsHid.u32Limit = 0xffff;
1401 pCtx->esHid.u32Limit = 0xffff;
1402 pCtx->fsHid.u32Limit = 0xffff;
1403 pCtx->gsHid.u32Limit = 0xffff;
1404 pCtx->ssHid.u32Limit = 0xffff;
1405
1406 Assert(pCtx->csHid.u64Base <= 0xfffff);
1407 Assert(pCtx->dsHid.u64Base <= 0xfffff);
1408 Assert(pCtx->esHid.u64Base <= 0xfffff);
1409 Assert(pCtx->fsHid.u64Base <= 0xfffff);
1410 Assert(pCtx->gsHid.u64Base <= 0xfffff);
1411 }
1412 pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode = enmGuestMode;
1413 }
1414 else
1415 /* VT-x will fail with a guest invalid state otherwise... (CPU state after a reset) */
1416 if ( CPUMIsGuestInRealModeEx(pCtx)
1417 && pCtx->csHid.u64Base == 0xffff0000)
1418 {
1419 pCtx->csHid.u64Base = 0xf0000;
1420 pCtx->cs = 0xf000;
1421 }
1422 }
1423#endif /* HWACCM_VMX_EMULATE_REALMODE */
1424
1425 VMX_WRITE_SELREG(ES, es);
1426 AssertRC(rc);
1427
1428 VMX_WRITE_SELREG(CS, cs);
1429 AssertRC(rc);
1430
1431 VMX_WRITE_SELREG(SS, ss);
1432 AssertRC(rc);
1433
1434 VMX_WRITE_SELREG(DS, ds);
1435 AssertRC(rc);
1436
1437 VMX_WRITE_SELREG(FS, fs);
1438 AssertRC(rc);
1439
1440 VMX_WRITE_SELREG(GS, gs);
1441 AssertRC(rc);
1442 }
1443
1444 /* Guest CPU context: LDTR. */
1445 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_LDTR)
1446 {
1447 if (pCtx->ldtr == 0)
1448 {
1449 rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_LDTR, 0);
1450 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_LIMIT, 0);
1451 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_LDTR_BASE, 0);
1452 /* Note: vmlaunch will fail with 0 or just 0x02. No idea why. */
1453 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, 0x82 /* present, LDT */);
1454 }
1455 else
1456 {
1457 rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_LDTR, pCtx->ldtr);
1458 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtrHid.u32Limit);
1459 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_LDTR_BASE, pCtx->ldtrHid.u64Base);
1460 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, pCtx->ldtrHid.Attr.u);
1461 }
1462 AssertRC(rc);
1463 }
1464 /* Guest CPU context: TR. */
1465 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_TR)
1466 {
1467#ifdef HWACCM_VMX_EMULATE_REALMODE
1468 /* Real mode emulation using v86 mode with CR4.VME (interrupt redirection using the int bitmap in the TSS) */
1469 if ( CPUMIsGuestInRealModeEx(pCtx)
1470 && pVM->hwaccm.s.vmx.pRealModeTSS)
1471 {
1472 RTGCPHYS GCPhys;
1473
1474 /* We convert it here every time as pci regions could be reconfigured. */
1475 rc = PDMVMMDevHeapR3ToGCPhys(pVM, pVM->hwaccm.s.vmx.pRealModeTSS, &GCPhys);
1476 AssertRC(rc);
1477
1478 rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_TR, 0);
1479 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_TR_LIMIT, HWACCM_VTX_TSS_SIZE);
1480 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_TR_BASE, GCPhys /* phys = virt in this mode */);
1481
1482 X86DESCATTR attr;
1483
1484 attr.u = 0;
1485 attr.n.u1Present = 1;
1486 attr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
1487 val = attr.u;
1488 }
1489 else
1490#endif /* HWACCM_VMX_EMULATE_REALMODE */
1491 {
1492 rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_TR, pCtx->tr);
1493 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_TR_LIMIT, pCtx->trHid.u32Limit);
1494 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_TR_BASE, pCtx->trHid.u64Base);
1495
1496 val = pCtx->trHid.Attr.u;
1497
1498 /* The TSS selector must be busy. */
1499 if ((val & 0xF) == X86_SEL_TYPE_SYS_286_TSS_AVAIL)
1500 val = (val & ~0xF) | X86_SEL_TYPE_SYS_286_TSS_BUSY;
1501 else
1502 /* Default even if no TR selector has been set (otherwise vmlaunch will fail!) */
1503 val = (val & ~0xF) | X86_SEL_TYPE_SYS_386_TSS_BUSY;
1504
1505 }
1506 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, val);
1507 AssertRC(rc);
1508 }
1509 /* Guest CPU context: GDTR. */
1510 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_GDTR)
1511 {
1512 rc = VMXWriteVMCS(VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt);
1513 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_GDTR_BASE, pCtx->gdtr.pGdt);
1514 AssertRC(rc);
1515 }
1516 /* Guest CPU context: IDTR. */
1517 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_IDTR)
1518 {
1519 rc = VMXWriteVMCS(VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt);
1520 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_IDTR_BASE, pCtx->idtr.pIdt);
1521 AssertRC(rc);
1522 }
1523
1524 /*
1525 * Sysenter MSRs (unconditional)
1526 */
1527 rc = VMXWriteVMCS(VMX_VMCS32_GUEST_SYSENTER_CS, pCtx->SysEnter.cs);
1528 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_SYSENTER_EIP, pCtx->SysEnter.eip);
1529 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_SYSENTER_ESP, pCtx->SysEnter.esp);
1530 AssertRC(rc);
1531
1532 /* Control registers */
1533 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_CR0)
1534 {
1535 val = pCtx->cr0;
1536 rc = VMXWriteVMCS(VMX_VMCS_CTRL_CR0_READ_SHADOW, val);
1537 Log2(("Guest CR0-shadow %08x\n", val));
1538 if (CPUMIsGuestFPUStateActive(pVCpu) == false)
1539 {
1540 /* Always use #NM exceptions to load the FPU/XMM state on demand. */
1541 val |= X86_CR0_TS | X86_CR0_ET | X86_CR0_NE | X86_CR0_MP;
1542 }
1543 else
1544 {
1545 /** @todo check if we support the old style mess correctly. */
1546 if (!(val & X86_CR0_NE))
1547 Log(("Forcing X86_CR0_NE!!!\n"));
1548
1549 val |= X86_CR0_NE; /* always turn on the native mechanism to report FPU errors (old style uses interrupts) */
1550 }
1551 /* Note: protected mode & paging are always enabled; we use them for emulating real and protected mode without paging too. */
1552 if (!pVM->hwaccm.s.vmx.fUnrestrictedGuest)
1553 val |= X86_CR0_PE | X86_CR0_PG;
1554
1555 if (pVM->hwaccm.s.fNestedPaging)
1556 {
1557 if (CPUMIsGuestInPagedProtectedModeEx(pCtx))
1558 {
1559 /* Disable cr3 read/write monitoring as we don't need it for EPT. */
1560 pVCpu->hwaccm.s.vmx.proc_ctls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
1561 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT);
1562 }
1563 else
1564 {
1565 /* Reenable cr3 read/write monitoring as our identity mapped page table is active. */
1566 pVCpu->hwaccm.s.vmx.proc_ctls |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
1567 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
1568 }
1569 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1570 AssertRC(rc);
1571 }
1572 else
1573 {
1574 /* Note: We must also set this as we rely on protecting various pages for which supervisor writes must be caught. */
1575 val |= X86_CR0_WP;
1576 }
1577
1578 /* Always enable caching. */
1579 val &= ~(X86_CR0_CD|X86_CR0_NW);
1580
1581 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_CR0, val);
1582 Log2(("Guest CR0 %08x\n", val));
1583 /* CR0 flags owned by the host; if the guests attempts to change them, then
1584 * the VM will exit.
1585 */
1586 val = X86_CR0_PE /* Must monitor this bit (assumptions are made for real mode emulation) */
1587 | X86_CR0_WP /* Must monitor this bit (it must always be enabled). */
1588 | X86_CR0_PG /* Must monitor this bit (assumptions are made for real mode & protected mode without paging emulation) */
1589 | X86_CR0_TS
1590 | X86_CR0_ET /* Bit not restored during VM-exit! */
1591 | X86_CR0_CD /* Bit not restored during VM-exit! */
1592 | X86_CR0_NW /* Bit not restored during VM-exit! */
1593 | X86_CR0_NE
1594 | X86_CR0_MP;
1595 pVCpu->hwaccm.s.vmx.cr0_mask = val;
1596
1597 rc |= VMXWriteVMCS(VMX_VMCS_CTRL_CR0_MASK, val);
1598 Log2(("Guest CR0-mask %08x\n", val));
1599 AssertRC(rc);
1600 }
1601 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_CR4)
1602 {
1603 /* CR4 */
1604 rc = VMXWriteVMCS(VMX_VMCS_CTRL_CR4_READ_SHADOW, pCtx->cr4);
1605 Log2(("Guest CR4-shadow %08x\n", pCtx->cr4));
1606 /* Set the required bits in cr4 too (currently X86_CR4_VMXE). */
1607 val = pCtx->cr4 | (uint32_t)pVM->hwaccm.s.vmx.msr.vmx_cr4_fixed0;
1608
1609 if (!pVM->hwaccm.s.fNestedPaging)
1610 {
1611 switch(pVCpu->hwaccm.s.enmShadowMode)
1612 {
1613 case PGMMODE_REAL: /* Real mode -> emulated using v86 mode */
1614 case PGMMODE_PROTECTED: /* Protected mode, no paging -> emulated using identity mapping. */
1615 case PGMMODE_32_BIT: /* 32-bit paging. */
1616 val &= ~X86_CR4_PAE;
1617 break;
1618
1619 case PGMMODE_PAE: /* PAE paging. */
1620 case PGMMODE_PAE_NX: /* PAE paging with NX enabled. */
1621 /** @todo use normal 32 bits paging */
1622 val |= X86_CR4_PAE;
1623 break;
1624
1625 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
1626 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
1627#ifdef VBOX_ENABLE_64_BITS_GUESTS
1628 break;
1629#else
1630 AssertFailed();
1631 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1632#endif
1633 default: /* shut up gcc */
1634 AssertFailed();
1635 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1636 }
1637 }
1638 else
1639 if ( !CPUMIsGuestInPagedProtectedModeEx(pCtx)
1640 && !pVM->hwaccm.s.vmx.fUnrestrictedGuest)
1641 {
1642 /* We use 4 MB pages in our identity mapping page table for real and protected mode without paging. */
1643 val |= X86_CR4_PSE;
1644 /* Our identity mapping is a 32 bits page directory. */
1645 val &= ~X86_CR4_PAE;
1646 }
1647
1648#ifdef HWACCM_VMX_EMULATE_REALMODE
1649 /* Turn off VME if we're in emulated real mode. */
1650 if ( CPUMIsGuestInRealModeEx(pCtx)
1651 && pVM->hwaccm.s.vmx.pRealModeTSS)
1652 val &= ~X86_CR4_VME;
1653#endif /* HWACCM_VMX_EMULATE_REALMODE */
1654
1655 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_CR4, val);
1656 Log2(("Guest CR4 %08x\n", val));
1657 /* CR4 flags owned by the host; if the guests attempts to change them, then
1658 * the VM will exit.
1659 */
1660 val = 0
1661 | X86_CR4_VME
1662 | X86_CR4_PAE
1663 | X86_CR4_PGE
1664 | X86_CR4_PSE
1665 | X86_CR4_VMXE;
1666 pVCpu->hwaccm.s.vmx.cr4_mask = val;
1667
1668 rc |= VMXWriteVMCS(VMX_VMCS_CTRL_CR4_MASK, val);
1669 Log2(("Guest CR4-mask %08x\n", val));
1670 AssertRC(rc);
1671 }
1672
1673 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_CR3)
1674 {
1675 if (pVM->hwaccm.s.fNestedPaging)
1676 {
1677 Assert(PGMGetHyperCR3(pVCpu));
1678 pVCpu->hwaccm.s.vmx.GCPhysEPTP = PGMGetHyperCR3(pVCpu);
1679
1680 Assert(!(pVCpu->hwaccm.s.vmx.GCPhysEPTP & 0xfff));
1681 /** @todo Check the IA32_VMX_EPT_VPID_CAP MSR for other supported memory types. */
1682 pVCpu->hwaccm.s.vmx.GCPhysEPTP |= VMX_EPT_MEMTYPE_WB
1683 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
1684
1685 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_EPTP_FULL, pVCpu->hwaccm.s.vmx.GCPhysEPTP);
1686 AssertRC(rc);
1687
1688 if ( !CPUMIsGuestInPagedProtectedModeEx(pCtx)
1689 && !pVM->hwaccm.s.vmx.fUnrestrictedGuest)
1690 {
1691 RTGCPHYS GCPhys;
1692
1693 /* We convert it here every time as pci regions could be reconfigured. */
1694 rc = PDMVMMDevHeapR3ToGCPhys(pVM, pVM->hwaccm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
1695 AssertMsgRC(rc, ("pNonPagingModeEPTPageTable = %RGv\n", pVM->hwaccm.s.vmx.pNonPagingModeEPTPageTable));
1696
1697 /* We use our identity mapping page table here as we need to map guest virtual to guest physical addresses; EPT will
1698 * take care of the translation to host physical addresses.
1699 */
1700 val = GCPhys;
1701 }
1702 else
1703 {
1704 /* Save the real guest CR3 in VMX_VMCS_GUEST_CR3 */
1705 val = pCtx->cr3;
1706 /* Prefetch the four PDPT entries in PAE mode. */
1707 vmxR0PrefetchPAEPdptrs(pVM, pVCpu, pCtx);
1708 }
1709 }
1710 else
1711 {
1712 val = PGMGetHyperCR3(pVCpu);
1713 Assert(val || VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL));
1714 }
1715
1716 /* Save our shadow CR3 register. */
1717 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_CR3, val);
1718 AssertRC(rc);
1719 }
1720
1721 /* Debug registers. */
1722 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_DEBUG)
1723 {
1724 pCtx->dr[6] |= X86_DR6_INIT_VAL; /* set all reserved bits to 1. */
1725 pCtx->dr[6] &= ~RT_BIT(12); /* must be zero. */
1726
1727 pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
1728 pCtx->dr[7] &= ~(RT_BIT(11) | RT_BIT(12) | RT_BIT(14) | RT_BIT(15)); /* must be zero */
1729 pCtx->dr[7] |= 0x400; /* must be one */
1730
1731 /* Resync DR7 */
1732 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_DR7, pCtx->dr[7]);
1733 AssertRC(rc);
1734
1735#ifdef DEBUG
1736 /* Sync the hypervisor debug state now if any breakpoint is armed. */
1737 if ( CPUMGetHyperDR7(pVCpu) & (X86_DR7_ENABLED_MASK|X86_DR7_GD)
1738 && !CPUMIsHyperDebugStateActive(pVCpu)
1739 && !DBGFIsStepping(pVCpu))
1740 {
1741 /* Save the host and load the hypervisor debug state. */
1742 rc = CPUMR0LoadHyperDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
1743 AssertRC(rc);
1744
1745 /* DRx intercepts remain enabled. */
1746
1747 /* Override dr7 with the hypervisor value. */
1748 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_DR7, CPUMGetHyperDR7(pVCpu));
1749 AssertRC(rc);
1750 }
1751 else
1752#endif
1753 /* Sync the debug state now if any breakpoint is armed. */
1754 if ( (pCtx->dr[7] & (X86_DR7_ENABLED_MASK|X86_DR7_GD))
1755 && !CPUMIsGuestDebugStateActive(pVCpu)
1756 && !DBGFIsStepping(pVCpu))
1757 {
1758 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatDRxArmed);
1759
1760 /* Disable drx move intercepts. */
1761 pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
1762 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1763 AssertRC(rc);
1764
1765 /* Save the host and load the guest debug state. */
1766 rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
1767 AssertRC(rc);
1768 }
1769
1770 /* IA32_DEBUGCTL MSR. */
1771 rc = VMXWriteVMCS64(VMX_VMCS_GUEST_DEBUGCTL_FULL, 0);
1772 AssertRC(rc);
1773
1774 /** @todo do we really ever need this? */
1775 rc |= VMXWriteVMCS(VMX_VMCS_GUEST_DEBUG_EXCEPTIONS, 0);
1776 AssertRC(rc);
1777 }
1778
1779 /* EIP, ESP and EFLAGS */
1780 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_RIP, pCtx->rip);
1781 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_RSP, pCtx->rsp);
1782 AssertRC(rc);
1783
1784 /* Bits 22-31, 15, 5 & 3 must be zero. Bit 1 must be 1. */
1785 eflags = pCtx->eflags;
1786 eflags.u32 &= VMX_EFLAGS_RESERVED_0;
1787 eflags.u32 |= VMX_EFLAGS_RESERVED_1;
1788
1789#ifdef HWACCM_VMX_EMULATE_REALMODE
1790 /* Real mode emulation using v86 mode. */
1791 if ( CPUMIsGuestInRealModeEx(pCtx)
1792 && pVM->hwaccm.s.vmx.pRealModeTSS)
1793 {
1794 pVCpu->hwaccm.s.vmx.RealMode.eflags = eflags;
1795
1796 eflags.Bits.u1VM = 1;
1797 eflags.Bits.u2IOPL = 0; /* must always be 0 or else certain instructions won't cause faults. */
1798 }
1799#endif /* HWACCM_VMX_EMULATE_REALMODE */
1800 rc = VMXWriteVMCS(VMX_VMCS_GUEST_RFLAGS, eflags.u32);
1801 AssertRC(rc);
1802
1803 if (TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hwaccm.s.vmx.u64TSCOffset))
1804 {
1805 uint64_t u64CurTSC = ASMReadTSC();
1806 if (u64CurTSC + pVCpu->hwaccm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
1807 {
1808 /* Note: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT takes precedence over TSC_OFFSET */
1809 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_TSC_OFFSET_FULL, pVCpu->hwaccm.s.vmx.u64TSCOffset);
1810 AssertRC(rc);
1811
1812 pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
1813 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1814 AssertRC(rc);
1815 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTSCOffset);
1816 }
1817 else
1818 {
1819 /* Fall back to rdtsc emulation as we would otherwise pass decreasing tsc values to the guest. */
1820 LogFlow(("TSC %RX64 offset %RX64 time=%RX64 last=%RX64 (diff=%RX64, virt_tsc=%RX64)\n", u64CurTSC, pVCpu->hwaccm.s.vmx.u64TSCOffset, u64CurTSC + pVCpu->hwaccm.s.vmx.u64TSCOffset, TMCpuTickGetLastSeen(pVCpu), TMCpuTickGetLastSeen(pVCpu) - u64CurTSC - pVCpu->hwaccm.s.vmx.u64TSCOffset, TMCpuTickGet(pVCpu)));
1821 pVCpu->hwaccm.s.vmx.proc_ctls |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
1822 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1823 AssertRC(rc);
1824 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTSCInterceptOverFlow);
1825 }
1826 }
1827 else
1828 {
1829 pVCpu->hwaccm.s.vmx.proc_ctls |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
1830 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1831 AssertRC(rc);
1832 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTSCIntercept);
1833 }
1834
1835 /* 64 bits guest mode? */
1836 if (CPUMIsGuestInLongModeEx(pCtx))
1837 {
1838#if !defined(VBOX_ENABLE_64_BITS_GUESTS)
1839 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1840#elif HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1841 pVCpu->hwaccm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
1842#else
1843# ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1844 if (!pVM->hwaccm.s.fAllow64BitGuests)
1845 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1846# endif
1847 pVCpu->hwaccm.s.vmx.pfnStartVM = VMXR0StartVM64;
1848#endif
1849 /* Unconditionally update these as wrmsr might have changed them. */
1850 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_FS_BASE, pCtx->fsHid.u64Base);
1851 AssertRC(rc);
1852 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_GS_BASE, pCtx->gsHid.u64Base);
1853 AssertRC(rc);
1854 }
1855 else
1856 {
1857 pVCpu->hwaccm.s.vmx.pfnStartVM = VMXR0StartVM32;
1858 }
1859
1860 vmxR0UpdateExceptionBitmap(pVM, pVCpu, pCtx);
1861
1862#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
1863 /* Store all guest MSRs in the VM-Entry load area, so they will be loaded during the world switch. */
1864 PVMXMSR pMsr = (PVMXMSR)pVCpu->hwaccm.s.vmx.pGuestMSR;
1865 unsigned idxMsr = 0;
1866
1867 uint32_t ulEdx;
1868 uint32_t ulTemp;
1869 CPUMGetGuestCpuId(pVCpu, 0x80000001, &ulTemp, &ulTemp, &ulTemp, &ulEdx);
1870 /* EFER MSR present? */
1871 if (ulEdx & (X86_CPUID_AMD_FEATURE_EDX_NX|X86_CPUID_AMD_FEATURE_EDX_LONG_MODE))
1872 {
1873 pMsr->u32IndexMSR = MSR_K6_EFER;
1874 pMsr->u32Reserved = 0;
1875 pMsr->u64Value = pCtx->msrEFER;
1876 /* VT-x will complain if only MSR_K6_EFER_LME is set. */
1877 if (!CPUMIsGuestInLongModeEx(pCtx))
1878 pMsr->u64Value &= ~(MSR_K6_EFER_LMA|MSR_K6_EFER_LME);
1879 pMsr++; idxMsr++;
1880
1881 if (ulEdx & X86_CPUID_AMD_FEATURE_EDX_LONG_MODE)
1882 {
1883 pMsr->u32IndexMSR = MSR_K8_LSTAR;
1884 pMsr->u32Reserved = 0;
1885 pMsr->u64Value = pCtx->msrLSTAR; /* 64 bits mode syscall rip */
1886 pMsr++; idxMsr++;
1887 pMsr->u32IndexMSR = MSR_K6_STAR;
1888 pMsr->u32Reserved = 0;
1889 pMsr->u64Value = pCtx->msrSTAR; /* legacy syscall eip, cs & ss */
1890 pMsr++; idxMsr++;
1891 pMsr->u32IndexMSR = MSR_K8_SF_MASK;
1892 pMsr->u32Reserved = 0;
1893 pMsr->u64Value = pCtx->msrSFMASK; /* syscall flag mask */
1894 pMsr++; idxMsr++;
1895 pMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
1896 pMsr->u32Reserved = 0;
1897 pMsr->u64Value = pCtx->msrKERNELGSBASE; /* swapgs exchange value */
1898 pMsr++; idxMsr++;
1899 }
1900 }
1901 pVCpu->hwaccm.s.vmx.cCachedMSRs = idxMsr;
1902
1903 rc = VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_MSR_LOAD_COUNT, idxMsr);
1904 AssertRC(rc);
1905
1906 rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_MSR_STORE_COUNT, idxMsr);
1907 AssertRC(rc);
1908#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
1909
1910 /* Done. */
1911 pVCpu->hwaccm.s.fContextUseFlags &= ~HWACCM_CHANGED_ALL_GUEST;
1912
1913 return rc;
1914}
1915
1916/**
1917 * Syncs back the guest state
1918 *
1919 * @returns VBox status code.
1920 * @param pVM The VM to operate on.
1921 * @param pVCpu The VMCPU to operate on.
1922 * @param pCtx Guest context
1923 */
1924DECLINLINE(int) VMXR0SaveGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1925{
1926 RTGCUINTREG val, valShadow;
1927 RTGCUINTPTR uInterruptState;
1928 int rc;
1929
1930 /* Let's first sync back eip, esp, and eflags. */
1931 rc = VMXReadCachedVMCS(VMX_VMCS64_GUEST_RIP, &val);
1932 AssertRC(rc);
1933 pCtx->rip = val;
1934 rc = VMXReadCachedVMCS(VMX_VMCS64_GUEST_RSP, &val);
1935 AssertRC(rc);
1936 pCtx->rsp = val;
1937 rc = VMXReadCachedVMCS(VMX_VMCS_GUEST_RFLAGS, &val);
1938 AssertRC(rc);
1939 pCtx->eflags.u32 = val;
1940
1941 /* Take care of instruction fusing (sti, mov ss) */
1942 rc |= VMXReadCachedVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &val);
1943 uInterruptState = val;
1944 if (uInterruptState != 0)
1945 {
1946 Assert(uInterruptState <= 2); /* only sti & mov ss */
1947 Log(("uInterruptState %x eip=%RGv\n", (uint32_t)uInterruptState, pCtx->rip));
1948 EMSetInhibitInterruptsPC(pVCpu, pCtx->rip);
1949 }
1950 else
1951 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
1952
1953 /* Control registers. */
1954 VMXReadCachedVMCS(VMX_VMCS_CTRL_CR0_READ_SHADOW, &valShadow);
1955 VMXReadCachedVMCS(VMX_VMCS64_GUEST_CR0, &val);
1956 val = (valShadow & pVCpu->hwaccm.s.vmx.cr0_mask) | (val & ~pVCpu->hwaccm.s.vmx.cr0_mask);
1957 CPUMSetGuestCR0(pVCpu, val);
1958
1959 VMXReadCachedVMCS(VMX_VMCS_CTRL_CR4_READ_SHADOW, &valShadow);
1960 VMXReadCachedVMCS(VMX_VMCS64_GUEST_CR4, &val);
1961 val = (valShadow & pVCpu->hwaccm.s.vmx.cr4_mask) | (val & ~pVCpu->hwaccm.s.vmx.cr4_mask);
1962 CPUMSetGuestCR4(pVCpu, val);
1963
1964 /* Note: no reason to sync back the CRx registers. They can't be changed by the guest. */
1965 /* Note: only in the nested paging case can CR3 & CR4 be changed by the guest. */
1966 if ( pVM->hwaccm.s.fNestedPaging
1967 && CPUMIsGuestInPagedProtectedModeEx(pCtx))
1968 {
1969 PVMCSCACHE pCache = &pVCpu->hwaccm.s.vmx.VMCSCache;
1970
1971 /* Can be updated behind our back in the nested paging case. */
1972 CPUMSetGuestCR2(pVCpu, pCache->cr2);
1973
1974 VMXReadCachedVMCS(VMX_VMCS64_GUEST_CR3, &val);
1975
1976 if (val != pCtx->cr3)
1977 {
1978 CPUMSetGuestCR3(pVCpu, val);
1979 PGMUpdateCR3(pVCpu, val);
1980 }
1981 /* Prefetch the four PDPT entries in PAE mode. */
1982 vmxR0PrefetchPAEPdptrs(pVM, pVCpu, pCtx);
1983 }
1984
1985 /* Sync back DR7 here. */
1986 VMXReadCachedVMCS(VMX_VMCS64_GUEST_DR7, &val);
1987 pCtx->dr[7] = val;
1988
1989 /* Guest CPU context: ES, CS, SS, DS, FS, GS. */
1990 VMX_READ_SELREG(ES, es);
1991 VMX_READ_SELREG(SS, ss);
1992 VMX_READ_SELREG(CS, cs);
1993 VMX_READ_SELREG(DS, ds);
1994 VMX_READ_SELREG(FS, fs);
1995 VMX_READ_SELREG(GS, gs);
1996
1997 /*
1998 * System MSRs
1999 */
2000 VMXReadCachedVMCS(VMX_VMCS32_GUEST_SYSENTER_CS, &val);
2001 pCtx->SysEnter.cs = val;
2002 VMXReadCachedVMCS(VMX_VMCS64_GUEST_SYSENTER_EIP, &val);
2003 pCtx->SysEnter.eip = val;
2004 VMXReadCachedVMCS(VMX_VMCS64_GUEST_SYSENTER_ESP, &val);
2005 pCtx->SysEnter.esp = val;
2006
2007 /* Misc. registers; must sync everything otherwise we can get out of sync when jumping to ring 3. */
2008 VMX_READ_SELREG(LDTR, ldtr);
2009
2010 VMXReadCachedVMCS(VMX_VMCS32_GUEST_GDTR_LIMIT, &val);
2011 pCtx->gdtr.cbGdt = val;
2012 VMXReadCachedVMCS(VMX_VMCS64_GUEST_GDTR_BASE, &val);
2013 pCtx->gdtr.pGdt = val;
2014
2015 VMXReadCachedVMCS(VMX_VMCS32_GUEST_IDTR_LIMIT, &val);
2016 pCtx->idtr.cbIdt = val;
2017 VMXReadCachedVMCS(VMX_VMCS64_GUEST_IDTR_BASE, &val);
2018 pCtx->idtr.pIdt = val;
2019
2020#ifdef HWACCM_VMX_EMULATE_REALMODE
2021 /* Real mode emulation using v86 mode. */
2022 if ( CPUMIsGuestInRealModeEx(pCtx)
2023 && pVM->hwaccm.s.vmx.pRealModeTSS)
2024 {
2025 /* Hide our emulation flags */
2026 pCtx->eflags.Bits.u1VM = 0;
2027
2028 /* Restore original IOPL setting as we always use 0. */
2029 pCtx->eflags.Bits.u2IOPL = pVCpu->hwaccm.s.vmx.RealMode.eflags.Bits.u2IOPL;
2030
2031 /* Force a TR resync every time in case we switch modes. */
2032 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_TR;
2033 }
2034 else
2035#endif /* HWACCM_VMX_EMULATE_REALMODE */
2036 {
2037 /* In real mode we have a fake TSS, so only sync it back when it's supposed to be valid. */
2038 VMX_READ_SELREG(TR, tr);
2039 }
2040
2041#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
2042 /* Save the possibly changed MSRs that we automatically restore and save during a world switch. */
2043 for (unsigned i = 0; i < pVCpu->hwaccm.s.vmx.cCachedMSRs; i++)
2044 {
2045 PVMXMSR pMsr = (PVMXMSR)pVCpu->hwaccm.s.vmx.pGuestMSR;
2046 pMsr += i;
2047
2048 switch (pMsr->u32IndexMSR)
2049 {
2050 case MSR_K8_LSTAR:
2051 pCtx->msrLSTAR = pMsr->u64Value;
2052 break;
2053 case MSR_K6_STAR:
2054 pCtx->msrSTAR = pMsr->u64Value;
2055 break;
2056 case MSR_K8_SF_MASK:
2057 pCtx->msrSFMASK = pMsr->u64Value;
2058 break;
2059 case MSR_K8_KERNEL_GS_BASE:
2060 pCtx->msrKERNELGSBASE = pMsr->u64Value;
2061 break;
2062 case MSR_K6_EFER:
2063 /* EFER can't be changed without causing a VM-exit. */
2064// Assert(pCtx->msrEFER == pMsr->u64Value);
2065 break;
2066 default:
2067 AssertFailed();
2068 return VERR_INTERNAL_ERROR;
2069 }
2070 }
2071#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
2072 return VINF_SUCCESS;
2073}
2074
2075/**
2076 * Dummy placeholder
2077 *
2078 * @param pVM The VM to operate on.
2079 * @param pVCpu The VMCPU to operate on.
2080 */
2081static void vmxR0SetupTLBDummy(PVM pVM, PVMCPU pVCpu)
2082{
2083 NOREF(pVM);
2084 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
2085 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2086 pVCpu->hwaccm.s.TlbShootdown.cPages = 0;
2087 return;
2088}
2089
2090/**
2091 * Setup the tagged TLB for EPT
2092 *
2093 * @returns VBox status code.
2094 * @param pVM The VM to operate on.
2095 * @param pVCpu The VMCPU to operate on.
2096 */
2097static void vmxR0SetupTLBEPT(PVM pVM, PVMCPU pVCpu)
2098{
2099 PHWACCM_CPUINFO pCpu;
2100
2101 Assert(pVM->hwaccm.s.fNestedPaging);
2102 Assert(!pVM->hwaccm.s.vmx.fVPID);
2103
2104 /* Deal with tagged TLBs if VPID or EPT is supported. */
2105 pCpu = HWACCMR0GetCurrentCpu();
2106 /* Force a TLB flush for the first world switch if the current cpu differs from the one we ran on last. */
2107 /* Note that this can happen both for start and resume due to long jumps back to ring 3. */
2108 if ( pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu
2109 /* if the tlb flush count has changed, another VM has flushed the TLB of this cpu, so we can't use our current ASID anymore. */
2110 || pVCpu->hwaccm.s.cTLBFlushes != pCpu->cTLBFlushes)
2111 {
2112 /* Force a TLB flush on VM entry. */
2113 pVCpu->hwaccm.s.fForceTLBFlush = true;
2114 }
2115 else
2116 Assert(!pCpu->fFlushTLB);
2117
2118 /* Check for tlb shootdown flushes. */
2119 if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2120 pVCpu->hwaccm.s.fForceTLBFlush = true;
2121
2122 pVCpu->hwaccm.s.idLastCpu = pCpu->idCpu;
2123 pCpu->fFlushTLB = false;
2124
2125 if (pVCpu->hwaccm.s.fForceTLBFlush)
2126 {
2127 vmxR0FlushEPT(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushContext, 0);
2128 }
2129 else
2130 if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2131 {
2132 /* Deal with pending TLB shootdown actions which were queued when we were not executing code. */
2133 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTlbShootdown);
2134
2135 for (unsigned i=0;i<pVCpu->hwaccm.s.TlbShootdown.cPages;i++)
2136 {
2137 /* aTlbShootdownPages contains physical addresses in this case. */
2138 vmxR0FlushEPT(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushPage, pVCpu->hwaccm.s.TlbShootdown.aPages[i]);
2139 }
2140 }
2141 pVCpu->hwaccm.s.TlbShootdown.cPages= 0;
2142 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2143
2144#ifdef VBOX_WITH_STATISTICS
2145 if (pVCpu->hwaccm.s.fForceTLBFlush)
2146 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushTLBWorldSwitch);
2147 else
2148 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatNoFlushTLBWorldSwitch);
2149#endif
2150}
2151
2152#ifdef HWACCM_VTX_WITH_VPID
2153/**
2154 * Setup the tagged TLB for VPID
2155 *
2156 * @returns VBox status code.
2157 * @param pVM The VM to operate on.
2158 * @param pVCpu The VMCPU to operate on.
2159 */
2160static void vmxR0SetupTLBVPID(PVM pVM, PVMCPU pVCpu)
2161{
2162 PHWACCM_CPUINFO pCpu;
2163
2164 Assert(pVM->hwaccm.s.vmx.fVPID);
2165 Assert(!pVM->hwaccm.s.fNestedPaging);
2166
2167 /* Deal with tagged TLBs if VPID or EPT is supported. */
2168 pCpu = HWACCMR0GetCurrentCpu();
2169 /* Force a TLB flush for the first world switch if the current cpu differs from the one we ran on last. */
2170 /* Note that this can happen both for start and resume due to long jumps back to ring 3. */
2171 if ( pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu
2172 /* if the tlb flush count has changed, another VM has flushed the TLB of this cpu, so we can't use our current ASID anymore. */
2173 || pVCpu->hwaccm.s.cTLBFlushes != pCpu->cTLBFlushes)
2174 {
2175 /* Force a TLB flush on VM entry. */
2176 pVCpu->hwaccm.s.fForceTLBFlush = true;
2177 }
2178 else
2179 Assert(!pCpu->fFlushTLB);
2180
2181 pVCpu->hwaccm.s.idLastCpu = pCpu->idCpu;
2182
2183 /* Check for tlb shootdown flushes. */
2184 if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2185 pVCpu->hwaccm.s.fForceTLBFlush = true;
2186
2187 /* Make sure we flush the TLB when required. Switch ASID to achieve the same thing, but without actually flushing the whole TLB (which is expensive). */
2188 if (pVCpu->hwaccm.s.fForceTLBFlush)
2189 {
2190 if ( ++pCpu->uCurrentASID >= pVM->hwaccm.s.uMaxASID
2191 || pCpu->fFlushTLB)
2192 {
2193 pCpu->fFlushTLB = false;
2194 pCpu->uCurrentASID = 1; /* start at 1; host uses 0 */
2195 pCpu->cTLBFlushes++;
2196 vmxR0FlushVPID(pVM, pVCpu, VMX_FLUSH_ALL_CONTEXTS, 0);
2197 }
2198 else
2199 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushASID);
2200
2201 pVCpu->hwaccm.s.fForceTLBFlush = false;
2202 pVCpu->hwaccm.s.cTLBFlushes = pCpu->cTLBFlushes;
2203 pVCpu->hwaccm.s.uCurrentASID = pCpu->uCurrentASID;
2204 }
2205 else
2206 {
2207 Assert(!pCpu->fFlushTLB);
2208 Assert(pVCpu->hwaccm.s.uCurrentASID && pCpu->uCurrentASID);
2209
2210 if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2211 {
2212 /* Deal with pending TLB shootdown actions which were queued when we were not executing code. */
2213 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTlbShootdown);
2214 for (unsigned i=0;i<pVCpu->hwaccm.s.TlbShootdown.cPages;i++)
2215 vmxR0FlushVPID(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushPage, pVCpu->hwaccm.s.TlbShootdown.aPages[i]);
2216 }
2217 }
2218 pVCpu->hwaccm.s.TlbShootdown.cPages = 0;
2219 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2220
2221 AssertMsg(pVCpu->hwaccm.s.cTLBFlushes == pCpu->cTLBFlushes, ("Flush count mismatch for cpu %d (%x vs %x)\n", pCpu->idCpu, pVCpu->hwaccm.s.cTLBFlushes, pCpu->cTLBFlushes));
2222 AssertMsg(pCpu->uCurrentASID >= 1 && pCpu->uCurrentASID < pVM->hwaccm.s.uMaxASID, ("cpu%d uCurrentASID = %x\n", pCpu->idCpu, pCpu->uCurrentASID));
2223 AssertMsg(pVCpu->hwaccm.s.uCurrentASID >= 1 && pVCpu->hwaccm.s.uCurrentASID < pVM->hwaccm.s.uMaxASID, ("cpu%d VM uCurrentASID = %x\n", pCpu->idCpu, pVCpu->hwaccm.s.uCurrentASID));
2224
2225 int rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hwaccm.s.uCurrentASID);
2226 AssertRC(rc);
2227
2228 if (pVCpu->hwaccm.s.fForceTLBFlush)
2229 vmxR0FlushVPID(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushContext, 0);
2230
2231#ifdef VBOX_WITH_STATISTICS
2232 if (pVCpu->hwaccm.s.fForceTLBFlush)
2233 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushTLBWorldSwitch);
2234 else
2235 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatNoFlushTLBWorldSwitch);
2236#endif
2237}
2238#endif /* HWACCM_VTX_WITH_VPID */
2239
2240/**
2241 * Runs guest code in a VT-x VM.
2242 *
2243 * @returns VBox status code.
2244 * @param pVM The VM to operate on.
2245 * @param pVCpu The VMCPU to operate on.
2246 * @param pCtx Guest context
2247 */
2248VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2249{
2250 int rc = VINF_SUCCESS;
2251 RTGCUINTREG val;
2252 RTGCUINTREG exitReason = (RTGCUINTREG)VMX_EXIT_INVALID;
2253 RTGCUINTREG instrError, cbInstr;
2254 RTGCUINTPTR exitQualification = 0;
2255 RTGCUINTPTR intInfo = 0; /* shut up buggy gcc 4 */
2256 RTGCUINTPTR errCode, instrInfo;
2257 bool fSetupTPRCaching = false;
2258 uint64_t u64OldLSTAR = 0;
2259 uint8_t u8LastTPR = 0;
2260 RTCCUINTREG uOldEFlags = ~(RTCCUINTREG)0;
2261 unsigned cResume = 0;
2262#ifdef VBOX_STRICT
2263 RTCPUID idCpuCheck;
2264 bool fWasInLongMode = false;
2265#endif
2266#ifdef VBOX_HIGH_RES_TIMERS_HACK_IN_RING0
2267 uint64_t u64LastTime = RTTimeMilliTS();
2268#endif
2269#ifdef VBOX_WITH_STATISTICS
2270 bool fStatEntryStarted = true;
2271 bool fStatExit2Started = false;
2272#endif
2273
2274 Assert(!(pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC) || (pVCpu->hwaccm.s.vmx.pVAPIC && pVM->hwaccm.s.vmx.pAPIC));
2275
2276 /* Check if we need to use TPR shadowing. */
2277 if ( CPUMIsGuestInLongModeEx(pCtx)
2278 || ( ((pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC) || pVM->hwaccm.s.fTRPPatchingAllowed)
2279 && pVM->hwaccm.s.fHasIoApic)
2280 )
2281 {
2282 fSetupTPRCaching = true;
2283 }
2284
2285 Log2(("\nE"));
2286
2287 STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatEntry, x);
2288
2289#ifdef VBOX_STRICT
2290 {
2291 RTCCUINTREG val2;
2292
2293 rc = VMXReadVMCS(VMX_VMCS_CTRL_PIN_EXEC_CONTROLS, &val2);
2294 AssertRC(rc);
2295 Log2(("VMX_VMCS_CTRL_PIN_EXEC_CONTROLS = %08x\n", val2));
2296
2297 /* allowed zero */
2298 if ((val2 & pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.disallowed0)
2299 Log(("Invalid VMX_VMCS_CTRL_PIN_EXEC_CONTROLS: zero\n"));
2300
2301 /* allowed one */
2302 if ((val2 & ~pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.allowed1) != 0)
2303 Log(("Invalid VMX_VMCS_CTRL_PIN_EXEC_CONTROLS: one\n"));
2304
2305 rc = VMXReadVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, &val2);
2306 AssertRC(rc);
2307 Log2(("VMX_VMCS_CTRL_PROC_EXEC_CONTROLS = %08x\n", val2));
2308
2309 /* Must be set according to the MSR, but can be cleared in case of EPT. */
2310 if (pVM->hwaccm.s.fNestedPaging)
2311 val2 |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT
2312 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
2313 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
2314
2315 /* allowed zero */
2316 if ((val2 & pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.disallowed0)
2317 Log(("Invalid VMX_VMCS_CTRL_PROC_EXEC_CONTROLS: zero\n"));
2318
2319 /* allowed one */
2320 if ((val2 & ~pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1) != 0)
2321 Log(("Invalid VMX_VMCS_CTRL_PROC_EXEC_CONTROLS: one\n"));
2322
2323 rc = VMXReadVMCS(VMX_VMCS_CTRL_ENTRY_CONTROLS, &val2);
2324 AssertRC(rc);
2325 Log2(("VMX_VMCS_CTRL_ENTRY_CONTROLS = %08x\n", val2));
2326
2327 /* allowed zero */
2328 if ((val2 & pVM->hwaccm.s.vmx.msr.vmx_entry.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_entry.n.disallowed0)
2329 Log(("Invalid VMX_VMCS_CTRL_ENTRY_CONTROLS: zero\n"));
2330
2331 /* allowed one */
2332 if ((val2 & ~pVM->hwaccm.s.vmx.msr.vmx_entry.n.allowed1) != 0)
2333 Log(("Invalid VMX_VMCS_CTRL_ENTRY_CONTROLS: one\n"));
2334
2335 rc = VMXReadVMCS(VMX_VMCS_CTRL_EXIT_CONTROLS, &val2);
2336 AssertRC(rc);
2337 Log2(("VMX_VMCS_CTRL_EXIT_CONTROLS = %08x\n", val2));
2338
2339 /* allowed zero */
2340 if ((val2 & pVM->hwaccm.s.vmx.msr.vmx_exit.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_exit.n.disallowed0)
2341 Log(("Invalid VMX_VMCS_CTRL_EXIT_CONTROLS: zero\n"));
2342
2343 /* allowed one */
2344 if ((val2 & ~pVM->hwaccm.s.vmx.msr.vmx_exit.n.allowed1) != 0)
2345 Log(("Invalid VMX_VMCS_CTRL_EXIT_CONTROLS: one\n"));
2346 }
2347 fWasInLongMode = CPUMIsGuestInLongModeEx(pCtx);
2348#endif /* VBOX_STRICT */
2349
2350#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2351 pVCpu->hwaccm.s.vmx.VMCSCache.u64TimeEntry = RTTimeNanoTS();
2352#endif
2353
2354 /* We can jump to this point to resume execution after determining that a VM-exit is innocent.
2355 */
2356ResumeExecution:
2357 STAM_STATS({
2358 if (fStatExit2Started) { STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2, y); fStatExit2Started = false; }
2359 if (!fStatEntryStarted) { STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatEntry, x); fStatEntryStarted = true; }
2360 });
2361 AssertMsg(pVCpu->hwaccm.s.idEnteredCpu == RTMpCpuId(),
2362 ("Expected %d, I'm %d; cResume=%d exitReason=%RGv exitQualification=%RGv\n",
2363 (int)pVCpu->hwaccm.s.idEnteredCpu, (int)RTMpCpuId(), cResume, exitReason, exitQualification));
2364 Assert(!HWACCMR0SuspendPending());
2365 /* Not allowed to switch modes without reloading the host state (32->64 switcher)!! */
2366 Assert(fWasInLongMode == CPUMIsGuestInLongModeEx(pCtx));
2367
2368 /* Safety precaution; looping for too long here can have a very bad effect on the host */
2369 if (RT_UNLIKELY(++cResume > pVM->hwaccm.s.cMaxResumeLoops))
2370 {
2371 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitMaxResume);
2372 rc = VINF_EM_RAW_INTERRUPT;
2373 goto end;
2374 }
2375
2376 /* Check for irq inhibition due to instruction fusing (sti, mov ss). */
2377 if (VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
2378 {
2379 Log(("VM_FF_INHIBIT_INTERRUPTS at %RGv successor %RGv\n", (RTGCPTR)pCtx->rip, EMGetInhibitInterruptsPC(pVCpu)));
2380 if (pCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
2381 {
2382 /* Note: we intentionally don't clear VM_FF_INHIBIT_INTERRUPTS here.
2383 * Before we are able to execute this instruction in raw mode (iret to guest code) an external interrupt might
2384 * force a world switch again. Possibly allowing a guest interrupt to be dispatched in the process. This could
2385 * break the guest. Sounds very unlikely, but such timing sensitive problems are not as rare as you might think.
2386 */
2387 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2388 /* Irq inhibition is no longer active; clear the corresponding VMX state. */
2389 rc = VMXWriteVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, 0);
2390 AssertRC(rc);
2391 }
2392 }
2393 else
2394 {
2395 /* Irq inhibition is no longer active; clear the corresponding VMX state. */
2396 rc = VMXWriteVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, 0);
2397 AssertRC(rc);
2398 }
2399
2400#ifdef VBOX_HIGH_RES_TIMERS_HACK_IN_RING0
2401 if (RT_UNLIKELY(cResume & 0xf) == 0)
2402 {
2403 uint64_t u64CurTime = RTTimeMilliTS();
2404
2405 if (RT_UNLIKELY(u64CurTime > u64LastTime))
2406 {
2407 u64LastTime = u64CurTime;
2408 TMTimerPollVoid(pVM, pVCpu);
2409 }
2410 }
2411#endif
2412
2413 /* Check for pending actions that force us to go back to ring 3. */
2414 if ( VM_FF_ISPENDING(pVM, VM_FF_HWACCM_TO_R3_MASK | VM_FF_REQUEST | VM_FF_PGM_POOL_FLUSH_PENDING)
2415 || VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_HWACCM_TO_R3_MASK | VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL | VMCPU_FF_REQUEST))
2416 {
2417 /* Check if a sync operation is pending. */
2418 if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
2419 {
2420 rc = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4, VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
2421 AssertRC(rc);
2422 if (rc != VINF_SUCCESS)
2423 {
2424 Log(("Pending pool sync is forcing us back to ring 3; rc=%d\n", rc));
2425 goto end;
2426 }
2427 }
2428
2429#ifdef DEBUG
2430 /* Intercept X86_XCPT_DB if stepping is enabled */
2431 if (!DBGFIsStepping(pVCpu))
2432#endif
2433 {
2434 if ( VM_FF_ISPENDING(pVM, VM_FF_HWACCM_TO_R3_MASK)
2435 || VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_HWACCM_TO_R3_MASK))
2436 {
2437 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
2438 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatSwitchToR3);
2439 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatEntry, x);
2440 rc = RT_UNLIKELY(VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
2441 goto end;
2442 }
2443 }
2444
2445 /* Pending request packets might contain actions that need immediate attention, such as pending hardware interrupts. */
2446 if ( VM_FF_ISPENDING(pVM, VM_FF_REQUEST)
2447 || VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_REQUEST))
2448 {
2449 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatEntry, x);
2450 rc = VINF_EM_PENDING_REQUEST;
2451 goto end;
2452 }
2453
2454 /* Check if a pgm pool flush is in progress. */
2455 if (VM_FF_ISPENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
2456 {
2457 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatEntry, x);
2458 rc = VINF_PGM_POOL_FLUSH_PENDING;
2459 goto end;
2460 }
2461 }
2462
2463#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
2464 /*
2465 * Exit to ring-3 preemption/work is pending.
2466 *
2467 * Interrupts are disabled before the call to make sure we don't miss any interrupt
2468 * that would flag preemption (IPI, timer tick, ++). (Would've been nice to do this
2469 * further down, but VMXR0CheckPendingInterrupt makes that impossible.)
2470 *
2471 * Note! Interrupts must be disabled done *before* we check for TLB flushes; TLB
2472 * shootdowns rely on this.
2473 */
2474 uOldEFlags = ASMIntDisableFlags();
2475 if (RTThreadPreemptIsPending(NIL_RTTHREAD))
2476 {
2477 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitPreemptPending);
2478 rc = VINF_EM_RAW_INTERRUPT;
2479 goto end;
2480 }
2481 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
2482#endif
2483
2484 /* When external interrupts are pending, we should exit the VM when IF is set. */
2485 /* Note! *After* VM_FF_INHIBIT_INTERRUPTS check!!! */
2486 rc = VMXR0CheckPendingInterrupt(pVM, pVCpu, pCtx);
2487 if (RT_FAILURE(rc))
2488 goto end;
2489
2490 /** @todo check timers?? */
2491
2492 /* TPR caching using CR8 is only available in 64 bits mode */
2493 /* Note the 32 bits exception for AMD (X86_CPUID_AMD_FEATURE_ECX_CR8L), but that appears missing in Intel CPUs */
2494 /* Note: we can't do this in LoadGuestState as PDMApicGetTPR can jump back to ring 3 (lock)!!!!! (no longer true) */
2495 /**
2496 * @todo query and update the TPR only when it could have been changed (mmio access & wrmsr (x2apic))
2497 */
2498 if (fSetupTPRCaching)
2499 {
2500 /* TPR caching in CR8 */
2501 bool fPending;
2502
2503 int rc2 = PDMApicGetTPR(pVCpu, &u8LastTPR, &fPending);
2504 AssertRC(rc2);
2505 /* The TPR can be found at offset 0x80 in the APIC mmio page. */
2506 pVCpu->hwaccm.s.vmx.pVAPIC[0x80] = u8LastTPR;
2507
2508 /* Two options here:
2509 * - external interrupt pending, but masked by the TPR value.
2510 * -> a CR8 update that lower the current TPR value should cause an exit
2511 * - no pending interrupts
2512 * -> We don't need to be explicitely notified. There are enough world switches for detecting pending interrupts.
2513 */
2514 rc = VMXWriteVMCS(VMX_VMCS_CTRL_TPR_THRESHOLD, (fPending) ? (u8LastTPR >> 4) : 0); /* cr8 bits 3-0 correspond to bits 7-4 of the task priority mmio register. */
2515 AssertRC(rc);
2516
2517 if (pVM->hwaccm.s.fTPRPatchingActive)
2518 {
2519 Assert(!CPUMIsGuestInLongModeEx(pCtx));
2520 /* Our patch code uses LSTAR for TPR caching. */
2521 pCtx->msrLSTAR = u8LastTPR;
2522
2523 if (fPending)
2524 {
2525 /* A TPR change could activate a pending interrupt, so catch lstar writes. */
2526 vmxR0SetMSRPermission(pVCpu, MSR_K8_LSTAR, true, false);
2527 }
2528 else
2529 {
2530 /* No interrupts are pending, so we don't need to be explicitely notified.
2531 * There are enough world switches for detecting pending interrupts.
2532 */
2533 vmxR0SetMSRPermission(pVCpu, MSR_K8_LSTAR, true, true);
2534 }
2535 }
2536 }
2537
2538#if defined(HWACCM_VTX_WITH_EPT) && defined(LOG_ENABLED)
2539 if ( pVM->hwaccm.s.fNestedPaging
2540# ifdef HWACCM_VTX_WITH_VPID
2541 || pVM->hwaccm.s.vmx.fVPID
2542# endif /* HWACCM_VTX_WITH_VPID */
2543 )
2544 {
2545 PHWACCM_CPUINFO pCpu;
2546
2547 pCpu = HWACCMR0GetCurrentCpu();
2548 if ( pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu
2549 || pVCpu->hwaccm.s.cTLBFlushes != pCpu->cTLBFlushes)
2550 {
2551 if (pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu)
2552 LogFlow(("Force TLB flush due to rescheduling to a different cpu (%d vs %d)\n", pVCpu->hwaccm.s.idLastCpu, pCpu->idCpu));
2553 else
2554 LogFlow(("Force TLB flush due to changed TLB flush count (%x vs %x)\n", pVCpu->hwaccm.s.cTLBFlushes, pCpu->cTLBFlushes));
2555 }
2556 if (pCpu->fFlushTLB)
2557 LogFlow(("Force TLB flush: first time cpu %d is used -> flush\n", pCpu->idCpu));
2558 else
2559 if (pVCpu->hwaccm.s.fForceTLBFlush)
2560 LogFlow(("Manual TLB flush\n"));
2561 }
2562#endif
2563#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
2564 PGMDynMapFlushAutoSet(pVCpu);
2565#endif
2566
2567 /*
2568 * NOTE: DO NOT DO ANYTHING AFTER THIS POINT THAT MIGHT JUMP BACK TO RING 3!
2569 * (until the actual world switch)
2570 */
2571#ifdef VBOX_STRICT
2572 idCpuCheck = RTMpCpuId();
2573#endif
2574#ifdef LOG_ENABLED
2575 VMMR0LogFlushDisable(pVCpu);
2576#endif
2577 /* Save the host state first. */
2578 rc = VMXR0SaveHostState(pVM, pVCpu);
2579 if (RT_UNLIKELY(rc != VINF_SUCCESS))
2580 {
2581 VMMR0LogFlushEnable(pVCpu);
2582 goto end;
2583 }
2584 /* Load the guest state */
2585 rc = VMXR0LoadGuestState(pVM, pVCpu, pCtx);
2586 if (RT_UNLIKELY(rc != VINF_SUCCESS))
2587 {
2588 VMMR0LogFlushEnable(pVCpu);
2589 goto end;
2590 }
2591
2592#ifndef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
2593 /* Disable interrupts to make sure a poke will interrupt execution.
2594 * This must be done *before* we check for TLB flushes; TLB shootdowns rely on this.
2595 */
2596 uOldEFlags = ASMIntDisableFlags();
2597 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
2598#endif
2599
2600 /* Non-register state Guest Context */
2601 /** @todo change me according to cpu state */
2602 rc = VMXWriteVMCS(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_CMS_GUEST_ACTIVITY_ACTIVE);
2603 AssertRC(rc);
2604
2605 /** Set TLB flush state as checked until we return from the world switch. */
2606 ASMAtomicWriteU8(&pVCpu->hwaccm.s.fCheckedTLBFlush, true);
2607 /* Deal with tagged TLB setup and invalidation. */
2608 pVM->hwaccm.s.vmx.pfnSetupTaggedTLB(pVM, pVCpu);
2609
2610 STAM_STATS({ STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatEntry, x); fStatEntryStarted = false; });
2611
2612 /* Manual save and restore:
2613 * - General purpose registers except RIP, RSP
2614 *
2615 * Trashed:
2616 * - CR2 (we don't care)
2617 * - LDTR (reset to 0)
2618 * - DRx (presumably not changed at all)
2619 * - DR7 (reset to 0x400)
2620 * - EFLAGS (reset to RT_BIT(1); not relevant)
2621 *
2622 */
2623
2624 /* All done! Let's start VM execution. */
2625 STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatInGC, z);
2626 Assert(idCpuCheck == RTMpCpuId());
2627
2628#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2629 pVCpu->hwaccm.s.vmx.VMCSCache.cResume = cResume;
2630 pVCpu->hwaccm.s.vmx.VMCSCache.u64TimeSwitch = RTTimeNanoTS();
2631#endif
2632
2633 /* Save the current TPR value in the LSTAR msr so our patches can access it. */
2634 if (pVM->hwaccm.s.fTPRPatchingActive)
2635 {
2636 Assert(pVM->hwaccm.s.fTPRPatchingActive);
2637 u64OldLSTAR = ASMRdMsr(MSR_K8_LSTAR);
2638 ASMWrMsr(MSR_K8_LSTAR, u8LastTPR);
2639 }
2640
2641 TMNotifyStartOfExecution(pVCpu);
2642#ifdef VBOX_WITH_KERNEL_USING_XMM
2643 rc = hwaccmR0VMXStartVMWrapXMM(pVCpu->hwaccm.s.fResumeVM, pCtx, &pVCpu->hwaccm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hwaccm.s.vmx.pfnStartVM);
2644#else
2645 rc = pVCpu->hwaccm.s.vmx.pfnStartVM(pVCpu->hwaccm.s.fResumeVM, pCtx, &pVCpu->hwaccm.s.vmx.VMCSCache, pVM, pVCpu);
2646#endif
2647 ASMAtomicWriteU8(&pVCpu->hwaccm.s.fCheckedTLBFlush, false);
2648 ASMAtomicIncU32(&pVCpu->hwaccm.s.cWorldSwitchExit);
2649 /* Possibly the last TSC value seen by the guest (too high) (only when we're in tsc offset mode). */
2650 if (!(pVCpu->hwaccm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT))
2651 TMCpuTickSetLastSeen(pVCpu, ASMReadTSC() + pVCpu->hwaccm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
2652
2653 TMNotifyEndOfExecution(pVCpu);
2654 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED);
2655 Assert(!(ASMGetFlags() & X86_EFL_IF));
2656
2657 /* Restore the host LSTAR msr if the guest could have changed it. */
2658 if (pVM->hwaccm.s.fTPRPatchingActive)
2659 {
2660 Assert(pVM->hwaccm.s.fTPRPatchingActive);
2661 pVCpu->hwaccm.s.vmx.pVAPIC[0x80] = pCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
2662 ASMWrMsr(MSR_K8_LSTAR, u64OldLSTAR);
2663 }
2664
2665 ASMSetFlags(uOldEFlags);
2666#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
2667 uOldEFlags = ~(RTCCUINTREG)0;
2668#endif
2669
2670 AssertMsg(!pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries, ("pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries=%d\n", pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries));
2671
2672 /* In case we execute a goto ResumeExecution later on. */
2673 pVCpu->hwaccm.s.fResumeVM = true;
2674 pVCpu->hwaccm.s.fForceTLBFlush = false;
2675
2676 /*
2677 * !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
2678 * IMPORTANT: WE CAN'T DO ANY LOGGING OR OPERATIONS THAT CAN DO A LONGJMP BACK TO RING 3 *BEFORE* WE'VE SYNCED BACK (MOST OF) THE GUEST STATE
2679 * !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
2680 */
2681 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatInGC, z);
2682 STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatExit1, v);
2683
2684 if (RT_UNLIKELY(rc != VINF_SUCCESS))
2685 {
2686 VMXR0ReportWorldSwitchError(pVM, pVCpu, rc, pCtx);
2687 VMMR0LogFlushEnable(pVCpu);
2688 goto end;
2689 }
2690
2691 /* Success. Query the guest state and figure out what has happened. */
2692
2693 /* Investigate why there was a VM-exit. */
2694 rc = VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_REASON, &exitReason);
2695 STAM_COUNTER_INC(&pVCpu->hwaccm.s.paStatExitReasonR0[exitReason & MASK_EXITREASON_STAT]);
2696
2697 exitReason &= 0xffff; /* bit 0-15 contain the exit code. */
2698 rc |= VMXReadCachedVMCS(VMX_VMCS32_RO_VM_INSTR_ERROR, &instrError);
2699 rc |= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &cbInstr);
2700 rc |= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &intInfo);
2701 /* might not be valid; depends on VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID. */
2702 rc |= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERRCODE, &errCode);
2703 rc |= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INSTR_INFO, &instrInfo);
2704 rc |= VMXReadCachedVMCS(VMX_VMCS_RO_EXIT_QUALIFICATION, &exitQualification);
2705 AssertRC(rc);
2706
2707 /* Sync back the guest state */
2708 rc = VMXR0SaveGuestState(pVM, pVCpu, pCtx);
2709 AssertRC(rc);
2710
2711 /* Note! NOW IT'S SAFE FOR LOGGING! */
2712 VMMR0LogFlushEnable(pVCpu);
2713 Log2(("Raw exit reason %08x\n", exitReason));
2714
2715 /* Check if an injected event was interrupted prematurely. */
2716 rc = VMXReadCachedVMCS(VMX_VMCS32_RO_IDT_INFO, &val);
2717 AssertRC(rc);
2718 pVCpu->hwaccm.s.Event.intInfo = VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(val);
2719 if ( VMX_EXIT_INTERRUPTION_INFO_VALID(pVCpu->hwaccm.s.Event.intInfo)
2720 /* Ignore 'int xx' as they'll be restarted anyway. */
2721 && VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hwaccm.s.Event.intInfo) != VMX_EXIT_INTERRUPTION_INFO_TYPE_SW
2722 /* Ignore software exceptions (such as int3) as they'll reoccur when we restart the instruction anyway. */
2723 && VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hwaccm.s.Event.intInfo) != VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT)
2724 {
2725 Assert(!pVCpu->hwaccm.s.Event.fPending);
2726 pVCpu->hwaccm.s.Event.fPending = true;
2727 /* Error code present? */
2728 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVCpu->hwaccm.s.Event.intInfo))
2729 {
2730 rc = VMXReadCachedVMCS(VMX_VMCS32_RO_IDT_ERRCODE, &val);
2731 AssertRC(rc);
2732 pVCpu->hwaccm.s.Event.errCode = val;
2733 Log(("Pending inject %RX64 at %RGv exit=%08x intInfo=%08x exitQualification=%RGv pending error=%RX64\n", pVCpu->hwaccm.s.Event.intInfo, (RTGCPTR)pCtx->rip, exitReason, intInfo, exitQualification, val));
2734 }
2735 else
2736 {
2737 Log(("Pending inject %RX64 at %RGv exit=%08x intInfo=%08x exitQualification=%RGv\n", pVCpu->hwaccm.s.Event.intInfo, (RTGCPTR)pCtx->rip, exitReason, intInfo, exitQualification));
2738 pVCpu->hwaccm.s.Event.errCode = 0;
2739 }
2740 }
2741#ifdef VBOX_STRICT
2742 else
2743 if ( VMX_EXIT_INTERRUPTION_INFO_VALID(pVCpu->hwaccm.s.Event.intInfo)
2744 /* Ignore software exceptions (such as int3) as they're reoccur when we restart the instruction anyway. */
2745 && VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hwaccm.s.Event.intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT)
2746 {
2747 Log(("Ignore pending inject %RX64 at %RGv exit=%08x intInfo=%08x exitQualification=%RGv\n", pVCpu->hwaccm.s.Event.intInfo, (RTGCPTR)pCtx->rip, exitReason, intInfo, exitQualification));
2748 }
2749
2750 if (exitReason == VMX_EXIT_ERR_INVALID_GUEST_STATE)
2751 HWACCMDumpRegs(pVM, pVCpu, pCtx);
2752#endif
2753
2754 Log2(("E%d: New EIP=%x:%RGv\n", (uint32_t)exitReason, pCtx->cs, (RTGCPTR)pCtx->rip));
2755 Log2(("Exit reason %d, exitQualification %RGv\n", (uint32_t)exitReason, exitQualification));
2756 Log2(("instrInfo=%d instrError=%d instr length=%d\n", (uint32_t)instrInfo, (uint32_t)instrError, (uint32_t)cbInstr));
2757 Log2(("Interruption error code %d\n", (uint32_t)errCode));
2758 Log2(("IntInfo = %08x\n", (uint32_t)intInfo));
2759
2760 /* Sync back the TPR if it was changed. */
2761 if ( fSetupTPRCaching
2762 && u8LastTPR != pVCpu->hwaccm.s.vmx.pVAPIC[0x80])
2763 {
2764 rc = PDMApicSetTPR(pVCpu, pVCpu->hwaccm.s.vmx.pVAPIC[0x80]);
2765 AssertRC(rc);
2766 }
2767
2768 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit1, v);
2769 STAM_STATS({ STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatExit2, y); fStatExit2Started = true; });
2770
2771 /* Some cases don't need a complete resync of the guest CPU state; handle them here. */
2772 switch (exitReason)
2773 {
2774 case VMX_EXIT_EXCEPTION: /* 0 Exception or non-maskable interrupt (NMI). */
2775 case VMX_EXIT_EXTERNAL_IRQ: /* 1 External interrupt. */
2776 {
2777 uint32_t vector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(intInfo);
2778
2779 if (!VMX_EXIT_INTERRUPTION_INFO_VALID(intInfo))
2780 {
2781 Assert(exitReason == VMX_EXIT_EXTERNAL_IRQ);
2782 /* External interrupt; leave to allow it to be dispatched again. */
2783 rc = VINF_EM_RAW_INTERRUPT;
2784 break;
2785 }
2786 STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2787 switch (VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo))
2788 {
2789 case VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI: /* Non-maskable interrupt. */
2790 /* External interrupt; leave to allow it to be dispatched again. */
2791 rc = VINF_EM_RAW_INTERRUPT;
2792 break;
2793
2794 case VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT: /* External hardware interrupt. */
2795 AssertFailed(); /* can't come here; fails the first check. */
2796 break;
2797
2798 case VMX_EXIT_INTERRUPTION_INFO_TYPE_DBEXCPT: /* Unknown why we get this type for #DB */
2799 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT: /* Software exception. (#BP or #OF) */
2800 Assert(vector == 1 || vector == 3 || vector == 4);
2801 /* no break */
2802 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT: /* Hardware exception. */
2803 Log2(("Hardware/software interrupt %d\n", vector));
2804 switch (vector)
2805 {
2806 case X86_XCPT_NM:
2807 {
2808 Log(("#NM fault at %RGv error code %x\n", (RTGCPTR)pCtx->rip, errCode));
2809
2810 /** @todo don't intercept #NM exceptions anymore when we've activated the guest FPU state. */
2811 /* If we sync the FPU/XMM state on-demand, then we can continue execution as if nothing has happened. */
2812 rc = CPUMR0LoadGuestFPU(pVM, pVCpu, pCtx);
2813 if (rc == VINF_SUCCESS)
2814 {
2815 Assert(CPUMIsGuestFPUStateActive(pVCpu));
2816
2817 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitShadowNM);
2818
2819 /* Continue execution. */
2820 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_CR0;
2821
2822 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2823 goto ResumeExecution;
2824 }
2825
2826 Log(("Forward #NM fault to the guest\n"));
2827 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestNM);
2828 rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, 0);
2829 AssertRC(rc);
2830 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2831 goto ResumeExecution;
2832 }
2833
2834 case X86_XCPT_PF: /* Page fault */
2835 {
2836#ifdef DEBUG
2837 if (pVM->hwaccm.s.fNestedPaging)
2838 { /* A genuine pagefault.
2839 * Forward the trap to the guest by injecting the exception and resuming execution.
2840 */
2841 Log(("Guest page fault at %RGv cr2=%RGv error code %x rsp=%RGv\n", (RTGCPTR)pCtx->rip, exitQualification, errCode, (RTGCPTR)pCtx->rsp));
2842
2843 Assert(CPUMIsGuestInPagedProtectedModeEx(pCtx));
2844
2845 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestPF);
2846
2847 /* Now we must update CR2. */
2848 pCtx->cr2 = exitQualification;
2849 rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
2850 AssertRC(rc);
2851
2852 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2853 goto ResumeExecution;
2854 }
2855#endif
2856 Assert(!pVM->hwaccm.s.fNestedPaging);
2857
2858#ifdef VBOX_HWACCM_WITH_GUEST_PATCHING
2859 /* Shortcut for APIC TPR reads and writes; 32 bits guests only */
2860 if ( pVM->hwaccm.s.fTRPPatchingAllowed
2861 && pVM->hwaccm.s.pGuestPatchMem
2862 && (exitQualification & 0xfff) == 0x080
2863 && !(errCode & X86_TRAP_PF_P) /* not present */
2864 && CPUMGetGuestCPL(pVCpu, CPUMCTX2CORE(pCtx)) == 0
2865 && !CPUMIsGuestInLongModeEx(pCtx)
2866 && pVM->hwaccm.s.cPatches < RT_ELEMENTS(pVM->hwaccm.s.aPatches))
2867 {
2868 RTGCPHYS GCPhysApicBase, GCPhys;
2869 PDMApicGetBase(pVM, &GCPhysApicBase); /* @todo cache this */
2870 GCPhysApicBase &= PAGE_BASE_GC_MASK;
2871
2872 rc = PGMGstGetPage(pVCpu, (RTGCPTR)exitQualification, NULL, &GCPhys);
2873 if ( rc == VINF_SUCCESS
2874 && GCPhys == GCPhysApicBase)
2875 {
2876 /* Only attempt to patch the instruction once. */
2877 PHWACCMTPRPATCH pPatch = (PHWACCMTPRPATCH)RTAvloU32Get(&pVM->hwaccm.s.PatchTree, (AVLOU32KEY)pCtx->eip);
2878 if (!pPatch)
2879 {
2880 rc = VINF_EM_HWACCM_PATCH_TPR_INSTR;
2881 break;
2882 }
2883 }
2884 }
2885#endif
2886
2887 Log2(("Page fault at %RGv error code %x\n", exitQualification, errCode));
2888 /* Exit qualification contains the linear address of the page fault. */
2889 TRPMAssertTrap(pVCpu, X86_XCPT_PF, TRPM_TRAP);
2890 TRPMSetErrorCode(pVCpu, errCode);
2891 TRPMSetFaultAddress(pVCpu, exitQualification);
2892
2893 /* Shortcut for APIC TPR reads and writes. */
2894 if ( (exitQualification & 0xfff) == 0x080
2895 && !(errCode & X86_TRAP_PF_P) /* not present */
2896 && fSetupTPRCaching
2897 && (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
2898 {
2899 RTGCPHYS GCPhysApicBase, GCPhys;
2900 PDMApicGetBase(pVM, &GCPhysApicBase); /* @todo cache this */
2901 GCPhysApicBase &= PAGE_BASE_GC_MASK;
2902
2903 rc = PGMGstGetPage(pVCpu, (RTGCPTR)exitQualification, NULL, &GCPhys);
2904 if ( rc == VINF_SUCCESS
2905 && GCPhys == GCPhysApicBase)
2906 {
2907 Log(("Enable VT-x virtual APIC access filtering\n"));
2908 rc = IOMMMIOMapMMIOHCPage(pVM, GCPhysApicBase, pVM->hwaccm.s.vmx.pAPICPhys, X86_PTE_RW | X86_PTE_P);
2909 AssertRC(rc);
2910 }
2911 }
2912
2913 /* Forward it to our trap handler first, in case our shadow pages are out of sync. */
2914 rc = PGMTrap0eHandler(pVCpu, errCode, CPUMCTX2CORE(pCtx), (RTGCPTR)exitQualification);
2915 Log2(("PGMTrap0eHandler %RGv returned %Rrc\n", (RTGCPTR)pCtx->rip, rc));
2916
2917 if (rc == VINF_SUCCESS)
2918 { /* We've successfully synced our shadow pages, so let's just continue execution. */
2919 Log2(("Shadow page fault at %RGv cr2=%RGv error code %x\n", (RTGCPTR)pCtx->rip, exitQualification ,errCode));
2920 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitShadowPF);
2921
2922 TRPMResetTrap(pVCpu);
2923 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2924 goto ResumeExecution;
2925 }
2926 else
2927 if (rc == VINF_EM_RAW_GUEST_TRAP)
2928 { /* A genuine pagefault.
2929 * Forward the trap to the guest by injecting the exception and resuming execution.
2930 */
2931 Log2(("Forward page fault to the guest\n"));
2932
2933 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestPF);
2934 /* The error code might have been changed. */
2935 errCode = TRPMGetErrorCode(pVCpu);
2936
2937 TRPMResetTrap(pVCpu);
2938
2939 /* Now we must update CR2. */
2940 pCtx->cr2 = exitQualification;
2941 rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
2942 AssertRC(rc);
2943
2944 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2945 goto ResumeExecution;
2946 }
2947#ifdef VBOX_STRICT
2948 if (rc != VINF_EM_RAW_EMULATE_INSTR && rc != VINF_EM_RAW_EMULATE_IO_BLOCK)
2949 Log2(("PGMTrap0eHandler failed with %d\n", rc));
2950#endif
2951 /* Need to go back to the recompiler to emulate the instruction. */
2952 TRPMResetTrap(pVCpu);
2953 break;
2954 }
2955
2956 case X86_XCPT_MF: /* Floating point exception. */
2957 {
2958 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestMF);
2959 if (!(pCtx->cr0 & X86_CR0_NE))
2960 {
2961 /* old style FPU error reporting needs some extra work. */
2962 /** @todo don't fall back to the recompiler, but do it manually. */
2963 rc = VINF_EM_RAW_EMULATE_INSTR;
2964 break;
2965 }
2966 Log(("Trap %x at %04X:%RGv\n", vector, pCtx->cs, (RTGCPTR)pCtx->rip));
2967 rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
2968 AssertRC(rc);
2969
2970 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2971 goto ResumeExecution;
2972 }
2973
2974 case X86_XCPT_DB: /* Debug exception. */
2975 {
2976 uint64_t uDR6;
2977
2978 /* DR6, DR7.GD and IA32_DEBUGCTL.LBR are not updated yet.
2979 *
2980 * Exit qualification bits:
2981 * 3:0 B0-B3 which breakpoint condition was met
2982 * 12:4 Reserved (0)
2983 * 13 BD - debug register access detected
2984 * 14 BS - single step execution or branch taken
2985 * 63:15 Reserved (0)
2986 */
2987 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestDB);
2988
2989 /* Note that we don't support guest and host-initiated debugging at the same time. */
2990
2991 uDR6 = X86_DR6_INIT_VAL;
2992 uDR6 |= (exitQualification & (X86_DR6_B0|X86_DR6_B1|X86_DR6_B2|X86_DR6_B3|X86_DR6_BD|X86_DR6_BS));
2993 rc = DBGFRZTrap01Handler(pVM, pVCpu, CPUMCTX2CORE(pCtx), uDR6);
2994 if (rc == VINF_EM_RAW_GUEST_TRAP)
2995 {
2996 /* Update DR6 here. */
2997 pCtx->dr[6] = uDR6;
2998
2999 /* Resync DR6 if the debug state is active. */
3000 if (CPUMIsGuestDebugStateActive(pVCpu))
3001 ASMSetDR6(pCtx->dr[6]);
3002
3003 /* X86_DR7_GD will be cleared if drx accesses should be trapped inside the guest. */
3004 pCtx->dr[7] &= ~X86_DR7_GD;
3005
3006 /* Paranoia. */
3007 pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
3008 pCtx->dr[7] &= ~(RT_BIT(11) | RT_BIT(12) | RT_BIT(14) | RT_BIT(15)); /* must be zero */
3009 pCtx->dr[7] |= 0x400; /* must be one */
3010
3011 /* Resync DR7 */
3012 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_DR7, pCtx->dr[7]);
3013 AssertRC(rc);
3014
3015 Log(("Trap %x (debug) at %RGv exit qualification %RX64 dr6=%x dr7=%x\n", vector, (RTGCPTR)pCtx->rip, exitQualification, (uint32_t)pCtx->dr[6], (uint32_t)pCtx->dr[7]));
3016 rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
3017 AssertRC(rc);
3018
3019 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3020 goto ResumeExecution;
3021 }
3022 /* Return to ring 3 to deal with the debug exit code. */
3023 Log(("Debugger hardware BP at %04x:%RGv (rc=%Rrc)\n", pCtx->cs, pCtx->rip, rc));
3024 break;
3025 }
3026
3027 case X86_XCPT_BP: /* Breakpoint. */
3028 {
3029 rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pCtx));
3030 if (rc == VINF_EM_RAW_GUEST_TRAP)
3031 {
3032 Log(("Guest #BP at %04x:%RGv\n", pCtx->cs, pCtx->rip));
3033 rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
3034 AssertRC(rc);
3035 goto ResumeExecution;
3036 }
3037 if (rc == VINF_SUCCESS)
3038 goto ResumeExecution;
3039 Log(("Debugger BP at %04x:%RGv (rc=%Rrc)\n", pCtx->cs, pCtx->rip, rc));
3040 break;
3041 }
3042
3043 case X86_XCPT_GP: /* General protection failure exception.*/
3044 {
3045 uint32_t cbOp;
3046 uint32_t cbSize;
3047 PDISCPUSTATE pDis = &pVCpu->hwaccm.s.DisState;
3048
3049 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestGP);
3050#ifdef VBOX_STRICT
3051 if ( !CPUMIsGuestInRealModeEx(pCtx)
3052 || !pVM->hwaccm.s.vmx.pRealModeTSS)
3053 {
3054 Log(("Trap %x at %04X:%RGv errorCode=%x\n", vector, pCtx->cs, (RTGCPTR)pCtx->rip, errCode));
3055 rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
3056 AssertRC(rc);
3057 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3058 goto ResumeExecution;
3059 }
3060#endif
3061 Assert(CPUMIsGuestInRealModeEx(pCtx));
3062
3063 LogFlow(("Real mode X86_XCPT_GP instruction emulation at %x:%RGv\n", pCtx->cs, (RTGCPTR)pCtx->rip));
3064
3065 rc = EMInterpretDisasOne(pVM, pVCpu, CPUMCTX2CORE(pCtx), pDis, &cbOp);
3066 if (RT_SUCCESS(rc))
3067 {
3068 bool fUpdateRIP = true;
3069
3070 Assert(cbOp == pDis->opsize);
3071 switch (pDis->pCurInstr->opcode)
3072 {
3073 case OP_CLI:
3074 pCtx->eflags.Bits.u1IF = 0;
3075 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCli);
3076 break;
3077
3078 case OP_STI:
3079 pCtx->eflags.Bits.u1IF = 1;
3080 EMSetInhibitInterruptsPC(pVCpu, pCtx->rip + pDis->opsize);
3081 Assert(VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
3082 rc = VMXWriteVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
3083 AssertRC(rc);
3084 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitSti);
3085 break;
3086
3087 case OP_HLT:
3088 fUpdateRIP = false;
3089 rc = VINF_EM_HALT;
3090 pCtx->rip += pDis->opsize;
3091 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitHlt);
3092 break;
3093
3094 case OP_POPF:
3095 {
3096 RTGCPTR GCPtrStack;
3097 uint32_t cbParm;
3098 uint32_t uMask;
3099 X86EFLAGS eflags;
3100
3101 if (pDis->prefix & PREFIX_OPSIZE)
3102 {
3103 cbParm = 4;
3104 uMask = 0xffffffff;
3105 }
3106 else
3107 {
3108 cbParm = 2;
3109 uMask = 0xffff;
3110 }
3111
3112 rc = SELMToFlatEx(pVM, DIS_SELREG_SS, CPUMCTX2CORE(pCtx), pCtx->esp & uMask, 0, &GCPtrStack);
3113 if (RT_FAILURE(rc))
3114 {
3115 rc = VERR_EM_INTERPRETER;
3116 break;
3117 }
3118 eflags.u = 0;
3119 rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &eflags.u, cbParm);
3120 if (RT_FAILURE(rc))
3121 {
3122 rc = VERR_EM_INTERPRETER;
3123 break;
3124 }
3125 LogFlow(("POPF %x -> %RGv mask=%x\n", eflags.u, pCtx->rsp, uMask));
3126 pCtx->eflags.u = (pCtx->eflags.u & ~(X86_EFL_POPF_BITS & uMask)) | (eflags.u & X86_EFL_POPF_BITS & uMask);
3127 /* RF cleared when popped in real mode; see pushf description in AMD manual. */
3128 pCtx->eflags.Bits.u1RF = 0;
3129 pCtx->esp += cbParm;
3130 pCtx->esp &= uMask;
3131
3132 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitPopf);
3133 break;
3134 }
3135
3136 case OP_PUSHF:
3137 {
3138 RTGCPTR GCPtrStack;
3139 uint32_t cbParm;
3140 uint32_t uMask;
3141 X86EFLAGS eflags;
3142
3143 if (pDis->prefix & PREFIX_OPSIZE)
3144 {
3145 cbParm = 4;
3146 uMask = 0xffffffff;
3147 }
3148 else
3149 {
3150 cbParm = 2;
3151 uMask = 0xffff;
3152 }
3153
3154 rc = SELMToFlatEx(pVM, DIS_SELREG_SS, CPUMCTX2CORE(pCtx), (pCtx->esp - cbParm) & uMask, 0, &GCPtrStack);
3155 if (RT_FAILURE(rc))
3156 {
3157 rc = VERR_EM_INTERPRETER;
3158 break;
3159 }
3160 eflags = pCtx->eflags;
3161 /* RF & VM cleared when pushed in real mode; see pushf description in AMD manual. */
3162 eflags.Bits.u1RF = 0;
3163 eflags.Bits.u1VM = 0;
3164
3165 rc = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &eflags.u, cbParm);
3166 if (RT_FAILURE(rc))
3167 {
3168 rc = VERR_EM_INTERPRETER;
3169 break;
3170 }
3171 LogFlow(("PUSHF %x -> %RGv\n", eflags.u, GCPtrStack));
3172 pCtx->esp -= cbParm;
3173 pCtx->esp &= uMask;
3174 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitPushf);
3175 break;
3176 }
3177
3178 case OP_IRET:
3179 {
3180 RTGCPTR GCPtrStack;
3181 uint32_t uMask = 0xffff;
3182 uint16_t aIretFrame[3];
3183
3184 if (pDis->prefix & (PREFIX_OPSIZE | PREFIX_ADDRSIZE))
3185 {
3186 rc = VERR_EM_INTERPRETER;
3187 break;
3188 }
3189
3190 rc = SELMToFlatEx(pVM, DIS_SELREG_SS, CPUMCTX2CORE(pCtx), pCtx->esp & uMask, 0, &GCPtrStack);
3191 if (RT_FAILURE(rc))
3192 {
3193 rc = VERR_EM_INTERPRETER;
3194 break;
3195 }
3196 rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
3197 if (RT_FAILURE(rc))
3198 {
3199 rc = VERR_EM_INTERPRETER;
3200 break;
3201 }
3202 pCtx->ip = aIretFrame[0];
3203 pCtx->cs = aIretFrame[1];
3204 pCtx->csHid.u64Base = pCtx->cs << 4;
3205 pCtx->eflags.u = (pCtx->eflags.u & ~(X86_EFL_POPF_BITS & uMask)) | (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
3206 pCtx->sp += sizeof(aIretFrame);
3207
3208 LogFlow(("iret to %04x:%x\n", pCtx->cs, pCtx->ip));
3209 fUpdateRIP = false;
3210 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIret);
3211 break;
3212 }
3213
3214 case OP_INT:
3215 {
3216 uint32_t intInfo2;
3217
3218 LogFlow(("Realmode: INT %x\n", pDis->param1.parval & 0xff));
3219 intInfo2 = pDis->param1.parval & 0xff;
3220 intInfo2 |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
3221 intInfo2 |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
3222
3223 rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, intInfo2, cbOp, 0);
3224 AssertRC(rc);
3225 fUpdateRIP = false;
3226 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInt);
3227 break;
3228 }
3229
3230 case OP_INTO:
3231 {
3232 if (pCtx->eflags.Bits.u1OF)
3233 {
3234 uint32_t intInfo2;
3235
3236 LogFlow(("Realmode: INTO\n"));
3237 intInfo2 = X86_XCPT_OF;
3238 intInfo2 |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
3239 intInfo2 |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
3240
3241 rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, intInfo2, cbOp, 0);
3242 AssertRC(rc);
3243 fUpdateRIP = false;
3244 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInt);
3245 }
3246 break;
3247 }
3248
3249 case OP_INT3:
3250 {
3251 uint32_t intInfo2;
3252
3253 LogFlow(("Realmode: INT 3\n"));
3254 intInfo2 = 3;
3255 intInfo2 |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
3256 intInfo2 |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
3257
3258 rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, intInfo2, cbOp, 0);
3259 AssertRC(rc);
3260 fUpdateRIP = false;
3261 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInt);
3262 break;
3263 }
3264
3265 default:
3266 rc = EMInterpretInstructionCPU(pVM, pVCpu, pDis, CPUMCTX2CORE(pCtx), 0, &cbSize);
3267 break;
3268 }
3269
3270 if (rc == VINF_SUCCESS)
3271 {
3272 if (fUpdateRIP)
3273 pCtx->rip += cbOp; /* Move on to the next instruction. */
3274
3275 /* lidt, lgdt can end up here. In the future crx changes as well. Just reload the whole context to be done with it. */
3276 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_ALL;
3277
3278 /* Only resume if successful. */
3279 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3280 goto ResumeExecution;
3281 }
3282 }
3283 else
3284 rc = VERR_EM_INTERPRETER;
3285
3286 AssertMsg(rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_EM_HALT, ("Unexpected rc=%Rrc\n", rc));
3287 break;
3288 }
3289
3290#ifdef VBOX_STRICT
3291 case X86_XCPT_XF: /* SIMD exception. */
3292 case X86_XCPT_DE: /* Divide error. */
3293 case X86_XCPT_UD: /* Unknown opcode exception. */
3294 case X86_XCPT_SS: /* Stack segment exception. */
3295 case X86_XCPT_NP: /* Segment not present exception. */
3296 {
3297 switch(vector)
3298 {
3299 case X86_XCPT_DE:
3300 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestDE);
3301 break;
3302 case X86_XCPT_UD:
3303 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestUD);
3304 break;
3305 case X86_XCPT_SS:
3306 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestSS);
3307 break;
3308 case X86_XCPT_NP:
3309 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestNP);
3310 break;
3311 }
3312
3313 Log(("Trap %x at %04X:%RGv\n", vector, pCtx->cs, (RTGCPTR)pCtx->rip));
3314 rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
3315 AssertRC(rc);
3316
3317 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3318 goto ResumeExecution;
3319 }
3320#endif
3321 default:
3322#ifdef HWACCM_VMX_EMULATE_REALMODE
3323 if ( CPUMIsGuestInRealModeEx(pCtx)
3324 && pVM->hwaccm.s.vmx.pRealModeTSS)
3325 {
3326 Log(("Real Mode Trap %x at %04x:%04X error code %x\n", vector, pCtx->cs, pCtx->eip, errCode));
3327 rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
3328 AssertRC(rc);
3329
3330 /* Go back to ring 3 in case of a triple fault. */
3331 if ( vector == X86_XCPT_DF
3332 && rc == VINF_EM_RESET)
3333 break;
3334
3335 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3336 goto ResumeExecution;
3337 }
3338#endif
3339 AssertMsgFailed(("Unexpected vm-exit caused by exception %x\n", vector));
3340 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
3341 break;
3342 } /* switch (vector) */
3343
3344 break;
3345
3346 default:
3347 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_CODE;
3348 AssertMsgFailed(("Unexpected interuption code %x\n", intInfo));
3349 break;
3350 }
3351
3352 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3353 break;
3354 }
3355
3356 case VMX_EXIT_EPT_VIOLATION: /* 48 EPT violation. An attempt to access memory with a guest-physical address was disallowed by the configuration of the EPT paging structures. */
3357 {
3358 RTGCPHYS GCPhys;
3359
3360 Assert(pVM->hwaccm.s.fNestedPaging);
3361
3362 rc = VMXReadVMCS64(VMX_VMCS_EXIT_PHYS_ADDR_FULL, &GCPhys);
3363 AssertRC(rc);
3364 Assert(((exitQualification >> 7) & 3) != 2);
3365
3366 /* Determine the kind of violation. */
3367 errCode = 0;
3368 if (exitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
3369 errCode |= X86_TRAP_PF_ID;
3370
3371 if (exitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
3372 errCode |= X86_TRAP_PF_RW;
3373
3374 /* If the page is present, then it's a page level protection fault. */
3375 if (exitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
3376 {
3377 errCode |= X86_TRAP_PF_P;
3378 }
3379 else
3380 {
3381 /* Shortcut for APIC TPR reads and writes. */
3382 if ( (GCPhys & 0xfff) == 0x080
3383 && GCPhys > 0x1000000 /* to skip VGA frame buffer accesses */
3384 && fSetupTPRCaching
3385 && (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
3386 {
3387 RTGCPHYS GCPhysApicBase;
3388 PDMApicGetBase(pVM, &GCPhysApicBase); /* @todo cache this */
3389 GCPhysApicBase &= PAGE_BASE_GC_MASK;
3390 if (GCPhys == GCPhysApicBase + 0x80)
3391 {
3392 Log(("Enable VT-x virtual APIC access filtering\n"));
3393 rc = IOMMMIOMapMMIOHCPage(pVM, GCPhysApicBase, pVM->hwaccm.s.vmx.pAPICPhys, X86_PTE_RW | X86_PTE_P);
3394 AssertRC(rc);
3395 }
3396 }
3397 }
3398 Log(("EPT Page fault %x at %RGp error code %x\n", (uint32_t)exitQualification, GCPhys, errCode));
3399
3400 /* GCPhys contains the guest physical address of the page fault. */
3401 TRPMAssertTrap(pVCpu, X86_XCPT_PF, TRPM_TRAP);
3402 TRPMSetErrorCode(pVCpu, errCode);
3403 TRPMSetFaultAddress(pVCpu, GCPhys);
3404
3405 /* Handle the pagefault trap for the nested shadow table. */
3406 rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, errCode, CPUMCTX2CORE(pCtx), GCPhys);
3407 Log2(("PGMR0Trap0eHandlerNestedPaging %RGv returned %Rrc\n", (RTGCPTR)pCtx->rip, rc));
3408 if (rc == VINF_SUCCESS)
3409 { /* We've successfully synced our shadow pages, so let's just continue execution. */
3410 Log2(("Shadow page fault at %RGv cr2=%RGp error code %x\n", (RTGCPTR)pCtx->rip, exitQualification , errCode));
3411 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitReasonNPF);
3412
3413 TRPMResetTrap(pVCpu);
3414 goto ResumeExecution;
3415 }
3416
3417#ifdef VBOX_STRICT
3418 if (rc != VINF_EM_RAW_EMULATE_INSTR)
3419 LogFlow(("PGMTrap0eHandlerNestedPaging failed with %d\n", rc));
3420#endif
3421 /* Need to go back to the recompiler to emulate the instruction. */
3422 TRPMResetTrap(pVCpu);
3423 break;
3424 }
3425
3426 case VMX_EXIT_EPT_MISCONFIG:
3427 {
3428 RTGCPHYS GCPhys;
3429
3430 Assert(pVM->hwaccm.s.fNestedPaging);
3431
3432 rc = VMXReadVMCS64(VMX_VMCS_EXIT_PHYS_ADDR_FULL, &GCPhys);
3433 AssertRC(rc);
3434
3435 Log(("VMX_EXIT_EPT_MISCONFIG for %RGp\n", GCPhys));
3436 break;
3437 }
3438
3439 case VMX_EXIT_IRQ_WINDOW: /* 7 Interrupt window. */
3440 /* Clear VM-exit on IF=1 change. */
3441 LogFlow(("VMX_EXIT_IRQ_WINDOW %RGv pending=%d IF=%d\n", (RTGCPTR)pCtx->rip, VMCPU_FF_ISPENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC|VMCPU_FF_INTERRUPT_PIC)), pCtx->eflags.Bits.u1IF));
3442 pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_IRQ_WINDOW_EXIT;
3443 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
3444 AssertRC(rc);
3445 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIrqWindow);
3446 goto ResumeExecution; /* we check for pending guest interrupts there */
3447
3448 case VMX_EXIT_WBINVD: /* 54 Guest software attempted to execute WBINVD. (conditional) */
3449 case VMX_EXIT_INVD: /* 13 Guest software attempted to execute INVD. (unconditional) */
3450 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInvd);
3451 /* Skip instruction and continue directly. */
3452 pCtx->rip += cbInstr;
3453 /* Continue execution.*/
3454 goto ResumeExecution;
3455
3456 case VMX_EXIT_CPUID: /* 10 Guest software attempted to execute CPUID. */
3457 {
3458 Log2(("VMX: Cpuid %x\n", pCtx->eax));
3459 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCpuid);
3460 rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pCtx));
3461 if (rc == VINF_SUCCESS)
3462 {
3463 /* Update EIP and continue execution. */
3464 Assert(cbInstr == 2);
3465 pCtx->rip += cbInstr;
3466 goto ResumeExecution;
3467 }
3468 AssertMsgFailed(("EMU: cpuid failed with %Rrc\n", rc));
3469 rc = VINF_EM_RAW_EMULATE_INSTR;
3470 break;
3471 }
3472
3473 case VMX_EXIT_RDPMC: /* 15 Guest software attempted to execute RDPMC. */
3474 {
3475 Log2(("VMX: Rdpmc %x\n", pCtx->ecx));
3476 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitRdpmc);
3477 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pCtx));
3478 if (rc == VINF_SUCCESS)
3479 {
3480 /* Update EIP and continue execution. */
3481 Assert(cbInstr == 2);
3482 pCtx->rip += cbInstr;
3483 goto ResumeExecution;
3484 }
3485 rc = VINF_EM_RAW_EMULATE_INSTR;
3486 break;
3487 }
3488
3489 case VMX_EXIT_RDTSC: /* 16 Guest software attempted to execute RDTSC. */
3490 {
3491 Log2(("VMX: Rdtsc\n"));
3492 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitRdtsc);
3493 rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pCtx));
3494 if (rc == VINF_SUCCESS)
3495 {
3496 /* Update EIP and continue execution. */
3497 Assert(cbInstr == 2);
3498 pCtx->rip += cbInstr;
3499 goto ResumeExecution;
3500 }
3501 rc = VINF_EM_RAW_EMULATE_INSTR;
3502 break;
3503 }
3504
3505 case VMX_EXIT_INVPG: /* 14 Guest software attempted to execute INVPG. */
3506 {
3507 Log2(("VMX: invlpg\n"));
3508 Assert(!pVM->hwaccm.s.fNestedPaging);
3509
3510 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInvpg);
3511 rc = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pCtx), exitQualification);
3512 if (rc == VINF_SUCCESS)
3513 {
3514 /* Update EIP and continue execution. */
3515 pCtx->rip += cbInstr;
3516 goto ResumeExecution;
3517 }
3518 AssertMsg(rc == VERR_EM_INTERPRETER, ("EMU: invlpg %RGv failed with %Rrc\n", exitQualification, rc));
3519 break;
3520 }
3521
3522 case VMX_EXIT_MONITOR: /* 39 Guest software attempted to execute MONITOR. */
3523 {
3524 Log2(("VMX: monitor\n"));
3525
3526 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitMonitor);
3527 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pCtx));
3528 if (rc == VINF_SUCCESS)
3529 {
3530 /* Update EIP and continue execution. */
3531 pCtx->rip += cbInstr;
3532 goto ResumeExecution;
3533 }
3534 AssertMsg(rc == VERR_EM_INTERPRETER, ("EMU: monitor failed with %Rrc\n", rc));
3535 break;
3536 }
3537
3538 case VMX_EXIT_WRMSR: /* 32 WRMSR. Guest software attempted to execute WRMSR. */
3539 /* When an interrupt is pending, we'll let MSR_K8_LSTAR writes fault in our TPR patch code. */
3540 if ( pVM->hwaccm.s.fTPRPatchingActive
3541 && pCtx->ecx == MSR_K8_LSTAR)
3542 {
3543 Assert(!CPUMIsGuestInLongModeEx(pCtx));
3544 if ((pCtx->eax & 0xff) != u8LastTPR)
3545 {
3546 Log(("VMX: Faulting MSR_K8_LSTAR write with new TPR value %x\n", pCtx->eax & 0xff));
3547
3548 /* Our patch code uses LSTAR for TPR caching. */
3549 rc = PDMApicSetTPR(pVCpu, pCtx->eax & 0xff);
3550 AssertRC(rc);
3551 }
3552
3553 /* Skip the instruction and continue. */
3554 pCtx->rip += cbInstr; /* wrmsr = [0F 30] */
3555
3556 /* Only resume if successful. */
3557 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit1, x);
3558 goto ResumeExecution;
3559 }
3560 /* no break */
3561 case VMX_EXIT_RDMSR: /* 31 RDMSR. Guest software attempted to execute RDMSR. */
3562 {
3563 uint32_t cbSize;
3564
3565 STAM_COUNTER_INC((exitReason == VMX_EXIT_RDMSR) ? &pVCpu->hwaccm.s.StatExitRdmsr : &pVCpu->hwaccm.s.StatExitWrmsr);
3566
3567 /* Note: the intel manual claims there's a REX version of RDMSR that's slightly different, so we play safe by completely disassembling the instruction. */
3568 Log2(("VMX: %s\n", (exitReason == VMX_EXIT_RDMSR) ? "rdmsr" : "wrmsr"));
3569 rc = EMInterpretInstruction(pVM, pVCpu, CPUMCTX2CORE(pCtx), 0, &cbSize);
3570 if (rc == VINF_SUCCESS)
3571 {
3572 /* EIP has been updated already. */
3573
3574 /* Only resume if successful. */
3575 goto ResumeExecution;
3576 }
3577 AssertMsg(rc == VERR_EM_INTERPRETER, ("EMU: %s failed with %Rrc\n", (exitReason == VMX_EXIT_RDMSR) ? "rdmsr" : "wrmsr", rc));
3578 break;
3579 }
3580
3581 case VMX_EXIT_CRX_MOVE: /* 28 Control-register accesses. */
3582 {
3583 STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatExit2Sub2, y2);
3584
3585 switch (VMX_EXIT_QUALIFICATION_CRX_ACCESS(exitQualification))
3586 {
3587 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE:
3588 Log2(("VMX: %RGv mov cr%d, x\n", (RTGCPTR)pCtx->rip, VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification)));
3589 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification)]);
3590 rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
3591 VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification),
3592 VMX_EXIT_QUALIFICATION_CRX_GENREG(exitQualification));
3593
3594 switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification))
3595 {
3596 case 0:
3597 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_CR0 | HWACCM_CHANGED_GUEST_CR3;
3598 break;
3599 case 2:
3600 break;
3601 case 3:
3602 Assert(!pVM->hwaccm.s.fNestedPaging || !CPUMIsGuestInPagedProtectedModeEx(pCtx));
3603 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_CR3;
3604 break;
3605 case 4:
3606 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_CR4;
3607 break;
3608 case 8:
3609 /* CR8 contains the APIC TPR */
3610 Assert(!(pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
3611 break;
3612
3613 default:
3614 AssertFailed();
3615 break;
3616 }
3617 break;
3618
3619 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ:
3620 Log2(("VMX: mov x, crx\n"));
3621 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification)]);
3622
3623 Assert(!pVM->hwaccm.s.fNestedPaging || !CPUMIsGuestInPagedProtectedModeEx(pCtx) || VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification) != USE_REG_CR3);
3624
3625 /* CR8 reads only cause an exit when the TPR shadow feature isn't present. */
3626 Assert(VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification) != 8 || !(pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
3627
3628 rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
3629 VMX_EXIT_QUALIFICATION_CRX_GENREG(exitQualification),
3630 VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification));
3631 break;
3632
3633 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS:
3634 Log2(("VMX: clts\n"));
3635 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCLTS);
3636 rc = EMInterpretCLTS(pVM, pVCpu);
3637 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_CR0;
3638 break;
3639
3640 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW:
3641 Log2(("VMX: lmsw %x\n", VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(exitQualification)));
3642 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitLMSW);
3643 rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(exitQualification));
3644 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_CR0;
3645 break;
3646 }
3647
3648 /* Update EIP if no error occurred. */
3649 if (RT_SUCCESS(rc))
3650 pCtx->rip += cbInstr;
3651
3652 if (rc == VINF_SUCCESS)
3653 {
3654 /* Only resume if successful. */
3655 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub2, y2);
3656 goto ResumeExecution;
3657 }
3658 Assert(rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_PGM_SYNC_CR3);
3659 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub2, y2);
3660 break;
3661 }
3662
3663 case VMX_EXIT_DRX_MOVE: /* 29 Debug-register accesses. */
3664 {
3665 if ( !DBGFIsStepping(pVCpu)
3666 && !CPUMIsHyperDebugStateActive(pVCpu))
3667 {
3668 /* Disable drx move intercepts. */
3669 pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
3670 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
3671 AssertRC(rc);
3672
3673 /* Save the host and load the guest debug state. */
3674 rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
3675 AssertRC(rc);
3676
3677#ifdef LOG_ENABLED
3678 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
3679 Log(("VMX_EXIT_DRX_MOVE: write DR%d genreg %d\n", VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification), VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification)));
3680 else
3681 Log(("VMX_EXIT_DRX_MOVE: read DR%d\n", VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification)));
3682#endif
3683
3684#ifdef VBOX_WITH_STATISTICS
3685 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatDRxContextSwitch);
3686 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
3687 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxWrite);
3688 else
3689 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxRead);
3690#endif
3691
3692 goto ResumeExecution;
3693 }
3694
3695 /** @todo clear VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT after the first time and restore drx registers afterwards */
3696 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
3697 {
3698 Log2(("VMX: mov drx%d, genreg%d\n", VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification), VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification)));
3699 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxWrite);
3700 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
3701 VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification),
3702 VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification));
3703 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_DEBUG;
3704 Log2(("DR7=%08x\n", pCtx->dr[7]));
3705 }
3706 else
3707 {
3708 Log2(("VMX: mov x, drx\n"));
3709 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxRead);
3710 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
3711 VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification),
3712 VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification));
3713 }
3714 /* Update EIP if no error occurred. */
3715 if (RT_SUCCESS(rc))
3716 pCtx->rip += cbInstr;
3717
3718 if (rc == VINF_SUCCESS)
3719 {
3720 /* Only resume if successful. */
3721 goto ResumeExecution;
3722 }
3723 Assert(rc == VERR_EM_INTERPRETER);
3724 break;
3725 }
3726
3727 /* Note: We'll get a #GP if the IO instruction isn't allowed (IOPL or TSS bitmap); no need to double check. */
3728 case VMX_EXIT_PORT_IO: /* 30 I/O instruction. */
3729 {
3730 STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3731 uint32_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(exitQualification);
3732 uint32_t uPort;
3733 bool fIOWrite = (VMX_EXIT_QUALIFICATION_IO_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
3734
3735 /** @todo necessary to make the distinction? */
3736 if (VMX_EXIT_QUALIFICATION_IO_ENCODING(exitQualification) == VMX_EXIT_QUALIFICATION_IO_ENCODING_DX)
3737 {
3738 uPort = pCtx->edx & 0xffff;
3739 }
3740 else
3741 uPort = VMX_EXIT_QUALIFICATION_IO_PORT(exitQualification); /* Immediate encoding. */
3742
3743 /* paranoia */
3744 if (RT_UNLIKELY(uIOWidth == 2 || uIOWidth >= 4))
3745 {
3746 rc = fIOWrite ? VINF_IOM_HC_IOPORT_WRITE : VINF_IOM_HC_IOPORT_READ;
3747 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3748 break;
3749 }
3750
3751 uint32_t cbSize = g_aIOSize[uIOWidth];
3752
3753 if (VMX_EXIT_QUALIFICATION_IO_STRING(exitQualification))
3754 {
3755 /* ins/outs */
3756 PDISCPUSTATE pDis = &pVCpu->hwaccm.s.DisState;
3757
3758 /* Disassemble manually to deal with segment prefixes. */
3759 /** @todo VMX_VMCS_EXIT_GUEST_LINEAR_ADDR contains the flat pointer operand of the instruction. */
3760 /** @todo VMX_VMCS32_RO_EXIT_INSTR_INFO also contains segment prefix info. */
3761 rc = EMInterpretDisasOne(pVM, pVCpu, CPUMCTX2CORE(pCtx), pDis, NULL);
3762 if (rc == VINF_SUCCESS)
3763 {
3764 if (fIOWrite)
3765 {
3766 Log2(("IOMInterpretOUTSEx %RGv %x size=%d\n", (RTGCPTR)pCtx->rip, uPort, cbSize));
3767 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIOStringWrite);
3768 rc = VBOXSTRICTRC_TODO(IOMInterpretOUTSEx(pVM, CPUMCTX2CORE(pCtx), uPort, pDis->prefix, cbSize));
3769 }
3770 else
3771 {
3772 Log2(("IOMInterpretINSEx %RGv %x size=%d\n", (RTGCPTR)pCtx->rip, uPort, cbSize));
3773 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIOStringRead);
3774 rc = VBOXSTRICTRC_TODO(IOMInterpretINSEx(pVM, CPUMCTX2CORE(pCtx), uPort, pDis->prefix, cbSize));
3775 }
3776 }
3777 else
3778 rc = VINF_EM_RAW_EMULATE_INSTR;
3779 }
3780 else
3781 {
3782 /* normal in/out */
3783 uint32_t uAndVal = g_aIOOpAnd[uIOWidth];
3784
3785 Assert(!VMX_EXIT_QUALIFICATION_IO_REP(exitQualification));
3786
3787 if (fIOWrite)
3788 {
3789 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIOWrite);
3790 rc = VBOXSTRICTRC_TODO(IOMIOPortWrite(pVM, uPort, pCtx->eax & uAndVal, cbSize));
3791 if (rc == VINF_IOM_HC_IOPORT_WRITE)
3792 HWACCMR0SavePendingIOPortWrite(pVCpu, pCtx->rip, pCtx->rip + cbInstr, uPort, uAndVal, cbSize);
3793 }
3794 else
3795 {
3796 uint32_t u32Val = 0;
3797
3798 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIORead);
3799 rc = VBOXSTRICTRC_TODO(IOMIOPortRead(pVM, uPort, &u32Val, cbSize));
3800 if (IOM_SUCCESS(rc))
3801 {
3802 /* Write back to the EAX register. */
3803 pCtx->eax = (pCtx->eax & ~uAndVal) | (u32Val & uAndVal);
3804 }
3805 else
3806 if (rc == VINF_IOM_HC_IOPORT_READ)
3807 HWACCMR0SavePendingIOPortRead(pVCpu, pCtx->rip, pCtx->rip + cbInstr, uPort, uAndVal, cbSize);
3808 }
3809 }
3810 /*
3811 * Handled the I/O return codes.
3812 * (The unhandled cases end up with rc == VINF_EM_RAW_EMULATE_INSTR.)
3813 */
3814 if (IOM_SUCCESS(rc))
3815 {
3816 /* Update EIP and continue execution. */
3817 pCtx->rip += cbInstr;
3818 if (RT_LIKELY(rc == VINF_SUCCESS))
3819 {
3820 /* If any IO breakpoints are armed, then we should check if a debug trap needs to be generated. */
3821 if (pCtx->dr[7] & X86_DR7_ENABLED_MASK)
3822 {
3823 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatDRxIOCheck);
3824 for (unsigned i=0;i<4;i++)
3825 {
3826 unsigned uBPLen = g_aIOSize[X86_DR7_GET_LEN(pCtx->dr[7], i)];
3827
3828 if ( (uPort >= pCtx->dr[i] && uPort < pCtx->dr[i] + uBPLen)
3829 && (pCtx->dr[7] & (X86_DR7_L(i) | X86_DR7_G(i)))
3830 && (pCtx->dr[7] & X86_DR7_RW(i, X86_DR7_RW_IO)) == X86_DR7_RW(i, X86_DR7_RW_IO))
3831 {
3832 uint64_t uDR6;
3833
3834 Assert(CPUMIsGuestDebugStateActive(pVCpu));
3835
3836 uDR6 = ASMGetDR6();
3837
3838 /* Clear all breakpoint status flags and set the one we just hit. */
3839 uDR6 &= ~(X86_DR6_B0|X86_DR6_B1|X86_DR6_B2|X86_DR6_B3);
3840 uDR6 |= (uint64_t)RT_BIT(i);
3841
3842 /* Note: AMD64 Architecture Programmer's Manual 13.1:
3843 * Bits 15:13 of the DR6 register is never cleared by the processor and must be cleared by software after
3844 * the contents have been read.
3845 */
3846 ASMSetDR6(uDR6);
3847
3848 /* X86_DR7_GD will be cleared if drx accesses should be trapped inside the guest. */
3849 pCtx->dr[7] &= ~X86_DR7_GD;
3850
3851 /* Paranoia. */
3852 pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
3853 pCtx->dr[7] &= ~(RT_BIT(11) | RT_BIT(12) | RT_BIT(14) | RT_BIT(15)); /* must be zero */
3854 pCtx->dr[7] |= 0x400; /* must be one */
3855
3856 /* Resync DR7 */
3857 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_DR7, pCtx->dr[7]);
3858 AssertRC(rc);
3859
3860 /* Construct inject info. */
3861 intInfo = X86_XCPT_DB;
3862 intInfo |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
3863 intInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
3864
3865 Log(("Inject IO debug trap at %RGv\n", (RTGCPTR)pCtx->rip));
3866 rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), 0, 0);
3867 AssertRC(rc);
3868
3869 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3870 goto ResumeExecution;
3871 }
3872 }
3873 }
3874
3875 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3876 goto ResumeExecution;
3877 }
3878 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3879 break;
3880 }
3881
3882#ifdef VBOX_STRICT
3883 if (rc == VINF_IOM_HC_IOPORT_READ)
3884 Assert(!fIOWrite);
3885 else if (rc == VINF_IOM_HC_IOPORT_WRITE)
3886 Assert(fIOWrite);
3887 else
3888 AssertMsg(RT_FAILURE(rc) || rc == VINF_EM_RAW_EMULATE_INSTR || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", rc));
3889#endif
3890 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3891 break;
3892 }
3893
3894 case VMX_EXIT_TPR: /* 43 TPR below threshold. Guest software executed MOV to CR8. */
3895 LogFlow(("VMX_EXIT_TPR\n"));
3896 /* RIP is already set to the next instruction and the TPR has been synced back. Just resume. */
3897 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3898 goto ResumeExecution;
3899
3900 case VMX_EXIT_APIC_ACCESS: /* 44 APIC access. Guest software attempted to access memory at a physical address on the APIC-access page. */
3901 {
3902 LogFlow(("VMX_EXIT_APIC_ACCESS\n"));
3903 unsigned uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(exitQualification);
3904
3905 switch(uAccessType)
3906 {
3907 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
3908 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
3909 {
3910 RTGCPHYS GCPhys;
3911 PDMApicGetBase(pVM, &GCPhys);
3912 GCPhys &= PAGE_BASE_GC_MASK;
3913 GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(exitQualification);
3914
3915 LogFlow(("Apic access at %RGp\n", GCPhys));
3916 rc = VBOXSTRICTRC_TODO(IOMMMIOPhysHandler(pVM, (uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW, CPUMCTX2CORE(pCtx), GCPhys));
3917 if (rc == VINF_SUCCESS)
3918 {
3919 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3920 goto ResumeExecution; /* rip already updated */
3921 }
3922 break;
3923 }
3924
3925 default:
3926 rc = VINF_EM_RAW_EMULATE_INSTR;
3927 break;
3928 }
3929 break;
3930 }
3931
3932 case VMX_EXIT_PREEMPTION_TIMER: /* 52 VMX-preemption timer expired. The preemption timer counted down to zero. */
3933 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3934 goto ResumeExecution;
3935
3936 default:
3937 /* The rest is handled after syncing the entire CPU state. */
3938 break;
3939 }
3940
3941 /* Note: the guest state isn't entirely synced back at this stage. */
3942
3943 /* Investigate why there was a VM-exit. (part 2) */
3944 switch (exitReason)
3945 {
3946 case VMX_EXIT_EXCEPTION: /* 0 Exception or non-maskable interrupt (NMI). */
3947 case VMX_EXIT_EXTERNAL_IRQ: /* 1 External interrupt. */
3948 case VMX_EXIT_EPT_VIOLATION:
3949 case VMX_EXIT_PREEMPTION_TIMER: /* 52 VMX-preemption timer expired. The preemption timer counted down to zero. */
3950 /* Already handled above. */
3951 break;
3952
3953 case VMX_EXIT_TRIPLE_FAULT: /* 2 Triple fault. */
3954 rc = VINF_EM_RESET; /* Triple fault equals a reset. */
3955 break;
3956
3957 case VMX_EXIT_INIT_SIGNAL: /* 3 INIT signal. */
3958 case VMX_EXIT_SIPI: /* 4 Start-up IPI (SIPI). */
3959 rc = VINF_EM_RAW_INTERRUPT;
3960 AssertFailed(); /* Can't happen. Yet. */
3961 break;
3962
3963 case VMX_EXIT_IO_SMI_IRQ: /* 5 I/O system-management interrupt (SMI). */
3964 case VMX_EXIT_SMI_IRQ: /* 6 Other SMI. */
3965 rc = VINF_EM_RAW_INTERRUPT;
3966 AssertFailed(); /* Can't happen afaik. */
3967 break;
3968
3969 case VMX_EXIT_TASK_SWITCH: /* 9 Task switch: too complicated to emulate, so fall back to the recompiler */
3970 Log(("VMX_EXIT_TASK_SWITCH: exit=%RX64\n", exitQualification));
3971 if ( (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(exitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
3972 && pVCpu->hwaccm.s.Event.fPending)
3973 {
3974 /* Caused by an injected interrupt. */
3975 pVCpu->hwaccm.s.Event.fPending = false;
3976
3977 Log(("VMX_EXIT_TASK_SWITCH: reassert trap %d\n", VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVCpu->hwaccm.s.Event.intInfo)));
3978 Assert(!VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVCpu->hwaccm.s.Event.intInfo));
3979 rc = TRPMAssertTrap(pVCpu, VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVCpu->hwaccm.s.Event.intInfo), TRPM_HARDWARE_INT);
3980 AssertRC(rc);
3981 }
3982 /* else Exceptions and software interrupts can just be restarted. */
3983 rc = VERR_EM_INTERPRETER;
3984 break;
3985
3986 case VMX_EXIT_HLT: /* 12 Guest software attempted to execute HLT. */
3987 /** Check if external interrupts are pending; if so, don't switch back. */
3988 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitHlt);
3989 pCtx->rip++; /* skip hlt */
3990 if (EMShouldContinueAfterHalt(pVCpu, pCtx))
3991 goto ResumeExecution;
3992
3993 rc = VINF_EM_HALT;
3994 break;
3995
3996 case VMX_EXIT_MWAIT: /* 36 Guest software executed MWAIT. */
3997 Log2(("VMX: mwait\n"));
3998 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitMwait);
3999 rc = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pCtx));
4000 if ( rc == VINF_EM_HALT
4001 || rc == VINF_SUCCESS)
4002 {
4003 /* Update EIP and continue execution. */
4004 pCtx->rip += cbInstr;
4005
4006 /** Check if external interrupts are pending; if so, don't switch back. */
4007 if ( rc == VINF_SUCCESS
4008 || ( rc == VINF_EM_HALT
4009 && EMShouldContinueAfterHalt(pVCpu, pCtx))
4010 )
4011 goto ResumeExecution;
4012 }
4013 AssertMsg(rc == VERR_EM_INTERPRETER || rc == VINF_EM_HALT, ("EMU: mwait failed with %Rrc\n", rc));
4014 break;
4015
4016 case VMX_EXIT_RSM: /* 17 Guest software attempted to execute RSM in SMM. */
4017 AssertFailed(); /* can't happen. */
4018 rc = VERR_EM_INTERPRETER;
4019 break;
4020
4021 case VMX_EXIT_VMCALL: /* 18 Guest software executed VMCALL. */
4022 case VMX_EXIT_VMCLEAR: /* 19 Guest software executed VMCLEAR. */
4023 case VMX_EXIT_VMLAUNCH: /* 20 Guest software executed VMLAUNCH. */
4024 case VMX_EXIT_VMPTRLD: /* 21 Guest software executed VMPTRLD. */
4025 case VMX_EXIT_VMPTRST: /* 22 Guest software executed VMPTRST. */
4026 case VMX_EXIT_VMREAD: /* 23 Guest software executed VMREAD. */
4027 case VMX_EXIT_VMRESUME: /* 24 Guest software executed VMRESUME. */
4028 case VMX_EXIT_VMWRITE: /* 25 Guest software executed VMWRITE. */
4029 case VMX_EXIT_VMXOFF: /* 26 Guest software executed VMXOFF. */
4030 case VMX_EXIT_VMXON: /* 27 Guest software executed VMXON. */
4031 /** @todo inject #UD immediately */
4032 rc = VERR_EM_INTERPRETER;
4033 break;
4034
4035 case VMX_EXIT_CPUID: /* 10 Guest software attempted to execute CPUID. */
4036 case VMX_EXIT_RDTSC: /* 16 Guest software attempted to execute RDTSC. */
4037 case VMX_EXIT_INVPG: /* 14 Guest software attempted to execute INVPG. */
4038 case VMX_EXIT_CRX_MOVE: /* 28 Control-register accesses. */
4039 case VMX_EXIT_DRX_MOVE: /* 29 Debug-register accesses. */
4040 case VMX_EXIT_PORT_IO: /* 30 I/O instruction. */
4041 case VMX_EXIT_RDPMC: /* 15 Guest software attempted to execute RDPMC. */
4042 /* already handled above */
4043 AssertMsg( rc == VINF_PGM_CHANGE_MODE
4044 || rc == VINF_EM_RAW_INTERRUPT
4045 || rc == VERR_EM_INTERPRETER
4046 || rc == VINF_EM_RAW_EMULATE_INSTR
4047 || rc == VINF_PGM_SYNC_CR3
4048 || rc == VINF_IOM_HC_IOPORT_READ
4049 || rc == VINF_IOM_HC_IOPORT_WRITE
4050 || rc == VINF_EM_RAW_GUEST_TRAP
4051 || rc == VINF_TRPM_XCPT_DISPATCHED
4052 || rc == VINF_EM_RESCHEDULE_REM,
4053 ("rc = %d\n", rc));
4054 break;
4055
4056 case VMX_EXIT_TPR: /* 43 TPR below threshold. Guest software executed MOV to CR8. */
4057 case VMX_EXIT_APIC_ACCESS: /* 44 APIC access. Guest software attempted to access memory at a physical address on the APIC-access page. */
4058 case VMX_EXIT_RDMSR: /* 31 RDMSR. Guest software attempted to execute RDMSR. */
4059 case VMX_EXIT_WRMSR: /* 32 WRMSR. Guest software attempted to execute WRMSR. */
4060 case VMX_EXIT_PAUSE: /* 40 Guest software attempted to execute PAUSE. */
4061 case VMX_EXIT_MONITOR: /* 39 Guest software attempted to execute MONITOR. */
4062 /* Note: If we decide to emulate them here, then we must sync the MSRs that could have been changed (sysenter, fs/gs base)!!! */
4063 rc = VERR_EM_INTERPRETER;
4064 break;
4065
4066 case VMX_EXIT_IRQ_WINDOW: /* 7 Interrupt window. */
4067 Assert(rc == VINF_EM_RAW_INTERRUPT);
4068 break;
4069
4070 case VMX_EXIT_ERR_INVALID_GUEST_STATE: /* 33 VM-entry failure due to invalid guest state. */
4071 {
4072#ifdef VBOX_STRICT
4073 RTCCUINTREG val2 = 0;
4074
4075 Log(("VMX_EXIT_ERR_INVALID_GUEST_STATE\n"));
4076
4077 VMXReadVMCS(VMX_VMCS64_GUEST_RIP, &val2);
4078 Log(("Old eip %RGv new %RGv\n", (RTGCPTR)pCtx->rip, (RTGCPTR)val2));
4079
4080 VMXReadVMCS(VMX_VMCS64_GUEST_CR0, &val2);
4081 Log(("VMX_VMCS_GUEST_CR0 %RX64\n", (uint64_t)val2));
4082
4083 VMXReadVMCS(VMX_VMCS64_GUEST_CR3, &val2);
4084 Log(("VMX_VMCS_GUEST_CR3 %RX64\n", (uint64_t)val2));
4085
4086 VMXReadVMCS(VMX_VMCS64_GUEST_CR4, &val2);
4087 Log(("VMX_VMCS_GUEST_CR4 %RX64\n", (uint64_t)val2));
4088
4089 VMXReadVMCS(VMX_VMCS_GUEST_RFLAGS, &val2);
4090 Log(("VMX_VMCS_GUEST_RFLAGS %08x\n", val2));
4091
4092 VMX_LOG_SELREG(CS, "CS", val2);
4093 VMX_LOG_SELREG(DS, "DS", val2);
4094 VMX_LOG_SELREG(ES, "ES", val2);
4095 VMX_LOG_SELREG(FS, "FS", val2);
4096 VMX_LOG_SELREG(GS, "GS", val2);
4097 VMX_LOG_SELREG(SS, "SS", val2);
4098 VMX_LOG_SELREG(TR, "TR", val2);
4099 VMX_LOG_SELREG(LDTR, "LDTR", val2);
4100
4101 VMXReadVMCS(VMX_VMCS64_GUEST_GDTR_BASE, &val2);
4102 Log(("VMX_VMCS_GUEST_GDTR_BASE %RX64\n", (uint64_t)val2));
4103 VMXReadVMCS(VMX_VMCS64_GUEST_IDTR_BASE, &val2);
4104 Log(("VMX_VMCS_GUEST_IDTR_BASE %RX64\n", (uint64_t)val2));
4105#endif /* VBOX_STRICT */
4106 rc = VERR_VMX_INVALID_GUEST_STATE;
4107 break;
4108 }
4109
4110 case VMX_EXIT_ERR_MSR_LOAD: /* 34 VM-entry failure due to MSR loading. */
4111 case VMX_EXIT_ERR_MACHINE_CHECK: /* 41 VM-entry failure due to machine-check. */
4112 default:
4113 rc = VERR_VMX_UNEXPECTED_EXIT_CODE;
4114 AssertMsgFailed(("Unexpected exit code %d\n", exitReason)); /* Can't happen. */
4115 break;
4116
4117 }
4118end:
4119
4120 /* Signal changes for the recompiler. */
4121 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR | CPUM_CHANGED_LDTR | CPUM_CHANGED_GDTR | CPUM_CHANGED_IDTR | CPUM_CHANGED_TR | CPUM_CHANGED_HIDDEN_SEL_REGS);
4122
4123 /* If we executed vmlaunch/vmresume and an external irq was pending, then we don't have to do a full sync the next time. */
4124 if ( exitReason == VMX_EXIT_EXTERNAL_IRQ
4125 && !VMX_EXIT_INTERRUPTION_INFO_VALID(intInfo))
4126 {
4127 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatPendingHostIrq);
4128 /* On the next entry we'll only sync the host context. */
4129 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_HOST_CONTEXT;
4130 }
4131 else
4132 {
4133 /* On the next entry we'll sync everything. */
4134 /** @todo we can do better than this */
4135 /* Not in the VINF_PGM_CHANGE_MODE though! */
4136 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_ALL;
4137 }
4138
4139 /* translate into a less severe return code */
4140 if (rc == VERR_EM_INTERPRETER)
4141 rc = VINF_EM_RAW_EMULATE_INSTR;
4142 else
4143 /* Try to extract more information about what might have gone wrong here. */
4144 if (rc == VERR_VMX_INVALID_VMCS_PTR)
4145 {
4146 VMXGetActivateVMCS(&pVCpu->hwaccm.s.vmx.lasterror.u64VMCSPhys);
4147 pVCpu->hwaccm.s.vmx.lasterror.ulVMCSRevision = *(uint32_t *)pVCpu->hwaccm.s.vmx.pVMCS;
4148 pVCpu->hwaccm.s.vmx.lasterror.idEnteredCpu = pVCpu->hwaccm.s.idEnteredCpu;
4149 pVCpu->hwaccm.s.vmx.lasterror.idCurrentCpu = RTMpCpuId();
4150 }
4151
4152 /* Just set the correct state here instead of trying to catch every goto above. */
4153 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED, VMCPUSTATE_STARTED_EXEC);
4154
4155#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
4156 /* Restore interrupts if we exitted after disabling them. */
4157 if (uOldEFlags != ~(RTCCUINTREG)0)
4158 ASMSetFlags(uOldEFlags);
4159#endif
4160
4161 STAM_STATS({
4162 if (fStatExit2Started) STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2, y);
4163 else if (fStatEntryStarted) STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatEntry, x);
4164 });
4165 Log2(("X"));
4166 return rc;
4167}
4168
4169
4170/**
4171 * Enters the VT-x session
4172 *
4173 * @returns VBox status code.
4174 * @param pVM The VM to operate on.
4175 * @param pVCpu The VMCPU to operate on.
4176 * @param pCpu CPU info struct
4177 */
4178VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHWACCM_CPUINFO pCpu)
4179{
4180 Assert(pVM->hwaccm.s.vmx.fSupported);
4181
4182 unsigned cr4 = ASMGetCR4();
4183 if (!(cr4 & X86_CR4_VMXE))
4184 {
4185 AssertMsgFailed(("X86_CR4_VMXE should be set!\n"));
4186 return VERR_VMX_X86_CR4_VMXE_CLEARED;
4187 }
4188
4189 /* Activate the VM Control Structure. */
4190 int rc = VMXActivateVMCS(pVCpu->hwaccm.s.vmx.pVMCSPhys);
4191 if (RT_FAILURE(rc))
4192 return rc;
4193
4194 pVCpu->hwaccm.s.fResumeVM = false;
4195 return VINF_SUCCESS;
4196}
4197
4198
4199/**
4200 * Leaves the VT-x session
4201 *
4202 * @returns VBox status code.
4203 * @param pVM The VM to operate on.
4204 * @param pVCpu The VMCPU to operate on.
4205 * @param pCtx CPU context
4206 */
4207VMMR0DECL(int) VMXR0Leave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4208{
4209 Assert(pVM->hwaccm.s.vmx.fSupported);
4210
4211#ifdef DEBUG
4212 if (CPUMIsHyperDebugStateActive(pVCpu))
4213 {
4214 CPUMR0LoadHostDebugState(pVM, pVCpu);
4215 Assert(pVCpu->hwaccm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT);
4216 }
4217 else
4218#endif
4219 /* Save the guest debug state if necessary. */
4220 if (CPUMIsGuestDebugStateActive(pVCpu))
4221 {
4222 CPUMR0SaveGuestDebugState(pVM, pVCpu, pCtx, true /* save DR6 */);
4223
4224 /* Enable drx move intercepts again. */
4225 pVCpu->hwaccm.s.vmx.proc_ctls |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
4226 int rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
4227 AssertRC(rc);
4228
4229 /* Resync the debug registers the next time. */
4230 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_DEBUG;
4231 }
4232 else
4233 Assert(pVCpu->hwaccm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT);
4234
4235 /* Clear VM Control Structure. Marking it inactive, clearing implementation specific data and writing back VMCS data to memory. */
4236 int rc = VMXClearVMCS(pVCpu->hwaccm.s.vmx.pVMCSPhys);
4237 AssertRC(rc);
4238
4239 return VINF_SUCCESS;
4240}
4241
4242/**
4243 * Flush the TLB (EPT)
4244 *
4245 * @returns VBox status code.
4246 * @param pVM The VM to operate on.
4247 * @param pVCpu The VM CPU to operate on.
4248 * @param enmFlush Type of flush
4249 * @param GCPhys Physical address of the page to flush
4250 */
4251static void vmxR0FlushEPT(PVM pVM, PVMCPU pVCpu, VMX_FLUSH enmFlush, RTGCPHYS GCPhys)
4252{
4253 uint64_t descriptor[2];
4254
4255 LogFlow(("vmxR0FlushEPT %d %RGv\n", enmFlush, GCPhys));
4256 Assert(pVM->hwaccm.s.fNestedPaging);
4257 descriptor[0] = pVCpu->hwaccm.s.vmx.GCPhysEPTP;
4258 descriptor[1] = GCPhys;
4259 int rc = VMXR0InvEPT(enmFlush, &descriptor[0]);
4260 AssertRC(rc);
4261}
4262
4263#ifdef HWACCM_VTX_WITH_VPID
4264/**
4265 * Flush the TLB (EPT)
4266 *
4267 * @returns VBox status code.
4268 * @param pVM The VM to operate on.
4269 * @param pVCpu The VM CPU to operate on.
4270 * @param enmFlush Type of flush
4271 * @param GCPtr Virtual address of the page to flush
4272 */
4273static void vmxR0FlushVPID(PVM pVM, PVMCPU pVCpu, VMX_FLUSH enmFlush, RTGCPTR GCPtr)
4274{
4275#if HC_ARCH_BITS == 32
4276 /* If we get a flush in 64 bits guest mode, then force a full TLB flush. Invvpid probably takes only 32 bits addresses. (@todo) */
4277 if ( CPUMIsGuestInLongMode(pVCpu)
4278 && !VMX_IS_64BIT_HOST_MODE())
4279 {
4280 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
4281 }
4282 else
4283#endif
4284 {
4285 uint64_t descriptor[2];
4286
4287 Assert(pVM->hwaccm.s.vmx.fVPID);
4288 descriptor[0] = pVCpu->hwaccm.s.uCurrentASID;
4289 descriptor[1] = GCPtr;
4290 int rc = VMXR0InvVPID(enmFlush, &descriptor[0]);
4291 AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvVPID %x %x %RGv failed with %d\n", enmFlush, pVCpu->hwaccm.s.uCurrentASID, GCPtr, rc));
4292 }
4293}
4294#endif /* HWACCM_VTX_WITH_VPID */
4295
4296/**
4297 * Invalidates a guest page
4298 *
4299 * @returns VBox status code.
4300 * @param pVM The VM to operate on.
4301 * @param pVCpu The VM CPU to operate on.
4302 * @param GCVirt Page to invalidate
4303 */
4304VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
4305{
4306 bool fFlushPending = VMCPU_FF_ISSET(pVCpu, VMCPU_FF_TLB_FLUSH);
4307
4308 Log2(("VMXR0InvalidatePage %RGv\n", GCVirt));
4309
4310 /* Only relevant if we want to use VPID.
4311 * In the nested paging case we still see such calls, but
4312 * can safely ignore them. (e.g. after cr3 updates)
4313 */
4314#ifdef HWACCM_VTX_WITH_VPID
4315 /* Skip it if a TLB flush is already pending. */
4316 if ( !fFlushPending
4317 && pVM->hwaccm.s.vmx.fVPID)
4318 vmxR0FlushVPID(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushPage, GCVirt);
4319#endif /* HWACCM_VTX_WITH_VPID */
4320
4321 return VINF_SUCCESS;
4322}
4323
4324/**
4325 * Invalidates a guest page by physical address
4326 *
4327 * NOTE: Assumes the current instruction references this physical page though a virtual address!!
4328 *
4329 * @returns VBox status code.
4330 * @param pVM The VM to operate on.
4331 * @param pVCpu The VM CPU to operate on.
4332 * @param GCPhys Page to invalidate
4333 */
4334VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
4335{
4336 bool fFlushPending = VMCPU_FF_ISSET(pVCpu, VMCPU_FF_TLB_FLUSH);
4337
4338 Assert(pVM->hwaccm.s.fNestedPaging);
4339
4340 LogFlow(("VMXR0InvalidatePhysPage %RGp\n", GCPhys));
4341
4342 /* Skip it if a TLB flush is already pending. */
4343 if (!fFlushPending)
4344 vmxR0FlushEPT(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushPage, GCPhys);
4345
4346 return VINF_SUCCESS;
4347}
4348
4349/**
4350 * Report world switch error and dump some useful debug info
4351 *
4352 * @param pVM The VM to operate on.
4353 * @param pVCpu The VMCPU to operate on.
4354 * @param rc Return code
4355 * @param pCtx Current CPU context (not updated)
4356 */
4357static void VMXR0ReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rc, PCPUMCTX pCtx)
4358{
4359 switch (rc)
4360 {
4361 case VERR_VMX_INVALID_VMXON_PTR:
4362 AssertFailed();
4363 break;
4364
4365 case VERR_VMX_UNABLE_TO_START_VM:
4366 case VERR_VMX_UNABLE_TO_RESUME_VM:
4367 {
4368 int rc2;
4369 RTCCUINTREG exitReason, instrError;
4370
4371 rc2 = VMXReadVMCS(VMX_VMCS32_RO_EXIT_REASON, &exitReason);
4372 rc2 |= VMXReadVMCS(VMX_VMCS32_RO_VM_INSTR_ERROR, &instrError);
4373 AssertRC(rc2);
4374 if (rc2 == VINF_SUCCESS)
4375 {
4376 Log(("Unable to start/resume VM for reason: %x. Instruction error %x\n", (uint32_t)exitReason, (uint32_t)instrError));
4377 Log(("Current stack %08x\n", &rc2));
4378
4379 pVCpu->hwaccm.s.vmx.lasterror.ulInstrError = instrError;
4380 pVCpu->hwaccm.s.vmx.lasterror.ulExitReason = exitReason;
4381
4382#ifdef VBOX_STRICT
4383 RTGDTR gdtr;
4384 PCX86DESCHC pDesc;
4385 RTCCUINTREG val;
4386
4387 ASMGetGDTR(&gdtr);
4388
4389 VMXReadVMCS(VMX_VMCS64_GUEST_RIP, &val);
4390 Log(("Old eip %RGv new %RGv\n", (RTGCPTR)pCtx->rip, (RTGCPTR)val));
4391 VMXReadVMCS(VMX_VMCS_CTRL_PIN_EXEC_CONTROLS, &val);
4392 Log(("VMX_VMCS_CTRL_PIN_EXEC_CONTROLS %08x\n", val));
4393 VMXReadVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, &val);
4394 Log(("VMX_VMCS_CTRL_PROC_EXEC_CONTROLS %08x\n", val));
4395 VMXReadVMCS(VMX_VMCS_CTRL_ENTRY_CONTROLS, &val);
4396 Log(("VMX_VMCS_CTRL_ENTRY_CONTROLS %08x\n", val));
4397 VMXReadVMCS(VMX_VMCS_CTRL_EXIT_CONTROLS, &val);
4398 Log(("VMX_VMCS_CTRL_EXIT_CONTROLS %08x\n", val));
4399
4400 VMXReadVMCS(VMX_VMCS_HOST_CR0, &val);
4401 Log(("VMX_VMCS_HOST_CR0 %08x\n", val));
4402
4403 VMXReadVMCS(VMX_VMCS_HOST_CR3, &val);
4404 Log(("VMX_VMCS_HOST_CR3 %08x\n", val));
4405
4406 VMXReadVMCS(VMX_VMCS_HOST_CR4, &val);
4407 Log(("VMX_VMCS_HOST_CR4 %08x\n", val));
4408
4409 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_CS, &val);
4410 Log(("VMX_VMCS_HOST_FIELD_CS %08x\n", val));
4411
4412 VMXReadVMCS(VMX_VMCS_GUEST_RFLAGS, &val);
4413 Log(("VMX_VMCS_GUEST_RFLAGS %08x\n", val));
4414
4415 if (val < gdtr.cbGdt)
4416 {
4417 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
4418 HWACCMR0DumpDescriptor(pDesc, val, "CS: ");
4419 }
4420
4421 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_DS, &val);
4422 Log(("VMX_VMCS_HOST_FIELD_DS %08x\n", val));
4423 if (val < gdtr.cbGdt)
4424 {
4425 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
4426 HWACCMR0DumpDescriptor(pDesc, val, "DS: ");
4427 }
4428
4429 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_ES, &val);
4430 Log(("VMX_VMCS_HOST_FIELD_ES %08x\n", val));
4431 if (val < gdtr.cbGdt)
4432 {
4433 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
4434 HWACCMR0DumpDescriptor(pDesc, val, "ES: ");
4435 }
4436
4437 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_FS, &val);
4438 Log(("VMX_VMCS16_HOST_FIELD_FS %08x\n", val));
4439 if (val < gdtr.cbGdt)
4440 {
4441 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
4442 HWACCMR0DumpDescriptor(pDesc, val, "FS: ");
4443 }
4444
4445 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_GS, &val);
4446 Log(("VMX_VMCS16_HOST_FIELD_GS %08x\n", val));
4447 if (val < gdtr.cbGdt)
4448 {
4449 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
4450 HWACCMR0DumpDescriptor(pDesc, val, "GS: ");
4451 }
4452
4453 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_SS, &val);
4454 Log(("VMX_VMCS16_HOST_FIELD_SS %08x\n", val));
4455 if (val < gdtr.cbGdt)
4456 {
4457 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
4458 HWACCMR0DumpDescriptor(pDesc, val, "SS: ");
4459 }
4460
4461 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_TR, &val);
4462 Log(("VMX_VMCS16_HOST_FIELD_TR %08x\n", val));
4463 if (val < gdtr.cbGdt)
4464 {
4465 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
4466 HWACCMR0DumpDescriptor(pDesc, val, "TR: ");
4467 }
4468
4469 VMXReadVMCS(VMX_VMCS_HOST_TR_BASE, &val);
4470 Log(("VMX_VMCS_HOST_TR_BASE %RHv\n", val));
4471
4472 VMXReadVMCS(VMX_VMCS_HOST_GDTR_BASE, &val);
4473 Log(("VMX_VMCS_HOST_GDTR_BASE %RHv\n", val));
4474 VMXReadVMCS(VMX_VMCS_HOST_IDTR_BASE, &val);
4475 Log(("VMX_VMCS_HOST_IDTR_BASE %RHv\n", val));
4476
4477 VMXReadVMCS(VMX_VMCS32_HOST_SYSENTER_CS, &val);
4478 Log(("VMX_VMCS_HOST_SYSENTER_CS %08x\n", val));
4479
4480 VMXReadVMCS(VMX_VMCS_HOST_SYSENTER_EIP, &val);
4481 Log(("VMX_VMCS_HOST_SYSENTER_EIP %RHv\n", val));
4482
4483 VMXReadVMCS(VMX_VMCS_HOST_SYSENTER_ESP, &val);
4484 Log(("VMX_VMCS_HOST_SYSENTER_ESP %RHv\n", val));
4485
4486 VMXReadVMCS(VMX_VMCS_HOST_RSP, &val);
4487 Log(("VMX_VMCS_HOST_RSP %RHv\n", val));
4488 VMXReadVMCS(VMX_VMCS_HOST_RIP, &val);
4489 Log(("VMX_VMCS_HOST_RIP %RHv\n", val));
4490
4491# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4492 if (VMX_IS_64BIT_HOST_MODE())
4493 {
4494 Log(("MSR_K6_EFER = %RX64\n", ASMRdMsr(MSR_K6_EFER)));
4495 Log(("MSR_K6_STAR = %RX64\n", ASMRdMsr(MSR_K6_STAR)));
4496 Log(("MSR_K8_LSTAR = %RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
4497 Log(("MSR_K8_CSTAR = %RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
4498 Log(("MSR_K8_SF_MASK = %RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
4499 }
4500# endif
4501#endif /* VBOX_STRICT */
4502 }
4503 break;
4504 }
4505
4506 default:
4507 /* impossible */
4508 AssertMsgFailed(("%Rrc (%#x)\n", rc, rc));
4509 break;
4510 }
4511}
4512
4513#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4514/**
4515 * Prepares for and executes VMLAUNCH (64 bits guest mode)
4516 *
4517 * @returns VBox status code
4518 * @param fResume vmlauch/vmresume
4519 * @param pCtx Guest context
4520 * @param pCache VMCS cache
4521 * @param pVM The VM to operate on.
4522 * @param pVCpu The VMCPU to operate on.
4523 */
4524DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
4525{
4526 uint32_t aParam[6];
4527 PHWACCM_CPUINFO pCpu;
4528 RTHCPHYS pPageCpuPhys;
4529 int rc;
4530
4531 pCpu = HWACCMR0GetCurrentCpu();
4532 pPageCpuPhys = RTR0MemObjGetPagePhysAddr(pCpu->pMemObj, 0);
4533
4534#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4535 pCache->uPos = 1;
4536 pCache->interPD = PGMGetInterPaeCR3(pVM);
4537 pCache->pSwitcher = (uint64_t)pVM->hwaccm.s.pfnHost32ToGuest64R0;
4538#endif
4539
4540#ifdef DEBUG
4541 pCache->TestIn.pPageCpuPhys = 0;
4542 pCache->TestIn.pVMCSPhys = 0;
4543 pCache->TestIn.pCache = 0;
4544 pCache->TestOut.pVMCSPhys = 0;
4545 pCache->TestOut.pCache = 0;
4546 pCache->TestOut.pCtx = 0;
4547 pCache->TestOut.eflags = 0;
4548#endif
4549
4550 aParam[0] = (uint32_t)(pPageCpuPhys); /* Param 1: VMXON physical address - Lo. */
4551 aParam[1] = (uint32_t)(pPageCpuPhys >> 32); /* Param 1: VMXON physical address - Hi. */
4552 aParam[2] = (uint32_t)(pVCpu->hwaccm.s.vmx.pVMCSPhys); /* Param 2: VMCS physical address - Lo. */
4553 aParam[3] = (uint32_t)(pVCpu->hwaccm.s.vmx.pVMCSPhys >> 32); /* Param 2: VMCS physical address - Hi. */
4554 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hwaccm.s.vmx.VMCSCache);
4555 aParam[5] = 0;
4556
4557#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4558 pCtx->dr[4] = pVM->hwaccm.s.vmx.pScratchPhys + 16 + 8;
4559 *(uint32_t *)(pVM->hwaccm.s.vmx.pScratch + 16 + 8) = 1;
4560#endif
4561 rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, pVM->hwaccm.s.pfnVMXGCStartVM64, 6, &aParam[0]);
4562
4563#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4564 Assert(*(uint32_t *)(pVM->hwaccm.s.vmx.pScratch + 16 + 8) == 5);
4565 Assert(pCtx->dr[4] == 10);
4566 *(uint32_t *)(pVM->hwaccm.s.vmx.pScratch + 16 + 8) = 0xff;
4567#endif
4568
4569#ifdef DEBUG
4570 AssertMsg(pCache->TestIn.pPageCpuPhys == pPageCpuPhys, ("%RHp vs %RHp\n", pCache->TestIn.pPageCpuPhys, pPageCpuPhys));
4571 AssertMsg(pCache->TestIn.pVMCSPhys == pVCpu->hwaccm.s.vmx.pVMCSPhys, ("%RHp vs %RHp\n", pCache->TestIn.pVMCSPhys, pVCpu->hwaccm.s.vmx.pVMCSPhys));
4572 AssertMsg(pCache->TestIn.pVMCSPhys == pCache->TestOut.pVMCSPhys, ("%RHp vs %RHp\n", pCache->TestIn.pVMCSPhys, pCache->TestOut.pVMCSPhys));
4573 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache, pCache->TestOut.pCache));
4574 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hwaccm.s.vmx.VMCSCache), ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hwaccm.s.vmx.VMCSCache)));
4575 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx, pCache->TestOut.pCtx));
4576 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
4577#endif
4578 return rc;
4579}
4580
4581/**
4582 * Executes the specified handler in 64 mode
4583 *
4584 * @returns VBox status code.
4585 * @param pVM The VM to operate on.
4586 * @param pVCpu The VMCPU to operate on.
4587 * @param pCtx Guest context
4588 * @param pfnHandler RC handler
4589 * @param cbParam Number of parameters
4590 * @param paParam Array of 32 bits parameters
4591 */
4592VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, RTRCPTR pfnHandler, uint32_t cbParam, uint32_t *paParam)
4593{
4594 int rc, rc2;
4595 PHWACCM_CPUINFO pCpu;
4596 RTHCPHYS pPageCpuPhys;
4597 RTHCUINTREG uOldEFlags;
4598
4599 /* @todo This code is not guest SMP safe (hyper stack and switchers) */
4600 AssertReturn(pVM->cCpus == 1, VERR_TOO_MANY_CPUS);
4601 AssertReturn(pVM->hwaccm.s.pfnHost32ToGuest64R0, VERR_INTERNAL_ERROR);
4602 Assert(pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hwaccm.s.vmx.VMCSCache.Write.aField));
4603 Assert(pVCpu->hwaccm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hwaccm.s.vmx.VMCSCache.Read.aField));
4604
4605#ifdef VBOX_STRICT
4606 for (unsigned i=0;i<pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries;i++)
4607 Assert(vmxR0IsValidWriteField(pVCpu->hwaccm.s.vmx.VMCSCache.Write.aField[i]));
4608
4609 for (unsigned i=0;i<pVCpu->hwaccm.s.vmx.VMCSCache.Read.cValidEntries;i++)
4610 Assert(vmxR0IsValidReadField(pVCpu->hwaccm.s.vmx.VMCSCache.Read.aField[i]));
4611#endif
4612
4613 /* Disable interrupts. */
4614 uOldEFlags = ASMIntDisableFlags();
4615
4616 pCpu = HWACCMR0GetCurrentCpu();
4617 pPageCpuPhys = RTR0MemObjGetPagePhysAddr(pCpu->pMemObj, 0);
4618
4619 /* Clear VM Control Structure. Marking it inactive, clearing implementation specific data and writing back VMCS data to memory. */
4620 VMXClearVMCS(pVCpu->hwaccm.s.vmx.pVMCSPhys);
4621
4622 /* Leave VMX Root Mode. */
4623 VMXDisable();
4624
4625 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4626
4627 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVM));
4628 CPUMSetHyperEIP(pVCpu, pfnHandler);
4629 for (int i=(int)cbParam-1;i>=0;i--)
4630 CPUMPushHyper(pVCpu, paParam[i]);
4631
4632 STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatWorldSwitch3264, z);
4633 /* Call switcher. */
4634 rc = pVM->hwaccm.s.pfnHost32ToGuest64R0(pVM);
4635 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatWorldSwitch3264, z);
4636
4637 /* Make sure the VMX instructions don't cause #UD faults. */
4638 ASMSetCR4(ASMGetCR4() | X86_CR4_VMXE);
4639
4640 /* Enter VMX Root Mode */
4641 rc2 = VMXEnable(pPageCpuPhys);
4642 if (RT_FAILURE(rc2))
4643 {
4644 if (pVM)
4645 VMXR0CheckError(pVM, pVCpu, rc2);
4646 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4647 ASMSetFlags(uOldEFlags);
4648 return VERR_VMX_VMXON_FAILED;
4649 }
4650
4651 rc2 = VMXActivateVMCS(pVCpu->hwaccm.s.vmx.pVMCSPhys);
4652 AssertRC(rc2);
4653 Assert(!(ASMGetFlags() & X86_EFL_IF));
4654 ASMSetFlags(uOldEFlags);
4655 return rc;
4656}
4657
4658#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
4659
4660
4661#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
4662/**
4663 * Executes VMWRITE
4664 *
4665 * @returns VBox status code
4666 * @param pVCpu The VMCPU to operate on.
4667 * @param idxField VMCS index
4668 * @param u64Val 16, 32 or 64 bits value
4669 */
4670VMMR0DECL(int) VMXWriteVMCS64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4671{
4672 int rc;
4673
4674 switch (idxField)
4675 {
4676 case VMX_VMCS_CTRL_TSC_OFFSET_FULL:
4677 case VMX_VMCS_CTRL_IO_BITMAP_A_FULL:
4678 case VMX_VMCS_CTRL_IO_BITMAP_B_FULL:
4679 case VMX_VMCS_CTRL_MSR_BITMAP_FULL:
4680 case VMX_VMCS_CTRL_VMEXIT_MSR_STORE_FULL:
4681 case VMX_VMCS_CTRL_VMEXIT_MSR_LOAD_FULL:
4682 case VMX_VMCS_CTRL_VMENTRY_MSR_LOAD_FULL:
4683 case VMX_VMCS_CTRL_VAPIC_PAGEADDR_FULL:
4684 case VMX_VMCS_CTRL_APIC_ACCESSADDR_FULL:
4685 case VMX_VMCS_GUEST_LINK_PTR_FULL:
4686 case VMX_VMCS_GUEST_PDPTR0_FULL:
4687 case VMX_VMCS_GUEST_PDPTR1_FULL:
4688 case VMX_VMCS_GUEST_PDPTR2_FULL:
4689 case VMX_VMCS_GUEST_PDPTR3_FULL:
4690 case VMX_VMCS_GUEST_DEBUGCTL_FULL:
4691 case VMX_VMCS_GUEST_EFER_FULL:
4692 case VMX_VMCS_CTRL_EPTP_FULL:
4693 /* These fields consist of two parts, which are both writable in 32 bits mode. */
4694 rc = VMXWriteVMCS32(idxField, u64Val);
4695 rc |= VMXWriteVMCS32(idxField + 1, (uint32_t)(u64Val >> 32ULL));
4696 AssertRC(rc);
4697 return rc;
4698
4699 case VMX_VMCS64_GUEST_LDTR_BASE:
4700 case VMX_VMCS64_GUEST_TR_BASE:
4701 case VMX_VMCS64_GUEST_GDTR_BASE:
4702 case VMX_VMCS64_GUEST_IDTR_BASE:
4703 case VMX_VMCS64_GUEST_SYSENTER_EIP:
4704 case VMX_VMCS64_GUEST_SYSENTER_ESP:
4705 case VMX_VMCS64_GUEST_CR0:
4706 case VMX_VMCS64_GUEST_CR4:
4707 case VMX_VMCS64_GUEST_CR3:
4708 case VMX_VMCS64_GUEST_DR7:
4709 case VMX_VMCS64_GUEST_RIP:
4710 case VMX_VMCS64_GUEST_RSP:
4711 case VMX_VMCS64_GUEST_CS_BASE:
4712 case VMX_VMCS64_GUEST_DS_BASE:
4713 case VMX_VMCS64_GUEST_ES_BASE:
4714 case VMX_VMCS64_GUEST_FS_BASE:
4715 case VMX_VMCS64_GUEST_GS_BASE:
4716 case VMX_VMCS64_GUEST_SS_BASE:
4717 /* Queue a 64 bits value as we can't set it in 32 bits host mode. */
4718 if (u64Val >> 32ULL)
4719 rc = VMXWriteCachedVMCSEx(pVCpu, idxField, u64Val);
4720 else
4721 rc = VMXWriteVMCS32(idxField, (uint32_t)u64Val);
4722
4723 return rc;
4724
4725 default:
4726 AssertMsgFailed(("Unexpected field %x\n", idxField));
4727 return VERR_INVALID_PARAMETER;
4728 }
4729}
4730
4731/**
4732 * Cache VMCS writes for performance reasons (Darwin) and for running 64 bits guests on 32 bits hosts.
4733 *
4734 * @param pVCpu The VMCPU to operate on.
4735 * @param idxField VMCS field
4736 * @param u64Val Value
4737 */
4738VMMR0DECL(int) VMXWriteCachedVMCSEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4739{
4740 PVMCSCACHE pCache = &pVCpu->hwaccm.s.vmx.VMCSCache;
4741
4742 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1, ("entries=%x\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
4743
4744 /* Make sure there are no duplicates. */
4745 for (unsigned i=0;i<pCache->Write.cValidEntries;i++)
4746 {
4747 if (pCache->Write.aField[i] == idxField)
4748 {
4749 pCache->Write.aFieldVal[i] = u64Val;
4750 return VINF_SUCCESS;
4751 }
4752 }
4753
4754 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
4755 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
4756 pCache->Write.cValidEntries++;
4757 return VINF_SUCCESS;
4758}
4759
4760#endif /* HC_ARCH_BITS == 32 && !VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0 */
4761
4762#ifdef VBOX_STRICT
4763static bool vmxR0IsValidReadField(uint32_t idxField)
4764{
4765 switch(idxField)
4766 {
4767 case VMX_VMCS64_GUEST_RIP:
4768 case VMX_VMCS64_GUEST_RSP:
4769 case VMX_VMCS_GUEST_RFLAGS:
4770 case VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE:
4771 case VMX_VMCS_CTRL_CR0_READ_SHADOW:
4772 case VMX_VMCS64_GUEST_CR0:
4773 case VMX_VMCS_CTRL_CR4_READ_SHADOW:
4774 case VMX_VMCS64_GUEST_CR4:
4775 case VMX_VMCS64_GUEST_DR7:
4776 case VMX_VMCS32_GUEST_SYSENTER_CS:
4777 case VMX_VMCS64_GUEST_SYSENTER_EIP:
4778 case VMX_VMCS64_GUEST_SYSENTER_ESP:
4779 case VMX_VMCS32_GUEST_GDTR_LIMIT:
4780 case VMX_VMCS64_GUEST_GDTR_BASE:
4781 case VMX_VMCS32_GUEST_IDTR_LIMIT:
4782 case VMX_VMCS64_GUEST_IDTR_BASE:
4783 case VMX_VMCS16_GUEST_FIELD_CS:
4784 case VMX_VMCS32_GUEST_CS_LIMIT:
4785 case VMX_VMCS64_GUEST_CS_BASE:
4786 case VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS:
4787 case VMX_VMCS16_GUEST_FIELD_DS:
4788 case VMX_VMCS32_GUEST_DS_LIMIT:
4789 case VMX_VMCS64_GUEST_DS_BASE:
4790 case VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS:
4791 case VMX_VMCS16_GUEST_FIELD_ES:
4792 case VMX_VMCS32_GUEST_ES_LIMIT:
4793 case VMX_VMCS64_GUEST_ES_BASE:
4794 case VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS:
4795 case VMX_VMCS16_GUEST_FIELD_FS:
4796 case VMX_VMCS32_GUEST_FS_LIMIT:
4797 case VMX_VMCS64_GUEST_FS_BASE:
4798 case VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS:
4799 case VMX_VMCS16_GUEST_FIELD_GS:
4800 case VMX_VMCS32_GUEST_GS_LIMIT:
4801 case VMX_VMCS64_GUEST_GS_BASE:
4802 case VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS:
4803 case VMX_VMCS16_GUEST_FIELD_SS:
4804 case VMX_VMCS32_GUEST_SS_LIMIT:
4805 case VMX_VMCS64_GUEST_SS_BASE:
4806 case VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS:
4807 case VMX_VMCS16_GUEST_FIELD_LDTR:
4808 case VMX_VMCS32_GUEST_LDTR_LIMIT:
4809 case VMX_VMCS64_GUEST_LDTR_BASE:
4810 case VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS:
4811 case VMX_VMCS16_GUEST_FIELD_TR:
4812 case VMX_VMCS32_GUEST_TR_LIMIT:
4813 case VMX_VMCS64_GUEST_TR_BASE:
4814 case VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS:
4815 case VMX_VMCS32_RO_EXIT_REASON:
4816 case VMX_VMCS32_RO_VM_INSTR_ERROR:
4817 case VMX_VMCS32_RO_EXIT_INSTR_LENGTH:
4818 case VMX_VMCS32_RO_EXIT_INTERRUPTION_ERRCODE:
4819 case VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO:
4820 case VMX_VMCS32_RO_EXIT_INSTR_INFO:
4821 case VMX_VMCS_RO_EXIT_QUALIFICATION:
4822 case VMX_VMCS32_RO_IDT_INFO:
4823 case VMX_VMCS32_RO_IDT_ERRCODE:
4824 case VMX_VMCS64_GUEST_CR3:
4825 case VMX_VMCS_EXIT_PHYS_ADDR_FULL:
4826 return true;
4827 }
4828 return false;
4829}
4830
4831static bool vmxR0IsValidWriteField(uint32_t idxField)
4832{
4833 switch(idxField)
4834 {
4835 case VMX_VMCS64_GUEST_LDTR_BASE:
4836 case VMX_VMCS64_GUEST_TR_BASE:
4837 case VMX_VMCS64_GUEST_GDTR_BASE:
4838 case VMX_VMCS64_GUEST_IDTR_BASE:
4839 case VMX_VMCS64_GUEST_SYSENTER_EIP:
4840 case VMX_VMCS64_GUEST_SYSENTER_ESP:
4841 case VMX_VMCS64_GUEST_CR0:
4842 case VMX_VMCS64_GUEST_CR4:
4843 case VMX_VMCS64_GUEST_CR3:
4844 case VMX_VMCS64_GUEST_DR7:
4845 case VMX_VMCS64_GUEST_RIP:
4846 case VMX_VMCS64_GUEST_RSP:
4847 case VMX_VMCS64_GUEST_CS_BASE:
4848 case VMX_VMCS64_GUEST_DS_BASE:
4849 case VMX_VMCS64_GUEST_ES_BASE:
4850 case VMX_VMCS64_GUEST_FS_BASE:
4851 case VMX_VMCS64_GUEST_GS_BASE:
4852 case VMX_VMCS64_GUEST_SS_BASE:
4853 return true;
4854 }
4855 return false;
4856}
4857
4858#endif
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette