VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HWVMXR0.cpp@ 39700

Last change on this file since 39700 was 39700, checked in by vboxsync, 13 years ago

cleanup
  • Property svn:eol-style set to native
  • Property svn:keywords set to Id
File size: 200.3 KB
Line 
1/* $Id: HWVMXR0.cpp 39700 2012-01-04 15:06:46Z vboxsync $ */
2/** @file
3 * HM VMX (VT-x) - Host Context Ring 0.
4 */
5
6/*
7 * Copyright (C) 2006-2011 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18
19/*******************************************************************************
20* Header Files *
21*******************************************************************************/
22#define LOG_GROUP LOG_GROUP_HWACCM
23#include <iprt/asm-amd64-x86.h>
24#include <VBox/vmm/hwaccm.h>
25#include <VBox/vmm/pgm.h>
26#include <VBox/vmm/dbgf.h>
27#include <VBox/vmm/selm.h>
28#include <VBox/vmm/iom.h>
29#include <VBox/vmm/rem.h>
30#include <VBox/vmm/tm.h>
31#include "HWACCMInternal.h"
32#include <VBox/vmm/vm.h>
33#include <VBox/vmm/pdmapi.h>
34#include <VBox/err.h>
35#include <VBox/log.h>
36#include <iprt/assert.h>
37#include <iprt/param.h>
38#include <iprt/string.h>
39#include <iprt/time.h>
40#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
41# include <iprt/thread.h>
42#endif
43#include <iprt/x86.h>
44#include "HWVMXR0.h"
45
46/*******************************************************************************
47* Defined Constants And Macros *
48*******************************************************************************/
49#if defined(RT_ARCH_AMD64)
50# define VMX_IS_64BIT_HOST_MODE() (true)
51#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
52# define VMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
53#else
54# define VMX_IS_64BIT_HOST_MODE() (false)
55#endif
56
57/*******************************************************************************
58* Global Variables *
59*******************************************************************************/
60/* IO operation lookup arrays. */
61static uint32_t const g_aIOSize[4] = {1, 2, 0, 4};
62static uint32_t const g_aIOOpAnd[4] = {0xff, 0xffff, 0, 0xffffffff};
63
64#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
65/** See HWACCMR0A.asm. */
66extern "C" uint32_t g_fVMXIs64bitHost;
67#endif
68
69/*******************************************************************************
70* Local Functions *
71*******************************************************************************/
72static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, VBOXSTRICTRC rc, PCPUMCTX pCtx);
73static void hmR0VmxSetupTLBEPT(PVM pVM, PVMCPU pVCpu);
74static void hmR0VmxSetupTLBVPID(PVM pVM, PVMCPU pVCpu);
75static void hmR0VmxSetupTLBDummy(PVM pVM, PVMCPU pVCpu);
76static void hmR0VmxFlushEPT(PVM pVM, PVMCPU pVCpu, VMX_FLUSH enmFlush, RTGCPHYS GCPhys);
77static void hmR0VmxFlushVPID(PVM pVM, PVMCPU pVCpu, VMX_FLUSH enmFlush, RTGCPTR GCPtr);
78static void hmR0VmxUpdateExceptionBitmap(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
79static void hmR0VmxSetMSRPermission(PVMCPU pVCpu, unsigned ulMSR, bool fRead, bool fWrite);
80
81
82static void hmR0VmxCheckError(PVM pVM, PVMCPU pVCpu, int rc)
83{
84 if (rc == VERR_VMX_GENERIC)
85 {
86 RTCCUINTREG instrError;
87
88 VMXReadVMCS(VMX_VMCS32_RO_VM_INSTR_ERROR, &instrError);
89 pVCpu->hwaccm.s.vmx.lasterror.ulInstrError = instrError;
90 }
91 pVM->hwaccm.s.lLastError = rc;
92}
93
94/**
95 * Sets up and activates VT-x on the current CPU
96 *
97 * @returns VBox status code.
98 * @param pCpu CPU info struct
99 * @param pVM The VM to operate on. (can be NULL after a resume!!)
100 * @param pvCpuPage Pointer to the global cpu page.
101 * @param HCPhysCpuPage Physical address of the global cpu page.
102 */
103VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBLCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
104{
105 AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
106 AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
107 NOREF(pCpu);
108
109 if (pVM)
110 {
111 /* Set revision dword at the beginning of the VMXON structure. */
112 *(uint32_t *)pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hwaccm.s.vmx.msr.vmx_basic_info);
113 }
114
115 /** @todo we should unmap the two pages from the virtual address space in order to prevent accidental corruption.
116 * (which can have very bad consequences!!!)
117 */
118
119 if (ASMGetCR4() & X86_CR4_VMXE)
120 return VERR_VMX_IN_VMX_ROOT_MODE;
121
122 /* Make sure the VMX instructions don't cause #UD faults. */
123 ASMSetCR4(ASMGetCR4() | X86_CR4_VMXE);
124
125 /* Enter VMX Root Mode. */
126 int rc = VMXEnable(HCPhysCpuPage);
127 if (RT_FAILURE(rc))
128 {
129 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
130 return VERR_VMX_VMXON_FAILED;
131 }
132 return VINF_SUCCESS;
133}
134
135/**
136 * Deactivates VT-x on the current CPU
137 *
138 * @returns VBox status code.
139 * @param pCpu CPU info struct
140 * @param pvCpuPage Pointer to the global cpu page.
141 * @param HCPhysCpuPage Physical address of the global cpu page.
142 */
143VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBLCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
144{
145 AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
146 AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
147 NOREF(pCpu);
148
149 /* If we're somehow not in VMX root mode, then we shouldn't dare leaving it. */
150 if (!(ASMGetCR4() & X86_CR4_VMXE))
151 return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
152
153 /* Leave VMX Root Mode. */
154 VMXDisable();
155
156 /* And clear the X86_CR4_VMXE bit. */
157 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
158 return VINF_SUCCESS;
159}
160
161/**
162 * Does Ring-0 per VM VT-x init.
163 *
164 * @returns VBox status code.
165 * @param pVM The VM to operate on.
166 */
167VMMR0DECL(int) VMXR0InitVM(PVM pVM)
168{
169 int rc;
170
171#ifdef LOG_ENABLED
172 SUPR0Printf("VMXR0InitVM %x\n", pVM);
173#endif
174
175 pVM->hwaccm.s.vmx.pMemObjAPIC = NIL_RTR0MEMOBJ;
176
177 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
178 {
179 /* Allocate one page for the APIC physical page (serves for filtering accesses). */
180 rc = RTR0MemObjAllocCont(&pVM->hwaccm.s.vmx.pMemObjAPIC, PAGE_SIZE, true /* executable R0 mapping */);
181 AssertRC(rc);
182 if (RT_FAILURE(rc))
183 return rc;
184
185 pVM->hwaccm.s.vmx.pAPIC = (uint8_t *)RTR0MemObjAddress(pVM->hwaccm.s.vmx.pMemObjAPIC);
186 pVM->hwaccm.s.vmx.pAPICPhys = RTR0MemObjGetPagePhysAddr(pVM->hwaccm.s.vmx.pMemObjAPIC, 0);
187 ASMMemZero32(pVM->hwaccm.s.vmx.pAPIC, PAGE_SIZE);
188 }
189 else
190 {
191 pVM->hwaccm.s.vmx.pMemObjAPIC = 0;
192 pVM->hwaccm.s.vmx.pAPIC = 0;
193 pVM->hwaccm.s.vmx.pAPICPhys = 0;
194 }
195
196#ifdef VBOX_WITH_CRASHDUMP_MAGIC
197 {
198 rc = RTR0MemObjAllocCont(&pVM->hwaccm.s.vmx.pMemObjScratch, PAGE_SIZE, true /* executable R0 mapping */);
199 AssertRC(rc);
200 if (RT_FAILURE(rc))
201 return rc;
202
203 pVM->hwaccm.s.vmx.pScratch = (uint8_t *)RTR0MemObjAddress(pVM->hwaccm.s.vmx.pMemObjScratch);
204 pVM->hwaccm.s.vmx.pScratchPhys = RTR0MemObjGetPagePhysAddr(pVM->hwaccm.s.vmx.pMemObjScratch, 0);
205
206 ASMMemZero32(pVM->hwaccm.s.vmx.pScratch, PAGE_SIZE);
207 strcpy((char *)pVM->hwaccm.s.vmx.pScratch, "SCRATCH Magic");
208 *(uint64_t *)(pVM->hwaccm.s.vmx.pScratch + 16) = UINT64_C(0xDEADBEEFDEADBEEF);
209 }
210#endif
211
212 /* Allocate VMCBs for all guest CPUs. */
213 for (VMCPUID i = 0; i < pVM->cCpus; i++)
214 {
215 PVMCPU pVCpu = &pVM->aCpus[i];
216
217 pVCpu->hwaccm.s.vmx.hMemObjVMCS = NIL_RTR0MEMOBJ;
218
219 /* Allocate one page for the VM control structure (VMCS). */
220 rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.hMemObjVMCS, PAGE_SIZE, true /* executable R0 mapping */);
221 AssertRC(rc);
222 if (RT_FAILURE(rc))
223 return rc;
224
225 pVCpu->hwaccm.s.vmx.pvVMCS = RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.hMemObjVMCS);
226 pVCpu->hwaccm.s.vmx.HCPhysVMCS = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.hMemObjVMCS, 0);
227 ASMMemZeroPage(pVCpu->hwaccm.s.vmx.pvVMCS);
228
229 pVCpu->hwaccm.s.vmx.cr0_mask = 0;
230 pVCpu->hwaccm.s.vmx.cr4_mask = 0;
231
232 /* Allocate one page for the virtual APIC page for TPR caching. */
233 rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.hMemObjVAPIC, PAGE_SIZE, true /* executable R0 mapping */);
234 AssertRC(rc);
235 if (RT_FAILURE(rc))
236 return rc;
237
238 pVCpu->hwaccm.s.vmx.pbVAPIC = (uint8_t *)RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.hMemObjVAPIC);
239 pVCpu->hwaccm.s.vmx.HCPhysVAPIC = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.hMemObjVAPIC, 0);
240 ASMMemZeroPage(pVCpu->hwaccm.s.vmx.pbVAPIC);
241
242 /* Allocate the MSR bitmap if this feature is supported. */
243 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
244 {
245 rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap, PAGE_SIZE, true /* executable R0 mapping */);
246 AssertRC(rc);
247 if (RT_FAILURE(rc))
248 return rc;
249
250 pVCpu->hwaccm.s.vmx.pMSRBitmap = (uint8_t *)RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap);
251 pVCpu->hwaccm.s.vmx.pMSRBitmapPhys = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap, 0);
252 memset(pVCpu->hwaccm.s.vmx.pMSRBitmap, 0xff, PAGE_SIZE);
253 }
254
255#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
256 /* Allocate one page for the guest MSR load area (for preloading guest MSRs during the world switch). */
257 rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.pMemObjGuestMSR, PAGE_SIZE, true /* executable R0 mapping */);
258 AssertRC(rc);
259 if (RT_FAILURE(rc))
260 return rc;
261
262 pVCpu->hwaccm.s.vmx.pGuestMSR = (uint8_t *)RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.pMemObjGuestMSR);
263 pVCpu->hwaccm.s.vmx.pGuestMSRPhys = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.pMemObjGuestMSR, 0);
264 memset(pVCpu->hwaccm.s.vmx.pGuestMSR, 0, PAGE_SIZE);
265
266 /* Allocate one page for the host MSR load area (for restoring host MSRs after the world switch back). */
267 rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.pMemObjHostMSR, PAGE_SIZE, true /* executable R0 mapping */);
268 AssertRC(rc);
269 if (RT_FAILURE(rc))
270 return rc;
271
272 pVCpu->hwaccm.s.vmx.pHostMSR = (uint8_t *)RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.pMemObjHostMSR);
273 pVCpu->hwaccm.s.vmx.pHostMSRPhys = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.pMemObjHostMSR, 0);
274 memset(pVCpu->hwaccm.s.vmx.pHostMSR, 0, PAGE_SIZE);
275#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
276
277 /* Current guest paging mode. */
278 pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode = PGMMODE_REAL;
279
280#ifdef LOG_ENABLED
281 SUPR0Printf("VMXR0InitVM %x VMCS=%x (%x)\n", pVM, pVCpu->hwaccm.s.vmx.pvVMCS, (uint32_t)pVCpu->hwaccm.s.vmx.HCPhysVMCS);
282#endif
283 }
284
285 return VINF_SUCCESS;
286}
287
288/**
289 * Does Ring-0 per VM VT-x termination.
290 *
291 * @returns VBox status code.
292 * @param pVM The VM to operate on.
293 */
294VMMR0DECL(int) VMXR0TermVM(PVM pVM)
295{
296 for (VMCPUID i = 0; i < pVM->cCpus; i++)
297 {
298 PVMCPU pVCpu = &pVM->aCpus[i];
299
300 if (pVCpu->hwaccm.s.vmx.hMemObjVMCS != NIL_RTR0MEMOBJ)
301 {
302 RTR0MemObjFree(pVCpu->hwaccm.s.vmx.hMemObjVMCS, false);
303 pVCpu->hwaccm.s.vmx.hMemObjVMCS = NIL_RTR0MEMOBJ;
304 pVCpu->hwaccm.s.vmx.pvVMCS = 0;
305 pVCpu->hwaccm.s.vmx.HCPhysVMCS = 0;
306 }
307 if (pVCpu->hwaccm.s.vmx.hMemObjVAPIC != NIL_RTR0MEMOBJ)
308 {
309 RTR0MemObjFree(pVCpu->hwaccm.s.vmx.hMemObjVAPIC, false);
310 pVCpu->hwaccm.s.vmx.hMemObjVAPIC = NIL_RTR0MEMOBJ;
311 pVCpu->hwaccm.s.vmx.pbVAPIC = 0;
312 pVCpu->hwaccm.s.vmx.HCPhysVAPIC = 0;
313 }
314 if (pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap != NIL_RTR0MEMOBJ)
315 {
316 RTR0MemObjFree(pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap, false);
317 pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap = NIL_RTR0MEMOBJ;
318 pVCpu->hwaccm.s.vmx.pMSRBitmap = 0;
319 pVCpu->hwaccm.s.vmx.pMSRBitmapPhys = 0;
320 }
321#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
322 if (pVCpu->hwaccm.s.vmx.pMemObjHostMSR != NIL_RTR0MEMOBJ)
323 {
324 RTR0MemObjFree(pVCpu->hwaccm.s.vmx.pMemObjHostMSR, false);
325 pVCpu->hwaccm.s.vmx.pMemObjHostMSR = NIL_RTR0MEMOBJ;
326 pVCpu->hwaccm.s.vmx.pHostMSR = 0;
327 pVCpu->hwaccm.s.vmx.pHostMSRPhys = 0;
328 }
329 if (pVCpu->hwaccm.s.vmx.pMemObjGuestMSR != NIL_RTR0MEMOBJ)
330 {
331 RTR0MemObjFree(pVCpu->hwaccm.s.vmx.pMemObjGuestMSR, false);
332 pVCpu->hwaccm.s.vmx.pMemObjGuestMSR = NIL_RTR0MEMOBJ;
333 pVCpu->hwaccm.s.vmx.pGuestMSR = 0;
334 pVCpu->hwaccm.s.vmx.pGuestMSRPhys = 0;
335 }
336#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
337 }
338 if (pVM->hwaccm.s.vmx.pMemObjAPIC != NIL_RTR0MEMOBJ)
339 {
340 RTR0MemObjFree(pVM->hwaccm.s.vmx.pMemObjAPIC, false);
341 pVM->hwaccm.s.vmx.pMemObjAPIC = NIL_RTR0MEMOBJ;
342 pVM->hwaccm.s.vmx.pAPIC = 0;
343 pVM->hwaccm.s.vmx.pAPICPhys = 0;
344 }
345#ifdef VBOX_WITH_CRASHDUMP_MAGIC
346 if (pVM->hwaccm.s.vmx.pMemObjScratch != NIL_RTR0MEMOBJ)
347 {
348 ASMMemZero32(pVM->hwaccm.s.vmx.pScratch, PAGE_SIZE);
349 RTR0MemObjFree(pVM->hwaccm.s.vmx.pMemObjScratch, false);
350 pVM->hwaccm.s.vmx.pMemObjScratch = NIL_RTR0MEMOBJ;
351 pVM->hwaccm.s.vmx.pScratch = 0;
352 pVM->hwaccm.s.vmx.pScratchPhys = 0;
353 }
354#endif
355 return VINF_SUCCESS;
356}
357
358/**
359 * Sets up VT-x for the specified VM
360 *
361 * @returns VBox status code.
362 * @param pVM The VM to operate on.
363 */
364VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
365{
366 int rc = VINF_SUCCESS;
367 uint32_t val;
368
369 AssertReturn(pVM, VERR_INVALID_PARAMETER);
370
371 for (VMCPUID i = 0; i < pVM->cCpus; i++)
372 {
373 PVMCPU pVCpu = &pVM->aCpus[i];
374
375 AssertPtr(pVCpu->hwaccm.s.vmx.pvVMCS);
376
377 /* Set revision dword at the beginning of the VMCS structure. */
378 *(uint32_t *)pVCpu->hwaccm.s.vmx.pvVMCS = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hwaccm.s.vmx.msr.vmx_basic_info);
379
380 /* Clear VM Control Structure. */
381 Log(("HCPhysVMCS = %RHp\n", pVCpu->hwaccm.s.vmx.HCPhysVMCS));
382 rc = VMXClearVMCS(pVCpu->hwaccm.s.vmx.HCPhysVMCS);
383 if (RT_FAILURE(rc))
384 goto vmx_end;
385
386 /* Activate the VM Control Structure. */
387 rc = VMXActivateVMCS(pVCpu->hwaccm.s.vmx.HCPhysVMCS);
388 if (RT_FAILURE(rc))
389 goto vmx_end;
390
391 /* VMX_VMCS_CTRL_PIN_EXEC_CONTROLS
392 * Set required bits to one and zero according to the MSR capabilities.
393 */
394 val = pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.disallowed0;
395 /* External and non-maskable interrupts cause VM-exits. */
396 val |= VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_EXT_INT_EXIT | VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_NMI_EXIT;
397 /* enable the preemption timer. */
398 if (pVM->hwaccm.s.vmx.fUsePreemptTimer)
399 val |= VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_PREEMPT_TIMER;
400 val &= pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.allowed1;
401
402 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PIN_EXEC_CONTROLS, val);
403 AssertRC(rc);
404
405 /* VMX_VMCS_CTRL_PROC_EXEC_CONTROLS
406 * Set required bits to one and zero according to the MSR capabilities.
407 */
408 val = pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.disallowed0;
409 /* Program which event cause VM-exits and which features we want to use. */
410 val = val | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_HLT_EXIT
411 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_TSC_OFFSET
412 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT
413 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_UNCOND_IO_EXIT
414 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDPMC_EXIT
415 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_EXIT
416 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MWAIT_EXIT; /* don't execute mwait or else we'll idle inside the guest (host thinks the cpu load is high) */
417
418 /* Without nested paging we should intercept invlpg and cr3 mov instructions. */
419 if (!pVM->hwaccm.s.fNestedPaging)
420 val |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT
421 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
422 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
423
424 /* Note: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MWAIT_EXIT might cause a vmlaunch failure with an invalid control fields error. (combined with some other exit reasons) */
425 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
426 {
427 /* CR8 reads from the APIC shadow page; writes cause an exit is they lower the TPR below the threshold */
428 val |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW;
429 Assert(pVM->hwaccm.s.vmx.pAPIC);
430 }
431 else
432 /* Exit on CR8 reads & writes in case the TPR shadow feature isn't present. */
433 val |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_STORE_EXIT | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_LOAD_EXIT;
434
435 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
436 {
437 Assert(pVCpu->hwaccm.s.vmx.pMSRBitmapPhys);
438 val |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS;
439 }
440
441 /* We will use the secondary control if it's present. */
442 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
443
444 /* Mask away the bits that the CPU doesn't support */
445 /** @todo make sure they don't conflict with the above requirements. */
446 val &= pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1;
447 pVCpu->hwaccm.s.vmx.proc_ctls = val;
448
449 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, val);
450 AssertRC(rc);
451
452 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
453 {
454 /* VMX_VMCS_CTRL_PROC_EXEC_CONTROLS2
455 * Set required bits to one and zero according to the MSR capabilities.
456 */
457 val = pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0;
458 val |= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT;
459
460#ifdef HWACCM_VTX_WITH_EPT
461 if (pVM->hwaccm.s.fNestedPaging)
462 val |= VMX_VMCS_CTRL_PROC_EXEC2_EPT;
463#endif /* HWACCM_VTX_WITH_EPT */
464#ifdef HWACCM_VTX_WITH_VPID
465 else
466 if (pVM->hwaccm.s.vmx.fVPID)
467 val |= VMX_VMCS_CTRL_PROC_EXEC2_VPID;
468#endif /* HWACCM_VTX_WITH_VPID */
469
470 if (pVM->hwaccm.s.fHasIoApic)
471 val |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC;
472
473 if (pVM->hwaccm.s.vmx.fUnrestrictedGuest)
474 val |= VMX_VMCS_CTRL_PROC_EXEC2_REAL_MODE;
475
476 /* Mask away the bits that the CPU doesn't support */
477 /** @todo make sure they don't conflict with the above requirements. */
478 val &= pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1;
479 pVCpu->hwaccm.s.vmx.proc_ctls2 = val;
480 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS2, val);
481 AssertRC(rc);
482 }
483
484 /* VMX_VMCS_CTRL_CR3_TARGET_COUNT
485 * Set required bits to one and zero according to the MSR capabilities.
486 */
487 rc = VMXWriteVMCS(VMX_VMCS_CTRL_CR3_TARGET_COUNT, 0);
488 AssertRC(rc);
489
490 /* Forward all exception except #NM & #PF to the guest.
491 * We always need to check pagefaults since our shadow page table can be out of sync.
492 * And we always lazily sync the FPU & XMM state.
493 */
494
495 /** @todo Possible optimization:
496 * Keep the FPU and XMM state current in the EM thread. That way there's no need to
497 * lazily sync anything, but the downside is that we can't use the FPU stack or XMM
498 * registers ourselves of course.
499 *
500 * Note: only possible if the current state is actually ours (X86_CR0_TS flag)
501 */
502
503 /* Don't filter page faults; all of them should cause a switch. */
504 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PAGEFAULT_ERROR_MASK, 0);
505 rc |= VMXWriteVMCS(VMX_VMCS_CTRL_PAGEFAULT_ERROR_MATCH, 0);
506 AssertRC(rc);
507
508 /* Init TSC offset to zero. */
509 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_TSC_OFFSET_FULL, 0);
510 AssertRC(rc);
511
512 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_IO_BITMAP_A_FULL, 0);
513 AssertRC(rc);
514
515 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_IO_BITMAP_B_FULL, 0);
516 AssertRC(rc);
517
518 /* Set the MSR bitmap address. */
519 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
520 {
521 Assert(pVCpu->hwaccm.s.vmx.pMSRBitmapPhys);
522
523 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_MSR_BITMAP_FULL, pVCpu->hwaccm.s.vmx.pMSRBitmapPhys);
524 AssertRC(rc);
525
526 /* Allow the guest to directly modify these MSRs; they are restored and saved automatically. */
527 hmR0VmxSetMSRPermission(pVCpu, MSR_IA32_SYSENTER_CS, true, true);
528 hmR0VmxSetMSRPermission(pVCpu, MSR_IA32_SYSENTER_ESP, true, true);
529 hmR0VmxSetMSRPermission(pVCpu, MSR_IA32_SYSENTER_EIP, true, true);
530 hmR0VmxSetMSRPermission(pVCpu, MSR_K8_LSTAR, true, true);
531 hmR0VmxSetMSRPermission(pVCpu, MSR_K6_STAR, true, true);
532 hmR0VmxSetMSRPermission(pVCpu, MSR_K8_SF_MASK, true, true);
533 hmR0VmxSetMSRPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, true, true);
534 hmR0VmxSetMSRPermission(pVCpu, MSR_K8_GS_BASE, true, true);
535 hmR0VmxSetMSRPermission(pVCpu, MSR_K8_FS_BASE, true, true);
536 }
537
538#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
539 /* Set the guest & host MSR load/store physical addresses. */
540 Assert(pVCpu->hwaccm.s.vmx.pGuestMSRPhys);
541 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_VMENTRY_MSR_LOAD_FULL, pVCpu->hwaccm.s.vmx.pGuestMSRPhys);
542 AssertRC(rc);
543 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_VMEXIT_MSR_STORE_FULL, pVCpu->hwaccm.s.vmx.pGuestMSRPhys);
544 AssertRC(rc);
545
546 Assert(pVCpu->hwaccm.s.vmx.pHostMSRPhys);
547 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_VMEXIT_MSR_LOAD_FULL, pVCpu->hwaccm.s.vmx.pHostMSRPhys);
548 AssertRC(rc);
549#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
550
551 rc = VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
552 AssertRC(rc);
553
554 rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_MSR_STORE_COUNT, 0);
555 AssertRC(rc);
556
557 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
558 {
559 Assert(pVM->hwaccm.s.vmx.pMemObjAPIC);
560 /* Optional */
561 rc = VMXWriteVMCS(VMX_VMCS_CTRL_TPR_THRESHOLD, 0);
562 rc |= VMXWriteVMCS64(VMX_VMCS_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hwaccm.s.vmx.HCPhysVAPIC);
563
564 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
565 rc |= VMXWriteVMCS64(VMX_VMCS_CTRL_APIC_ACCESSADDR_FULL, pVM->hwaccm.s.vmx.pAPICPhys);
566
567 AssertRC(rc);
568 }
569
570 /* Set link pointer to -1. Not currently used. */
571 rc = VMXWriteVMCS64(VMX_VMCS_GUEST_LINK_PTR_FULL, 0xFFFFFFFFFFFFFFFFULL);
572 AssertRC(rc);
573
574 /* Clear VM Control Structure. Marking it inactive, clearing implementation specific data and writing back VMCS data to memory. */
575 rc = VMXClearVMCS(pVCpu->hwaccm.s.vmx.HCPhysVMCS);
576 AssertRC(rc);
577
578 /* Configure the VMCS read cache. */
579 PVMCSCACHE pCache = &pVCpu->hwaccm.s.vmx.VMCSCache;
580
581 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_RIP);
582 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_RSP);
583 VMXSetupCachedReadVMCS(pCache, VMX_VMCS_GUEST_RFLAGS);
584 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE);
585 VMXSetupCachedReadVMCS(pCache, VMX_VMCS_CTRL_CR0_READ_SHADOW);
586 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_CR0);
587 VMXSetupCachedReadVMCS(pCache, VMX_VMCS_CTRL_CR4_READ_SHADOW);
588 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_CR4);
589 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_DR7);
590 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_SYSENTER_CS);
591 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_SYSENTER_EIP);
592 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_SYSENTER_ESP);
593 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_GDTR_LIMIT);
594 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_GDTR_BASE);
595 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_IDTR_LIMIT);
596 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_IDTR_BASE);
597
598 VMX_SETUP_SELREG(ES, pCache);
599 VMX_SETUP_SELREG(SS, pCache);
600 VMX_SETUP_SELREG(CS, pCache);
601 VMX_SETUP_SELREG(DS, pCache);
602 VMX_SETUP_SELREG(FS, pCache);
603 VMX_SETUP_SELREG(GS, pCache);
604 VMX_SETUP_SELREG(LDTR, pCache);
605 VMX_SETUP_SELREG(TR, pCache);
606
607 /* Status code VMCS reads. */
608 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_REASON);
609 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_VM_INSTR_ERROR);
610 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INSTR_LENGTH);
611 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INTERRUPTION_ERRCODE);
612 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO);
613 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INSTR_INFO);
614 VMXSetupCachedReadVMCS(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
615 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_IDT_INFO);
616 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_IDT_ERRCODE);
617
618 if (pVM->hwaccm.s.fNestedPaging)
619 {
620 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_CR3);
621 VMXSetupCachedReadVMCS(pCache, VMX_VMCS_EXIT_PHYS_ADDR_FULL);
622 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
623 }
624 else
625 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
626 } /* for each VMCPU */
627
628 /* Choose the right TLB setup function. */
629 if (pVM->hwaccm.s.fNestedPaging)
630 {
631 pVM->hwaccm.s.vmx.pfnSetupTaggedTLB = hmR0VmxSetupTLBEPT;
632
633 /* Default values for flushing. */
634 pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_ALL_CONTEXTS;
635 pVM->hwaccm.s.vmx.enmFlushContext = VMX_FLUSH_ALL_CONTEXTS;
636
637 /* If the capabilities specify we can do more, then make use of it. */
638 if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVEPT_CAPS_INDIV)
639 pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_PAGE;
640 else
641 if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVEPT_CAPS_CONTEXT)
642 pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_SINGLE_CONTEXT;
643
644 if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVEPT_CAPS_CONTEXT)
645 pVM->hwaccm.s.vmx.enmFlushContext = VMX_FLUSH_SINGLE_CONTEXT;
646 }
647#ifdef HWACCM_VTX_WITH_VPID
648 else
649 if (pVM->hwaccm.s.vmx.fVPID)
650 {
651 pVM->hwaccm.s.vmx.pfnSetupTaggedTLB = hmR0VmxSetupTLBVPID;
652
653 /* Default values for flushing. */
654 pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_ALL_CONTEXTS;
655 pVM->hwaccm.s.vmx.enmFlushContext = VMX_FLUSH_ALL_CONTEXTS;
656
657 /* If the capabilities specify we can do more, then make use of it. */
658 if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_INDIV)
659 pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_PAGE;
660 else
661 if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_CONTEXT)
662 pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_SINGLE_CONTEXT;
663
664 if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_CONTEXT)
665 pVM->hwaccm.s.vmx.enmFlushContext = VMX_FLUSH_SINGLE_CONTEXT;
666 }
667#endif /* HWACCM_VTX_WITH_VPID */
668 else
669 pVM->hwaccm.s.vmx.pfnSetupTaggedTLB = hmR0VmxSetupTLBDummy;
670
671vmx_end:
672 hmR0VmxCheckError(pVM, &pVM->aCpus[0], rc);
673 return rc;
674}
675
676/**
677 * Sets the permission bits for the specified MSR
678 *
679 * @param pVCpu The VMCPU to operate on.
680 * @param ulMSR MSR value
681 * @param fRead Reading allowed/disallowed
682 * @param fWrite Writing allowed/disallowed
683 */
684static void hmR0VmxSetMSRPermission(PVMCPU pVCpu, unsigned ulMSR, bool fRead, bool fWrite)
685{
686 unsigned ulBit;
687 uint8_t *pMSRBitmap = (uint8_t *)pVCpu->hwaccm.s.vmx.pMSRBitmap;
688
689 /* Layout:
690 * 0x000 - 0x3ff - Low MSR read bits
691 * 0x400 - 0x7ff - High MSR read bits
692 * 0x800 - 0xbff - Low MSR write bits
693 * 0xc00 - 0xfff - High MSR write bits
694 */
695 if (ulMSR <= 0x00001FFF)
696 {
697 /* Pentium-compatible MSRs */
698 ulBit = ulMSR;
699 }
700 else
701 if ( ulMSR >= 0xC0000000
702 && ulMSR <= 0xC0001FFF)
703 {
704 /* AMD Sixth Generation x86 Processor MSRs */
705 ulBit = (ulMSR - 0xC0000000);
706 pMSRBitmap += 0x400;
707 }
708 else
709 {
710 AssertFailed();
711 return;
712 }
713
714 Assert(ulBit <= 0x1fff);
715 if (fRead)
716 ASMBitClear(pMSRBitmap, ulBit);
717 else
718 ASMBitSet(pMSRBitmap, ulBit);
719
720 if (fWrite)
721 ASMBitClear(pMSRBitmap + 0x800, ulBit);
722 else
723 ASMBitSet(pMSRBitmap + 0x800, ulBit);
724}
725
726
727/**
728 * Injects an event (trap or external interrupt)
729 *
730 * @returns VBox status code. Note that it may return VINF_EM_RESET to
731 * indicate a triple fault when injecting X86_XCPT_DF.
732 *
733 * @param pVM The VM to operate on.
734 * @param pVCpu The VMCPU to operate on.
735 * @param pCtx CPU Context
736 * @param intInfo VMX interrupt info
737 * @param cbInstr Opcode length of faulting instruction
738 * @param errCode Error code (optional)
739 */
740static int hmR0VmxInjectEvent(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, uint32_t intInfo, uint32_t cbInstr, uint32_t errCode)
741{
742 int rc;
743 uint32_t iGate = VMX_EXIT_INTERRUPTION_INFO_VECTOR(intInfo);
744
745#ifdef VBOX_WITH_STATISTICS
746 STAM_COUNTER_INC(&pVCpu->hwaccm.s.paStatInjectedIrqsR0[iGate & MASK_INJECT_IRQ_STAT]);
747#endif
748
749#ifdef VBOX_STRICT
750 if (iGate == 0xE)
751 LogFlow(("hmR0VmxInjectEvent: Injecting interrupt %d at %RGv error code=%08x CR2=%RGv intInfo=%08x\n", iGate, (RTGCPTR)pCtx->rip, errCode, pCtx->cr2, intInfo));
752 else
753 if (iGate < 0x20)
754 LogFlow(("hmR0VmxInjectEvent: Injecting interrupt %d at %RGv error code=%08x\n", iGate, (RTGCPTR)pCtx->rip, errCode));
755 else
756 {
757 LogFlow(("INJ-EI: %x at %RGv\n", iGate, (RTGCPTR)pCtx->rip));
758 Assert( VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW
759 || !VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
760 Assert( VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW
761 || pCtx->eflags.u32 & X86_EFL_IF);
762 }
763#endif
764
765 if ( CPUMIsGuestInRealModeEx(pCtx)
766 && pVM->hwaccm.s.vmx.pRealModeTSS)
767 {
768 RTGCPHYS GCPhysHandler;
769 uint16_t offset, ip;
770 RTSEL sel;
771
772 /* Injecting events doesn't work right with real mode emulation.
773 * (#GP if we try to inject external hardware interrupts)
774 * Inject the interrupt or trap directly instead.
775 *
776 * ASSUMES no access handlers for the bits we read or write below (should be safe).
777 */
778 Log(("Manual interrupt/trap '%x' inject (real mode)\n", iGate));
779
780 /* Check if the interrupt handler is present. */
781 if (iGate * 4 + 3 > pCtx->idtr.cbIdt)
782 {
783 Log(("IDT cbIdt violation\n"));
784 if (iGate != X86_XCPT_DF)
785 {
786 uint32_t intInfo2;
787
788 intInfo2 = (iGate == X86_XCPT_GP) ? (uint32_t)X86_XCPT_DF : iGate;
789 intInfo2 |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
790 intInfo2 |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
791 intInfo2 |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
792
793 return hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo2, 0, 0 /* no error code according to the Intel docs */);
794 }
795 Log(("Triple fault -> reset the VM!\n"));
796 return VINF_EM_RESET;
797 }
798 if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW
799 || iGate == 3 /* Both #BP and #OF point to the instruction after. */
800 || iGate == 4)
801 {
802 ip = pCtx->ip + cbInstr;
803 }
804 else
805 ip = pCtx->ip;
806
807 /* Read the selector:offset pair of the interrupt handler. */
808 GCPhysHandler = (RTGCPHYS)pCtx->idtr.pIdt + iGate * 4;
809 rc = PGMPhysSimpleReadGCPhys(pVM, &offset, GCPhysHandler, sizeof(offset)); AssertRC(rc);
810 rc = PGMPhysSimpleReadGCPhys(pVM, &sel, GCPhysHandler + 2, sizeof(sel)); AssertRC(rc);
811
812 LogFlow(("IDT handler %04X:%04X\n", sel, offset));
813
814 /* Construct the stack frame. */
815 /** @todo should check stack limit. */
816 pCtx->sp -= 2;
817 LogFlow(("ss:sp %04X:%04X eflags=%x\n", pCtx->ss, pCtx->sp, pCtx->eflags.u));
818 rc = PGMPhysSimpleWriteGCPhys(pVM, pCtx->ssHid.u64Base + pCtx->sp, &pCtx->eflags, sizeof(uint16_t)); AssertRC(rc);
819 pCtx->sp -= 2;
820 LogFlow(("ss:sp %04X:%04X cs=%x\n", pCtx->ss, pCtx->sp, pCtx->cs));
821 rc = PGMPhysSimpleWriteGCPhys(pVM, pCtx->ssHid.u64Base + pCtx->sp, &pCtx->cs, sizeof(uint16_t)); AssertRC(rc);
822 pCtx->sp -= 2;
823 LogFlow(("ss:sp %04X:%04X ip=%x\n", pCtx->ss, pCtx->sp, ip));
824 rc = PGMPhysSimpleWriteGCPhys(pVM, pCtx->ssHid.u64Base + pCtx->sp, &ip, sizeof(ip)); AssertRC(rc);
825
826 /* Update the CPU state for executing the handler. */
827 pCtx->rip = offset;
828 pCtx->cs = sel;
829 pCtx->csHid.u64Base = sel << 4;
830 pCtx->eflags.u &= ~(X86_EFL_IF|X86_EFL_TF|X86_EFL_RF|X86_EFL_AC);
831
832 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_SEGMENT_REGS;
833 return VINF_SUCCESS;
834 }
835
836 /* Set event injection state. */
837 rc = VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_IRQ_INFO, intInfo | (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT));
838
839 rc |= VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
840 rc |= VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_EXCEPTION_ERRCODE, errCode);
841
842 AssertRC(rc);
843 return rc;
844}
845
846
847/**
848 * Checks for pending guest interrupts and injects them
849 *
850 * @returns VBox status code.
851 * @param pVM The VM to operate on.
852 * @param pVCpu The VMCPU to operate on.
853 * @param pCtx CPU Context
854 */
855static int hmR0VmxCheckPendingInterrupt(PVM pVM, PVMCPU pVCpu, CPUMCTX *pCtx)
856{
857 int rc;
858
859 /* Dispatch any pending interrupts. (injected before, but a VM exit occurred prematurely) */
860 if (pVCpu->hwaccm.s.Event.fPending)
861 {
862 Log(("CPU%d: Reinjecting event %RX64 %08x at %RGv cr2=%RX64\n", pVCpu->idCpu, pVCpu->hwaccm.s.Event.intInfo, pVCpu->hwaccm.s.Event.errCode, (RTGCPTR)pCtx->rip, pCtx->cr2));
863 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatIntReinject);
864 rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, pVCpu->hwaccm.s.Event.intInfo, 0, pVCpu->hwaccm.s.Event.errCode);
865 AssertRC(rc);
866
867 pVCpu->hwaccm.s.Event.fPending = false;
868 return VINF_SUCCESS;
869 }
870
871 /* If an active trap is already pending, then we must forward it first! */
872 if (!TRPMHasTrap(pVCpu))
873 {
874 if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI))
875 {
876 RTGCUINTPTR intInfo;
877
878 Log(("CPU%d: injecting #NMI\n", pVCpu->idCpu));
879
880 intInfo = X86_XCPT_NMI;
881 intInfo |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
882 intInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
883
884 rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo, 0, 0);
885 AssertRC(rc);
886
887 return VINF_SUCCESS;
888 }
889
890 /* @todo SMI interrupts. */
891
892 /* When external interrupts are pending, we should exit the VM when IF is set. */
893 if (VMCPU_FF_ISPENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC|VMCPU_FF_INTERRUPT_PIC)))
894 {
895 if (!(pCtx->eflags.u32 & X86_EFL_IF))
896 {
897 if (!(pVCpu->hwaccm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_IRQ_WINDOW_EXIT))
898 {
899 LogFlow(("Enable irq window exit!\n"));
900 pVCpu->hwaccm.s.vmx.proc_ctls |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_IRQ_WINDOW_EXIT;
901 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
902 AssertRC(rc);
903 }
904 /* else nothing to do but wait */
905 }
906 else
907 if (!VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
908 {
909 uint8_t u8Interrupt;
910
911 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
912 Log(("CPU%d: Dispatch interrupt: u8Interrupt=%x (%d) rc=%Rrc cs:rip=%04X:%RGv\n", pVCpu->idCpu, u8Interrupt, u8Interrupt, rc, pCtx->cs, (RTGCPTR)pCtx->rip));
913 if (RT_SUCCESS(rc))
914 {
915 rc = TRPMAssertTrap(pVCpu, u8Interrupt, TRPM_HARDWARE_INT);
916 AssertRC(rc);
917 }
918 else
919 {
920 /* Can only happen in rare cases where a pending interrupt is cleared behind our back */
921 Assert(!VMCPU_FF_ISPENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC|VMCPU_FF_INTERRUPT_PIC)));
922 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatSwitchGuestIrq);
923 /* Just continue */
924 }
925 }
926 else
927 Log(("Pending interrupt blocked at %RGv by VM_FF_INHIBIT_INTERRUPTS!!\n", (RTGCPTR)pCtx->rip));
928 }
929 }
930
931#ifdef VBOX_STRICT
932 if (TRPMHasTrap(pVCpu))
933 {
934 uint8_t u8Vector;
935 rc = TRPMQueryTrapAll(pVCpu, &u8Vector, 0, 0, 0);
936 AssertRC(rc);
937 }
938#endif
939
940 if ( (pCtx->eflags.u32 & X86_EFL_IF)
941 && (!VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
942 && TRPMHasTrap(pVCpu)
943 )
944 {
945 uint8_t u8Vector;
946 TRPMEVENT enmType;
947 RTGCUINTPTR intInfo;
948 RTGCUINT errCode;
949
950 /* If a new event is pending, then dispatch it now. */
951 rc = TRPMQueryTrapAll(pVCpu, &u8Vector, &enmType, &errCode, 0);
952 AssertRC(rc);
953 Assert(pCtx->eflags.Bits.u1IF == 1 || enmType == TRPM_TRAP);
954 Assert(enmType != TRPM_SOFTWARE_INT);
955
956 /* Clear the pending trap. */
957 rc = TRPMResetTrap(pVCpu);
958 AssertRC(rc);
959
960 intInfo = u8Vector;
961 intInfo |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
962
963 if (enmType == TRPM_TRAP)
964 {
965 switch (u8Vector) {
966 case 8:
967 case 10:
968 case 11:
969 case 12:
970 case 13:
971 case 14:
972 case 17:
973 /* Valid error codes. */
974 intInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
975 break;
976 default:
977 break;
978 }
979 if (u8Vector == X86_XCPT_BP || u8Vector == X86_XCPT_OF)
980 intInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
981 else
982 intInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
983 }
984 else
985 intInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
986
987 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatIntInject);
988 rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo, 0, errCode);
989 AssertRC(rc);
990 } /* if (interrupts can be dispatched) */
991
992 return VINF_SUCCESS;
993}
994
995/**
996 * Save the host state
997 *
998 * @returns VBox status code.
999 * @param pVM The VM to operate on.
1000 * @param pVCpu The VMCPU to operate on.
1001 */
1002VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
1003{
1004 int rc = VINF_SUCCESS;
1005 NOREF(pVM);
1006
1007 /*
1008 * Host CPU Context
1009 */
1010 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_HOST_CONTEXT)
1011 {
1012 RTIDTR idtr;
1013 RTGDTR gdtr;
1014 RTSEL SelTR;
1015 PCX86DESCHC pDesc;
1016 uintptr_t trBase;
1017 RTSEL cs;
1018 RTSEL ss;
1019 uint64_t cr3;
1020
1021 /* Control registers */
1022 rc = VMXWriteVMCS(VMX_VMCS_HOST_CR0, ASMGetCR0());
1023#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1024 if (VMX_IS_64BIT_HOST_MODE())
1025 {
1026 cr3 = hwaccmR0Get64bitCR3();
1027 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_CR3, cr3);
1028 }
1029 else
1030#endif
1031 {
1032 cr3 = ASMGetCR3();
1033 rc |= VMXWriteVMCS(VMX_VMCS_HOST_CR3, cr3);
1034 }
1035 rc |= VMXWriteVMCS(VMX_VMCS_HOST_CR4, ASMGetCR4());
1036 AssertRC(rc);
1037 Log2(("VMX_VMCS_HOST_CR0 %08x\n", ASMGetCR0()));
1038 Log2(("VMX_VMCS_HOST_CR3 %08RX64\n", cr3));
1039 Log2(("VMX_VMCS_HOST_CR4 %08x\n", ASMGetCR4()));
1040
1041 /* Selector registers. */
1042#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1043 if (VMX_IS_64BIT_HOST_MODE())
1044 {
1045 cs = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
1046 ss = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
1047 }
1048 else
1049 {
1050 /* sysenter loads LDT cs & ss, VMX doesn't like this. Load the GDT ones (safe). */
1051 cs = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
1052 ss = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
1053 }
1054#else
1055 cs = ASMGetCS();
1056 ss = ASMGetSS();
1057#endif
1058 Assert(!(cs & X86_SEL_LDT)); Assert((cs & X86_SEL_RPL) == 0);
1059 Assert(!(ss & X86_SEL_LDT)); Assert((ss & X86_SEL_RPL) == 0);
1060 rc = VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_CS, cs);
1061 /* Note: VMX is (again) very picky about the RPL of the selectors here; we'll restore them manually. */
1062 rc |= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_DS, 0);
1063 rc |= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_ES, 0);
1064#if HC_ARCH_BITS == 32
1065 if (!VMX_IS_64BIT_HOST_MODE())
1066 {
1067 rc |= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_FS, 0);
1068 rc |= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_GS, 0);
1069 }
1070#endif
1071 rc |= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_SS, ss);
1072 SelTR = ASMGetTR();
1073 rc |= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_TR, SelTR);
1074 AssertRC(rc);
1075 Log2(("VMX_VMCS_HOST_FIELD_CS %08x (%08x)\n", cs, ASMGetSS()));
1076 Log2(("VMX_VMCS_HOST_FIELD_DS 00000000 (%08x)\n", ASMGetDS()));
1077 Log2(("VMX_VMCS_HOST_FIELD_ES 00000000 (%08x)\n", ASMGetES()));
1078 Log2(("VMX_VMCS_HOST_FIELD_FS 00000000 (%08x)\n", ASMGetFS()));
1079 Log2(("VMX_VMCS_HOST_FIELD_GS 00000000 (%08x)\n", ASMGetGS()));
1080 Log2(("VMX_VMCS_HOST_FIELD_SS %08x (%08x)\n", ss, ASMGetSS()));
1081 Log2(("VMX_VMCS_HOST_FIELD_TR %08x\n", ASMGetTR()));
1082
1083 /* GDTR & IDTR */
1084#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1085 if (VMX_IS_64BIT_HOST_MODE())
1086 {
1087 X86XDTR64 gdtr64, idtr64;
1088 hwaccmR0Get64bitGDTRandIDTR(&gdtr64, &idtr64);
1089 rc = VMXWriteVMCS64(VMX_VMCS_HOST_GDTR_BASE, gdtr64.uAddr);
1090 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_IDTR_BASE, gdtr64.uAddr);
1091 AssertRC(rc);
1092 Log2(("VMX_VMCS_HOST_GDTR_BASE %RX64\n", gdtr64.uAddr));
1093 Log2(("VMX_VMCS_HOST_IDTR_BASE %RX64\n", idtr64.uAddr));
1094 gdtr.cbGdt = gdtr64.cb;
1095 gdtr.pGdt = (uintptr_t)gdtr64.uAddr;
1096 }
1097 else
1098#endif
1099 {
1100 ASMGetGDTR(&gdtr);
1101 rc = VMXWriteVMCS(VMX_VMCS_HOST_GDTR_BASE, gdtr.pGdt);
1102 ASMGetIDTR(&idtr);
1103 rc |= VMXWriteVMCS(VMX_VMCS_HOST_IDTR_BASE, idtr.pIdt);
1104 AssertRC(rc);
1105 Log2(("VMX_VMCS_HOST_GDTR_BASE %RHv\n", gdtr.pGdt));
1106 Log2(("VMX_VMCS_HOST_IDTR_BASE %RHv\n", idtr.pIdt));
1107 }
1108
1109 /* Save the base address of the TR selector. */
1110 if (SelTR > gdtr.cbGdt)
1111 {
1112 AssertMsgFailed(("Invalid TR selector %x. GDTR.cbGdt=%x\n", SelTR, gdtr.cbGdt));
1113 return VERR_VMX_INVALID_HOST_STATE;
1114 }
1115
1116 pDesc = (PCX86DESCHC)(gdtr.pGdt + (SelTR & X86_SEL_MASK));
1117#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1118 if (VMX_IS_64BIT_HOST_MODE())
1119 {
1120 uint64_t trBase64 = X86DESC64_BASE(*(PX86DESC64)pDesc);
1121 rc = VMXWriteVMCS64(VMX_VMCS_HOST_TR_BASE, trBase64);
1122 Log2(("VMX_VMCS_HOST_TR_BASE %RX64\n", trBase64));
1123 AssertRC(rc);
1124 }
1125 else
1126#endif
1127 {
1128#if HC_ARCH_BITS == 64
1129 trBase = X86DESC64_BASE(*pDesc);
1130#else
1131 trBase = X86DESC_BASE(*pDesc);
1132#endif
1133 rc = VMXWriteVMCS(VMX_VMCS_HOST_TR_BASE, trBase);
1134 AssertRC(rc);
1135 Log2(("VMX_VMCS_HOST_TR_BASE %RHv\n", trBase));
1136 }
1137
1138 /* FS and GS base. */
1139#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1140 if (VMX_IS_64BIT_HOST_MODE())
1141 {
1142 Log2(("MSR_K8_FS_BASE = %RX64\n", ASMRdMsr(MSR_K8_FS_BASE)));
1143 Log2(("MSR_K8_GS_BASE = %RX64\n", ASMRdMsr(MSR_K8_GS_BASE)));
1144 rc = VMXWriteVMCS64(VMX_VMCS_HOST_FS_BASE, ASMRdMsr(MSR_K8_FS_BASE));
1145 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_GS_BASE, ASMRdMsr(MSR_K8_GS_BASE));
1146 }
1147#endif
1148 AssertRC(rc);
1149
1150 /* Sysenter MSRs. */
1151 /** @todo expensive!! */
1152 rc = VMXWriteVMCS(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
1153 Log2(("VMX_VMCS_HOST_SYSENTER_CS %08x\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_CS)));
1154#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1155 if (VMX_IS_64BIT_HOST_MODE())
1156 {
1157 Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_EIP)));
1158 Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_ESP)));
1159 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
1160 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
1161 }
1162 else
1163 {
1164 rc |= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
1165 rc |= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
1166 Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP)));
1167 Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP)));
1168 }
1169#elif HC_ARCH_BITS == 32
1170 rc |= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
1171 rc |= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
1172 Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP)));
1173 Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP)));
1174#else
1175 Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_EIP)));
1176 Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_ESP)));
1177 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
1178 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
1179#endif
1180 AssertRC(rc);
1181
1182#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
1183 /* Store all host MSRs in the VM-Exit load area, so they will be reloaded after the world switch back to the host. */
1184 PVMXMSR pMsr = (PVMXMSR)pVCpu->hwaccm.s.vmx.pHostMSR;
1185 unsigned idxMsr = 0;
1186
1187 /* EFER MSR present? */
1188 if (ASMCpuId_EDX(0x80000001) & (X86_CPUID_AMD_FEATURE_EDX_NX|X86_CPUID_AMD_FEATURE_EDX_LONG_MODE))
1189 {
1190 if (ASMCpuId_EDX(0x80000001) & X86_CPUID_AMD_FEATURE_EDX_SEP)
1191 {
1192 pMsr->u32IndexMSR = MSR_K6_STAR;
1193 pMsr->u32Reserved = 0;
1194 pMsr->u64Value = ASMRdMsr(MSR_K6_STAR); /* legacy syscall eip, cs & ss */
1195 pMsr++; idxMsr++;
1196 }
1197
1198 pMsr->u32IndexMSR = MSR_K6_EFER;
1199 pMsr->u32Reserved = 0;
1200# if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1201 if (CPUMIsGuestInLongMode(pVCpu))
1202 {
1203 /* Must match the efer value in our 64 bits switcher. */
1204 pMsr->u64Value = ASMRdMsr(MSR_K6_EFER) | MSR_K6_EFER_LME | MSR_K6_EFER_SCE | MSR_K6_EFER_NXE;
1205 }
1206 else
1207# endif
1208 pMsr->u64Value = ASMRdMsr(MSR_K6_EFER);
1209 pMsr++; idxMsr++;
1210 }
1211
1212# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1213 if (VMX_IS_64BIT_HOST_MODE())
1214 {
1215 pMsr->u32IndexMSR = MSR_K8_LSTAR;
1216 pMsr->u32Reserved = 0;
1217 pMsr->u64Value = ASMRdMsr(MSR_K8_LSTAR); /* 64 bits mode syscall rip */
1218 pMsr++; idxMsr++;
1219 pMsr->u32IndexMSR = MSR_K8_SF_MASK;
1220 pMsr->u32Reserved = 0;
1221 pMsr->u64Value = ASMRdMsr(MSR_K8_SF_MASK); /* syscall flag mask */
1222 pMsr++; idxMsr++;
1223 pMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
1224 pMsr->u32Reserved = 0;
1225 pMsr->u64Value = ASMRdMsr(MSR_K8_KERNEL_GS_BASE); /* swapgs exchange value */
1226 pMsr++; idxMsr++;
1227 }
1228# endif
1229 rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_MSR_LOAD_COUNT, idxMsr);
1230 AssertRC(rc);
1231#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
1232
1233 pVCpu->hwaccm.s.fContextUseFlags &= ~HWACCM_CHANGED_HOST_CONTEXT;
1234 }
1235 return rc;
1236}
1237
1238/**
1239 * Loads the 4 PDPEs into the guest state when nested paging is used and the
1240 * guest operates in PAE mode.
1241 *
1242 * @returns VINF_SUCCESS or fatal error.
1243 * @param pVCpu The VMCPU to operate on.
1244 * @param pCtx Guest context
1245 */
1246static int hmR0VmxLoadPaePdpes(PVMCPU pVCpu, PCPUMCTX pCtx)
1247{
1248 if (CPUMIsGuestInPAEModeEx(pCtx))
1249 {
1250 X86PDPE aPdpes[4];
1251 int rc = PGMGstGetPaePdpes(pVCpu, &aPdpes[0]);
1252 AssertRCReturn(rc, rc);
1253
1254 rc = VMXWriteVMCS64(VMX_VMCS_GUEST_PDPTR0_FULL, aPdpes[0].u); AssertRCReturn(rc, rc);
1255 rc = VMXWriteVMCS64(VMX_VMCS_GUEST_PDPTR1_FULL, aPdpes[1].u); AssertRCReturn(rc, rc);
1256 rc = VMXWriteVMCS64(VMX_VMCS_GUEST_PDPTR2_FULL, aPdpes[2].u); AssertRCReturn(rc, rc);
1257 rc = VMXWriteVMCS64(VMX_VMCS_GUEST_PDPTR3_FULL, aPdpes[3].u); AssertRCReturn(rc, rc);
1258 }
1259 return VINF_SUCCESS;
1260}
1261
1262/**
1263 * Saves the 4 PDPEs into the guest state when nested paging is used and the
1264 * guest operates in PAE mode.
1265 *
1266 * @returns VINF_SUCCESS or fatal error.
1267 * @param pVCpu The VMCPU to operate on.
1268 * @param pCtx Guest context
1269 *
1270 * @remarks Tell PGM about CR3 changes before calling this helper.
1271 */
1272static int hmR0VmxSavePaePdpes(PVMCPU pVCpu, PCPUMCTX pCtx)
1273{
1274 if (CPUMIsGuestInPAEModeEx(pCtx))
1275 {
1276 int rc;
1277 X86PDPE aPdpes[4];
1278 rc = VMXReadVMCS64(VMX_VMCS_GUEST_PDPTR0_FULL, &aPdpes[0].u); AssertRCReturn(rc, rc);
1279 rc = VMXReadVMCS64(VMX_VMCS_GUEST_PDPTR1_FULL, &aPdpes[1].u); AssertRCReturn(rc, rc);
1280 rc = VMXReadVMCS64(VMX_VMCS_GUEST_PDPTR2_FULL, &aPdpes[2].u); AssertRCReturn(rc, rc);
1281 rc = VMXReadVMCS64(VMX_VMCS_GUEST_PDPTR3_FULL, &aPdpes[3].u); AssertRCReturn(rc, rc);
1282
1283 rc = PGMGstUpdatePaePdpes(pVCpu, &aPdpes[0]);
1284 AssertRCReturn(rc, rc);
1285 }
1286 return VINF_SUCCESS;
1287}
1288
1289
1290/**
1291 * Update the exception bitmap according to the current CPU state
1292 *
1293 * @param pVM The VM to operate on.
1294 * @param pVCpu The VMCPU to operate on.
1295 * @param pCtx Guest context
1296 */
1297static void hmR0VmxUpdateExceptionBitmap(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1298{
1299 uint32_t u32TrapMask;
1300 Assert(pCtx);
1301
1302 u32TrapMask = HWACCM_VMX_TRAP_MASK;
1303#ifndef DEBUG
1304 if (pVM->hwaccm.s.fNestedPaging)
1305 u32TrapMask &= ~RT_BIT(X86_XCPT_PF); /* no longer need to intercept #PF. */
1306#endif
1307
1308 /* Also catch floating point exceptions as we need to report them to the guest in a different way. */
1309 if ( CPUMIsGuestFPUStateActive(pVCpu) == true
1310 && !(pCtx->cr0 & X86_CR0_NE)
1311 && !pVCpu->hwaccm.s.fFPUOldStyleOverride)
1312 {
1313 u32TrapMask |= RT_BIT(X86_XCPT_MF);
1314 pVCpu->hwaccm.s.fFPUOldStyleOverride = true;
1315 }
1316
1317#ifdef VBOX_STRICT
1318 Assert(u32TrapMask & RT_BIT(X86_XCPT_GP));
1319#endif
1320
1321 /* Intercept all exceptions in real mode as none of them can be injected directly (#GP otherwise). */
1322 if ( CPUMIsGuestInRealModeEx(pCtx)
1323 && pVM->hwaccm.s.vmx.pRealModeTSS)
1324 u32TrapMask |= HWACCM_VMX_TRAP_MASK_REALMODE;
1325
1326 int rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXCEPTION_BITMAP, u32TrapMask);
1327 AssertRC(rc);
1328}
1329
1330/**
1331 * Loads a minimal guest state
1332 *
1333 * NOTE: Don't do anything here that can cause a jump back to ring 3!!!!!
1334 *
1335 * @param pVM The VM to operate on.
1336 * @param pVCpu The VMCPU to operate on.
1337 * @param pCtx Guest context
1338 */
1339VMMR0DECL(void) VMXR0LoadMinimalGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1340{
1341 int rc;
1342 X86EFLAGS eflags;
1343
1344 Assert(!(pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_ALL_GUEST));
1345
1346 /* EIP, ESP and EFLAGS */
1347 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_RIP, pCtx->rip);
1348 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_RSP, pCtx->rsp);
1349 AssertRC(rc);
1350
1351 /* Bits 22-31, 15, 5 & 3 must be zero. Bit 1 must be 1. */
1352 eflags = pCtx->eflags;
1353 eflags.u32 &= VMX_EFLAGS_RESERVED_0;
1354 eflags.u32 |= VMX_EFLAGS_RESERVED_1;
1355
1356 /* Real mode emulation using v86 mode. */
1357 if ( CPUMIsGuestInRealModeEx(pCtx)
1358 && pVM->hwaccm.s.vmx.pRealModeTSS)
1359 {
1360 pVCpu->hwaccm.s.vmx.RealMode.eflags = eflags;
1361
1362 eflags.Bits.u1VM = 1;
1363 eflags.Bits.u2IOPL = 0; /* must always be 0 or else certain instructions won't cause faults. */
1364 }
1365 rc = VMXWriteVMCS(VMX_VMCS_GUEST_RFLAGS, eflags.u32);
1366 AssertRC(rc);
1367}
1368
1369/**
1370 * Loads the guest state
1371 *
1372 * NOTE: Don't do anything here that can cause a jump back to ring 3!!!!!
1373 *
1374 * @returns VBox status code.
1375 * @param pVM The VM to operate on.
1376 * @param pVCpu The VMCPU to operate on.
1377 * @param pCtx Guest context
1378 */
1379VMMR0DECL(int) VMXR0LoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1380{
1381 int rc = VINF_SUCCESS;
1382 RTGCUINTPTR val;
1383
1384 /* VMX_VMCS_CTRL_ENTRY_CONTROLS
1385 * Set required bits to one and zero according to the MSR capabilities.
1386 */
1387 val = pVM->hwaccm.s.vmx.msr.vmx_entry.n.disallowed0;
1388 /* Load guest debug controls (dr7 & IA32_DEBUGCTL_MSR) (forced to 1 on the 'first' VT-x capable CPUs; this actually includes the newest Nehalem CPUs) */
1389 val |= VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_DEBUG;
1390 /* 64 bits guest mode? */
1391 if (CPUMIsGuestInLongModeEx(pCtx))
1392 val |= VMX_VMCS_CTRL_ENTRY_CONTROLS_IA64_MODE;
1393 /* else Must be zero when AMD64 is not available. */
1394
1395 /* Mask away the bits that the CPU doesn't support */
1396 val &= pVM->hwaccm.s.vmx.msr.vmx_entry.n.allowed1;
1397 rc = VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_CONTROLS, val);
1398 AssertRC(rc);
1399
1400 /* VMX_VMCS_CTRL_EXIT_CONTROLS
1401 * Set required bits to one and zero according to the MSR capabilities.
1402 */
1403 val = pVM->hwaccm.s.vmx.msr.vmx_exit.n.disallowed0;
1404
1405 /* Save debug controls (dr7 & IA32_DEBUGCTL_MSR) (forced to 1 on the 'first' VT-x capable CPUs; this actually includes the newest Nehalem CPUs) */
1406 val |= VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_DEBUG;
1407
1408#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1409 if (VMX_IS_64BIT_HOST_MODE())
1410 val |= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_AMD64;
1411 /* else: Must be zero when AMD64 is not available. */
1412#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
1413 if (CPUMIsGuestInLongModeEx(pCtx))
1414 val |= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_AMD64; /* our switcher goes to long mode */
1415 else
1416 Assert(!(val & VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_AMD64));
1417#endif
1418 val &= pVM->hwaccm.s.vmx.msr.vmx_exit.n.allowed1;
1419 /* Don't acknowledge external interrupts on VM-exit. */
1420 rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_CONTROLS, val);
1421 AssertRC(rc);
1422
1423 /* Guest CPU context: ES, CS, SS, DS, FS, GS. */
1424 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_SEGMENT_REGS)
1425 {
1426 if (pVM->hwaccm.s.vmx.pRealModeTSS)
1427 {
1428 PGMMODE enmGuestMode = PGMGetGuestMode(pVCpu);
1429 if (pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode != enmGuestMode)
1430 {
1431 /* Correct weird requirements for switching to protected mode. */
1432 if ( pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode == PGMMODE_REAL
1433 && enmGuestMode >= PGMMODE_PROTECTED)
1434 {
1435 /* Flush the recompiler code cache as it's not unlikely
1436 * the guest will rewrite code it will later execute in real
1437 * mode (OpenBSD 4.0 is one such example)
1438 */
1439 REMFlushTBs(pVM);
1440
1441 /* DPL of all hidden selector registers must match the current CPL (0). */
1442 pCtx->csHid.Attr.n.u2Dpl = 0;
1443 pCtx->csHid.Attr.n.u4Type = X86_SEL_TYPE_CODE | X86_SEL_TYPE_RW_ACC;
1444
1445 pCtx->dsHid.Attr.n.u2Dpl = 0;
1446 pCtx->esHid.Attr.n.u2Dpl = 0;
1447 pCtx->fsHid.Attr.n.u2Dpl = 0;
1448 pCtx->gsHid.Attr.n.u2Dpl = 0;
1449 pCtx->ssHid.Attr.n.u2Dpl = 0;
1450
1451 /* The limit must correspond to the 32 bits setting. */
1452 if (!pCtx->csHid.Attr.n.u1DefBig)
1453 pCtx->csHid.u32Limit &= 0xffff;
1454 if (!pCtx->dsHid.Attr.n.u1DefBig)
1455 pCtx->dsHid.u32Limit &= 0xffff;
1456 if (!pCtx->esHid.Attr.n.u1DefBig)
1457 pCtx->esHid.u32Limit &= 0xffff;
1458 if (!pCtx->fsHid.Attr.n.u1DefBig)
1459 pCtx->fsHid.u32Limit &= 0xffff;
1460 if (!pCtx->gsHid.Attr.n.u1DefBig)
1461 pCtx->gsHid.u32Limit &= 0xffff;
1462 if (!pCtx->ssHid.Attr.n.u1DefBig)
1463 pCtx->ssHid.u32Limit &= 0xffff;
1464 }
1465 else
1466 /* Switching from protected mode to real mode. */
1467 if ( pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode >= PGMMODE_PROTECTED
1468 && enmGuestMode == PGMMODE_REAL)
1469 {
1470 /* The limit must also be set to 0xffff. */
1471 pCtx->csHid.u32Limit = 0xffff;
1472 pCtx->dsHid.u32Limit = 0xffff;
1473 pCtx->esHid.u32Limit = 0xffff;
1474 pCtx->fsHid.u32Limit = 0xffff;
1475 pCtx->gsHid.u32Limit = 0xffff;
1476 pCtx->ssHid.u32Limit = 0xffff;
1477
1478 Assert(pCtx->csHid.u64Base <= 0xfffff);
1479 Assert(pCtx->dsHid.u64Base <= 0xfffff);
1480 Assert(pCtx->esHid.u64Base <= 0xfffff);
1481 Assert(pCtx->fsHid.u64Base <= 0xfffff);
1482 Assert(pCtx->gsHid.u64Base <= 0xfffff);
1483 }
1484 pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode = enmGuestMode;
1485 }
1486 else
1487 /* VT-x will fail with a guest invalid state otherwise... (CPU state after a reset) */
1488 if ( CPUMIsGuestInRealModeEx(pCtx)
1489 && pCtx->csHid.u64Base == 0xffff0000)
1490 {
1491 pCtx->csHid.u64Base = 0xf0000;
1492 pCtx->cs = 0xf000;
1493 }
1494 }
1495
1496 VMX_WRITE_SELREG(ES, es);
1497 AssertRC(rc);
1498
1499 VMX_WRITE_SELREG(CS, cs);
1500 AssertRC(rc);
1501
1502 VMX_WRITE_SELREG(SS, ss);
1503 AssertRC(rc);
1504
1505 VMX_WRITE_SELREG(DS, ds);
1506 AssertRC(rc);
1507
1508 VMX_WRITE_SELREG(FS, fs);
1509 AssertRC(rc);
1510
1511 VMX_WRITE_SELREG(GS, gs);
1512 AssertRC(rc);
1513 }
1514
1515 /* Guest CPU context: LDTR. */
1516 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_LDTR)
1517 {
1518 if (pCtx->ldtr == 0)
1519 {
1520 rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_LDTR, 0);
1521 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_LIMIT, 0);
1522 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_LDTR_BASE, 0);
1523 /* Note: vmlaunch will fail with 0 or just 0x02. No idea why. */
1524 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, 0x82 /* present, LDT */);
1525 }
1526 else
1527 {
1528 rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_LDTR, pCtx->ldtr);
1529 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtrHid.u32Limit);
1530 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_LDTR_BASE, pCtx->ldtrHid.u64Base);
1531 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, pCtx->ldtrHid.Attr.u);
1532 }
1533 AssertRC(rc);
1534 }
1535 /* Guest CPU context: TR. */
1536 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_TR)
1537 {
1538 /* Real mode emulation using v86 mode with CR4.VME (interrupt redirection using the int bitmap in the TSS) */
1539 if ( CPUMIsGuestInRealModeEx(pCtx)
1540 && pVM->hwaccm.s.vmx.pRealModeTSS)
1541 {
1542 RTGCPHYS GCPhys;
1543
1544 /* We convert it here every time as pci regions could be reconfigured. */
1545 rc = PDMVMMDevHeapR3ToGCPhys(pVM, pVM->hwaccm.s.vmx.pRealModeTSS, &GCPhys);
1546 AssertRC(rc);
1547
1548 rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_TR, 0);
1549 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_TR_LIMIT, HWACCM_VTX_TSS_SIZE);
1550 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_TR_BASE, GCPhys /* phys = virt in this mode */);
1551
1552 X86DESCATTR attr;
1553
1554 attr.u = 0;
1555 attr.n.u1Present = 1;
1556 attr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
1557 val = attr.u;
1558 }
1559 else
1560 {
1561 rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_TR, pCtx->tr);
1562 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_TR_LIMIT, pCtx->trHid.u32Limit);
1563 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_TR_BASE, pCtx->trHid.u64Base);
1564
1565 val = pCtx->trHid.Attr.u;
1566
1567 /* The TSS selector must be busy (REM bugs? see defect #XXXX). */
1568 if (!(val & X86_SEL_TYPE_SYS_TSS_BUSY_MASK))
1569 {
1570 if (val & 0xf)
1571 val |= X86_SEL_TYPE_SYS_TSS_BUSY_MASK;
1572 else
1573 /* Default if no TR selector has been set (otherwise vmlaunch will fail!) */
1574 val = (val & ~0xF) | X86_SEL_TYPE_SYS_386_TSS_BUSY;
1575 }
1576 AssertMsg((val & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY || (val & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("%#x\n", val));
1577 }
1578 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, val);
1579 AssertRC(rc);
1580 }
1581 /* Guest CPU context: GDTR. */
1582 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_GDTR)
1583 {
1584 rc = VMXWriteVMCS(VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt);
1585 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_GDTR_BASE, pCtx->gdtr.pGdt);
1586 AssertRC(rc);
1587 }
1588 /* Guest CPU context: IDTR. */
1589 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_IDTR)
1590 {
1591 rc = VMXWriteVMCS(VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt);
1592 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_IDTR_BASE, pCtx->idtr.pIdt);
1593 AssertRC(rc);
1594 }
1595
1596 /*
1597 * Sysenter MSRs
1598 */
1599 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_MSR)
1600 {
1601 rc = VMXWriteVMCS(VMX_VMCS32_GUEST_SYSENTER_CS, pCtx->SysEnter.cs);
1602 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_SYSENTER_EIP, pCtx->SysEnter.eip);
1603 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_SYSENTER_ESP, pCtx->SysEnter.esp);
1604 AssertRC(rc);
1605 }
1606
1607 /* Control registers */
1608 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_CR0)
1609 {
1610 val = pCtx->cr0;
1611 rc = VMXWriteVMCS(VMX_VMCS_CTRL_CR0_READ_SHADOW, val);
1612 Log2(("Guest CR0-shadow %08x\n", val));
1613 if (CPUMIsGuestFPUStateActive(pVCpu) == false)
1614 {
1615 /* Always use #NM exceptions to load the FPU/XMM state on demand. */
1616 val |= X86_CR0_TS | X86_CR0_ET | X86_CR0_NE | X86_CR0_MP;
1617 }
1618 else
1619 {
1620 /** @todo check if we support the old style mess correctly. */
1621 if (!(val & X86_CR0_NE))
1622 Log(("Forcing X86_CR0_NE!!!\n"));
1623
1624 val |= X86_CR0_NE; /* always turn on the native mechanism to report FPU errors (old style uses interrupts) */
1625 }
1626 /* Note: protected mode & paging are always enabled; we use them for emulating real and protected mode without paging too. */
1627 if (!pVM->hwaccm.s.vmx.fUnrestrictedGuest)
1628 val |= X86_CR0_PE | X86_CR0_PG;
1629
1630 if (pVM->hwaccm.s.fNestedPaging)
1631 {
1632 if (CPUMIsGuestInPagedProtectedModeEx(pCtx))
1633 {
1634 /* Disable cr3 read/write monitoring as we don't need it for EPT. */
1635 pVCpu->hwaccm.s.vmx.proc_ctls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
1636 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT);
1637 }
1638 else
1639 {
1640 /* Reenable cr3 read/write monitoring as our identity mapped page table is active. */
1641 pVCpu->hwaccm.s.vmx.proc_ctls |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
1642 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
1643 }
1644 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1645 AssertRC(rc);
1646 }
1647 else
1648 {
1649 /* Note: We must also set this as we rely on protecting various pages for which supervisor writes must be caught. */
1650 val |= X86_CR0_WP;
1651 }
1652
1653 /* Always enable caching. */
1654 val &= ~(X86_CR0_CD|X86_CR0_NW);
1655
1656 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_CR0, val);
1657 Log2(("Guest CR0 %08x\n", val));
1658 /* CR0 flags owned by the host; if the guests attempts to change them, then
1659 * the VM will exit.
1660 */
1661 val = X86_CR0_PE /* Must monitor this bit (assumptions are made for real mode emulation) */
1662 | X86_CR0_WP /* Must monitor this bit (it must always be enabled). */
1663 | X86_CR0_PG /* Must monitor this bit (assumptions are made for real mode & protected mode without paging emulation) */
1664 | X86_CR0_CD /* Bit not restored during VM-exit! */
1665 | X86_CR0_NW /* Bit not restored during VM-exit! */
1666 | X86_CR0_NE;
1667
1668 /* When the guest's FPU state is active, then we no longer care about
1669 * the FPU related bits.
1670 */
1671 if (CPUMIsGuestFPUStateActive(pVCpu) == false)
1672 val |= X86_CR0_TS | X86_CR0_ET | X86_CR0_MP;
1673
1674 pVCpu->hwaccm.s.vmx.cr0_mask = val;
1675
1676 rc |= VMXWriteVMCS(VMX_VMCS_CTRL_CR0_MASK, val);
1677 Log2(("Guest CR0-mask %08x\n", val));
1678 AssertRC(rc);
1679 }
1680 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_CR4)
1681 {
1682 /* CR4 */
1683 rc = VMXWriteVMCS(VMX_VMCS_CTRL_CR4_READ_SHADOW, pCtx->cr4);
1684 Log2(("Guest CR4-shadow %08x\n", pCtx->cr4));
1685 /* Set the required bits in cr4 too (currently X86_CR4_VMXE). */
1686 val = pCtx->cr4 | (uint32_t)pVM->hwaccm.s.vmx.msr.vmx_cr4_fixed0;
1687
1688 if (!pVM->hwaccm.s.fNestedPaging)
1689 {
1690 switch(pVCpu->hwaccm.s.enmShadowMode)
1691 {
1692 case PGMMODE_REAL: /* Real mode -> emulated using v86 mode */
1693 case PGMMODE_PROTECTED: /* Protected mode, no paging -> emulated using identity mapping. */
1694 case PGMMODE_32_BIT: /* 32-bit paging. */
1695 val &= ~X86_CR4_PAE;
1696 break;
1697
1698 case PGMMODE_PAE: /* PAE paging. */
1699 case PGMMODE_PAE_NX: /* PAE paging with NX enabled. */
1700 /** Must use PAE paging as we could use physical memory > 4 GB */
1701 val |= X86_CR4_PAE;
1702 break;
1703
1704 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
1705 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
1706#ifdef VBOX_ENABLE_64_BITS_GUESTS
1707 break;
1708#else
1709 AssertFailed();
1710 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1711#endif
1712 default: /* shut up gcc */
1713 AssertFailed();
1714 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1715 }
1716 }
1717 else
1718 if ( !CPUMIsGuestInPagedProtectedModeEx(pCtx)
1719 && !pVM->hwaccm.s.vmx.fUnrestrictedGuest)
1720 {
1721 /* We use 4 MB pages in our identity mapping page table for real and protected mode without paging. */
1722 val |= X86_CR4_PSE;
1723 /* Our identity mapping is a 32 bits page directory. */
1724 val &= ~X86_CR4_PAE;
1725 }
1726
1727 /* Turn off VME if we're in emulated real mode. */
1728 if ( CPUMIsGuestInRealModeEx(pCtx)
1729 && pVM->hwaccm.s.vmx.pRealModeTSS)
1730 val &= ~X86_CR4_VME;
1731
1732 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_CR4, val);
1733 Log2(("Guest CR4 %08x\n", val));
1734 /* CR4 flags owned by the host; if the guests attempts to change them, then
1735 * the VM will exit.
1736 */
1737 val = 0
1738 | X86_CR4_VME
1739 | X86_CR4_PAE
1740 | X86_CR4_PGE
1741 | X86_CR4_PSE
1742 | X86_CR4_VMXE;
1743 pVCpu->hwaccm.s.vmx.cr4_mask = val;
1744
1745 rc |= VMXWriteVMCS(VMX_VMCS_CTRL_CR4_MASK, val);
1746 Log2(("Guest CR4-mask %08x\n", val));
1747 AssertRC(rc);
1748 }
1749
1750 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_CR3)
1751 {
1752 if (pVM->hwaccm.s.fNestedPaging)
1753 {
1754 Assert(PGMGetHyperCR3(pVCpu));
1755 pVCpu->hwaccm.s.vmx.GCPhysEPTP = PGMGetHyperCR3(pVCpu);
1756
1757 Assert(!(pVCpu->hwaccm.s.vmx.GCPhysEPTP & 0xfff));
1758 /** @todo Check the IA32_VMX_EPT_VPID_CAP MSR for other supported memory types. */
1759 pVCpu->hwaccm.s.vmx.GCPhysEPTP |= VMX_EPT_MEMTYPE_WB
1760 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
1761
1762 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_EPTP_FULL, pVCpu->hwaccm.s.vmx.GCPhysEPTP);
1763 AssertRC(rc);
1764
1765 if ( !CPUMIsGuestInPagedProtectedModeEx(pCtx)
1766 && !pVM->hwaccm.s.vmx.fUnrestrictedGuest)
1767 {
1768 RTGCPHYS GCPhys;
1769
1770 /* We convert it here every time as pci regions could be reconfigured. */
1771 rc = PDMVMMDevHeapR3ToGCPhys(pVM, pVM->hwaccm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
1772 AssertMsgRC(rc, ("pNonPagingModeEPTPageTable = %RGv\n", pVM->hwaccm.s.vmx.pNonPagingModeEPTPageTable));
1773
1774 /* We use our identity mapping page table here as we need to map guest virtual to guest physical addresses; EPT will
1775 * take care of the translation to host physical addresses.
1776 */
1777 val = GCPhys;
1778 }
1779 else
1780 {
1781 /* Save the real guest CR3 in VMX_VMCS_GUEST_CR3 */
1782 val = pCtx->cr3;
1783 rc = hmR0VmxLoadPaePdpes(pVCpu, pCtx);
1784 AssertRCReturn(rc, rc);
1785 }
1786 }
1787 else
1788 {
1789 val = PGMGetHyperCR3(pVCpu);
1790 Assert(val || VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL));
1791 }
1792
1793 /* Save our shadow CR3 register. */
1794 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_CR3, val);
1795 AssertRC(rc);
1796 }
1797
1798 /* Debug registers. */
1799 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_DEBUG)
1800 {
1801 pCtx->dr[6] |= X86_DR6_INIT_VAL; /* set all reserved bits to 1. */
1802 pCtx->dr[6] &= ~RT_BIT(12); /* must be zero. */
1803
1804 pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
1805 pCtx->dr[7] &= ~(RT_BIT(11) | RT_BIT(12) | RT_BIT(14) | RT_BIT(15)); /* must be zero */
1806 pCtx->dr[7] |= 0x400; /* must be one */
1807
1808 /* Resync DR7 */
1809 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_DR7, pCtx->dr[7]);
1810 AssertRC(rc);
1811
1812#ifdef DEBUG
1813 /* Sync the hypervisor debug state now if any breakpoint is armed. */
1814 if ( CPUMGetHyperDR7(pVCpu) & (X86_DR7_ENABLED_MASK|X86_DR7_GD)
1815 && !CPUMIsHyperDebugStateActive(pVCpu)
1816 && !DBGFIsStepping(pVCpu))
1817 {
1818 /* Save the host and load the hypervisor debug state. */
1819 rc = CPUMR0LoadHyperDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
1820 AssertRC(rc);
1821
1822 /* DRx intercepts remain enabled. */
1823
1824 /* Override dr7 with the hypervisor value. */
1825 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_DR7, CPUMGetHyperDR7(pVCpu));
1826 AssertRC(rc);
1827 }
1828 else
1829#endif
1830 /* Sync the debug state now if any breakpoint is armed. */
1831 if ( (pCtx->dr[7] & (X86_DR7_ENABLED_MASK|X86_DR7_GD))
1832 && !CPUMIsGuestDebugStateActive(pVCpu)
1833 && !DBGFIsStepping(pVCpu))
1834 {
1835 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatDRxArmed);
1836
1837 /* Disable drx move intercepts. */
1838 pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
1839 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1840 AssertRC(rc);
1841
1842 /* Save the host and load the guest debug state. */
1843 rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
1844 AssertRC(rc);
1845 }
1846
1847 /* IA32_DEBUGCTL MSR. */
1848 rc = VMXWriteVMCS64(VMX_VMCS_GUEST_DEBUGCTL_FULL, 0);
1849 AssertRC(rc);
1850
1851 /** @todo do we really ever need this? */
1852 rc |= VMXWriteVMCS(VMX_VMCS_GUEST_DEBUG_EXCEPTIONS, 0);
1853 AssertRC(rc);
1854 }
1855
1856 /* 64 bits guest mode? */
1857 if (CPUMIsGuestInLongModeEx(pCtx))
1858 {
1859#if !defined(VBOX_ENABLE_64_BITS_GUESTS)
1860 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1861#elif HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1862 pVCpu->hwaccm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
1863#else
1864# ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1865 if (!pVM->hwaccm.s.fAllow64BitGuests)
1866 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1867# endif
1868 pVCpu->hwaccm.s.vmx.pfnStartVM = VMXR0StartVM64;
1869#endif
1870 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_MSR)
1871 {
1872 /* Update these as wrmsr might have changed them. */
1873 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_FS_BASE, pCtx->fsHid.u64Base);
1874 AssertRC(rc);
1875 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_GS_BASE, pCtx->gsHid.u64Base);
1876 AssertRC(rc);
1877 }
1878 }
1879 else
1880 {
1881 pVCpu->hwaccm.s.vmx.pfnStartVM = VMXR0StartVM32;
1882 }
1883
1884 hmR0VmxUpdateExceptionBitmap(pVM, pVCpu, pCtx);
1885
1886#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
1887 /* Store all guest MSRs in the VM-Entry load area, so they will be loaded during the world switch. */
1888 PVMXMSR pMsr = (PVMXMSR)pVCpu->hwaccm.s.vmx.pGuestMSR;
1889 unsigned idxMsr = 0;
1890
1891 uint32_t ulEdx;
1892 uint32_t ulTemp;
1893 CPUMGetGuestCpuId(pVCpu, 0x80000001, &ulTemp, &ulTemp, &ulTemp, &ulEdx);
1894 /* EFER MSR present? */
1895 if (ulEdx & (X86_CPUID_AMD_FEATURE_EDX_NX|X86_CPUID_AMD_FEATURE_EDX_LONG_MODE))
1896 {
1897 pMsr->u32IndexMSR = MSR_K6_EFER;
1898 pMsr->u32Reserved = 0;
1899 pMsr->u64Value = pCtx->msrEFER;
1900 /* VT-x will complain if only MSR_K6_EFER_LME is set. */
1901 if (!CPUMIsGuestInLongModeEx(pCtx))
1902 pMsr->u64Value &= ~(MSR_K6_EFER_LMA|MSR_K6_EFER_LME);
1903 pMsr++; idxMsr++;
1904
1905 if (ulEdx & X86_CPUID_AMD_FEATURE_EDX_LONG_MODE)
1906 {
1907 pMsr->u32IndexMSR = MSR_K8_LSTAR;
1908 pMsr->u32Reserved = 0;
1909 pMsr->u64Value = pCtx->msrLSTAR; /* 64 bits mode syscall rip */
1910 pMsr++; idxMsr++;
1911 pMsr->u32IndexMSR = MSR_K6_STAR;
1912 pMsr->u32Reserved = 0;
1913 pMsr->u64Value = pCtx->msrSTAR; /* legacy syscall eip, cs & ss */
1914 pMsr++; idxMsr++;
1915 pMsr->u32IndexMSR = MSR_K8_SF_MASK;
1916 pMsr->u32Reserved = 0;
1917 pMsr->u64Value = pCtx->msrSFMASK; /* syscall flag mask */
1918 pMsr++; idxMsr++;
1919 pMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
1920 pMsr->u32Reserved = 0;
1921 pMsr->u64Value = pCtx->msrKERNELGSBASE; /* swapgs exchange value */
1922 pMsr++; idxMsr++;
1923 }
1924 }
1925 pVCpu->hwaccm.s.vmx.cCachedMSRs = idxMsr;
1926
1927 rc = VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_MSR_LOAD_COUNT, idxMsr);
1928 AssertRC(rc);
1929
1930 rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_MSR_STORE_COUNT, idxMsr);
1931 AssertRC(rc);
1932#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
1933
1934 bool fOffsettedTsc;
1935 if (pVM->hwaccm.s.vmx.fUsePreemptTimer)
1936 {
1937 uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &pVCpu->hwaccm.s.vmx.u64TSCOffset);
1938
1939 /* Make sure the returned values have sane upper and lower boundaries. */
1940 uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
1941
1942 cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64 of a second */
1943 cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
1944
1945 cTicksToDeadline >>= pVM->hwaccm.s.vmx.cPreemptTimerShift;
1946 uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
1947 rc = VMXWriteVMCS(VMX_VMCS32_GUEST_PREEMPTION_TIMER_VALUE, cPreemptionTickCount);
1948 AssertRC(rc);
1949 }
1950 else
1951 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hwaccm.s.vmx.u64TSCOffset);
1952 if (fOffsettedTsc)
1953 {
1954 uint64_t u64CurTSC = ASMReadTSC();
1955 if (u64CurTSC + pVCpu->hwaccm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
1956 {
1957 /* Note: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT takes precedence over TSC_OFFSET */
1958 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_TSC_OFFSET_FULL, pVCpu->hwaccm.s.vmx.u64TSCOffset);
1959 AssertRC(rc);
1960
1961 pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
1962 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1963 AssertRC(rc);
1964 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTSCOffset);
1965 }
1966 else
1967 {
1968 /* Fall back to rdtsc emulation as we would otherwise pass decreasing tsc values to the guest. */
1969 LogFlow(("TSC %RX64 offset %RX64 time=%RX64 last=%RX64 (diff=%RX64, virt_tsc=%RX64)\n", u64CurTSC, pVCpu->hwaccm.s.vmx.u64TSCOffset, u64CurTSC + pVCpu->hwaccm.s.vmx.u64TSCOffset, TMCpuTickGetLastSeen(pVCpu), TMCpuTickGetLastSeen(pVCpu) - u64CurTSC - pVCpu->hwaccm.s.vmx.u64TSCOffset, TMCpuTickGet(pVCpu)));
1970 pVCpu->hwaccm.s.vmx.proc_ctls |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
1971 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1972 AssertRC(rc);
1973 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTSCInterceptOverFlow);
1974 }
1975 }
1976 else
1977 {
1978 pVCpu->hwaccm.s.vmx.proc_ctls |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
1979 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1980 AssertRC(rc);
1981 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTSCIntercept);
1982 }
1983
1984 /* Done with the major changes */
1985 pVCpu->hwaccm.s.fContextUseFlags &= ~HWACCM_CHANGED_ALL_GUEST;
1986
1987 /* Minimal guest state update (esp, eip, eflags mostly) */
1988 VMXR0LoadMinimalGuestState(pVM, pVCpu, pCtx);
1989 return rc;
1990}
1991
1992/**
1993 * Syncs back the guest state
1994 *
1995 * @returns VBox status code.
1996 * @param pVM The VM to operate on.
1997 * @param pVCpu The VMCPU to operate on.
1998 * @param pCtx Guest context
1999 */
2000DECLINLINE(int) VMXR0SaveGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2001{
2002 RTGCUINTREG val, valShadow;
2003 RTGCUINTPTR uInterruptState;
2004 int rc;
2005
2006 /* Let's first sync back eip, esp, and eflags. */
2007 rc = VMXReadCachedVMCS(VMX_VMCS64_GUEST_RIP, &val);
2008 AssertRC(rc);
2009 pCtx->rip = val;
2010 rc = VMXReadCachedVMCS(VMX_VMCS64_GUEST_RSP, &val);
2011 AssertRC(rc);
2012 pCtx->rsp = val;
2013 rc = VMXReadCachedVMCS(VMX_VMCS_GUEST_RFLAGS, &val);
2014 AssertRC(rc);
2015 pCtx->eflags.u32 = val;
2016
2017 /* Take care of instruction fusing (sti, mov ss) */
2018 rc |= VMXReadCachedVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &val);
2019 uInterruptState = val;
2020 if (uInterruptState != 0)
2021 {
2022 Assert(uInterruptState <= 2); /* only sti & mov ss */
2023 Log(("uInterruptState %x eip=%RGv\n", (uint32_t)uInterruptState, pCtx->rip));
2024 EMSetInhibitInterruptsPC(pVCpu, pCtx->rip);
2025 }
2026 else
2027 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2028
2029 /* Control registers. */
2030 VMXReadCachedVMCS(VMX_VMCS_CTRL_CR0_READ_SHADOW, &valShadow);
2031 VMXReadCachedVMCS(VMX_VMCS64_GUEST_CR0, &val);
2032 val = (valShadow & pVCpu->hwaccm.s.vmx.cr0_mask) | (val & ~pVCpu->hwaccm.s.vmx.cr0_mask);
2033 CPUMSetGuestCR0(pVCpu, val);
2034
2035 VMXReadCachedVMCS(VMX_VMCS_CTRL_CR4_READ_SHADOW, &valShadow);
2036 VMXReadCachedVMCS(VMX_VMCS64_GUEST_CR4, &val);
2037 val = (valShadow & pVCpu->hwaccm.s.vmx.cr4_mask) | (val & ~pVCpu->hwaccm.s.vmx.cr4_mask);
2038 CPUMSetGuestCR4(pVCpu, val);
2039
2040 /* Note: no reason to sync back the CRx registers. They can't be changed by the guest. */
2041 /* Note: only in the nested paging case can CR3 & CR4 be changed by the guest. */
2042 if ( pVM->hwaccm.s.fNestedPaging
2043 && CPUMIsGuestInPagedProtectedModeEx(pCtx))
2044 {
2045 PVMCSCACHE pCache = &pVCpu->hwaccm.s.vmx.VMCSCache;
2046
2047 /* Can be updated behind our back in the nested paging case. */
2048 CPUMSetGuestCR2(pVCpu, pCache->cr2);
2049
2050 VMXReadCachedVMCS(VMX_VMCS64_GUEST_CR3, &val);
2051
2052 if (val != pCtx->cr3)
2053 {
2054 CPUMSetGuestCR3(pVCpu, val);
2055 PGMUpdateCR3(pVCpu, val);
2056 }
2057 rc = hmR0VmxSavePaePdpes(pVCpu, pCtx);
2058 AssertRCReturn(rc, rc);
2059 }
2060
2061 /* Sync back DR7 here. */
2062 VMXReadCachedVMCS(VMX_VMCS64_GUEST_DR7, &val);
2063 pCtx->dr[7] = val;
2064
2065 /* Guest CPU context: ES, CS, SS, DS, FS, GS. */
2066 VMX_READ_SELREG(ES, es);
2067 VMX_READ_SELREG(SS, ss);
2068 VMX_READ_SELREG(CS, cs);
2069 VMX_READ_SELREG(DS, ds);
2070 VMX_READ_SELREG(FS, fs);
2071 VMX_READ_SELREG(GS, gs);
2072
2073 /*
2074 * System MSRs
2075 */
2076 VMXReadCachedVMCS(VMX_VMCS32_GUEST_SYSENTER_CS, &val);
2077 pCtx->SysEnter.cs = val;
2078 VMXReadCachedVMCS(VMX_VMCS64_GUEST_SYSENTER_EIP, &val);
2079 pCtx->SysEnter.eip = val;
2080 VMXReadCachedVMCS(VMX_VMCS64_GUEST_SYSENTER_ESP, &val);
2081 pCtx->SysEnter.esp = val;
2082
2083 /* Misc. registers; must sync everything otherwise we can get out of sync when jumping to ring 3. */
2084 VMX_READ_SELREG(LDTR, ldtr);
2085
2086 VMXReadCachedVMCS(VMX_VMCS32_GUEST_GDTR_LIMIT, &val);
2087 pCtx->gdtr.cbGdt = val;
2088 VMXReadCachedVMCS(VMX_VMCS64_GUEST_GDTR_BASE, &val);
2089 pCtx->gdtr.pGdt = val;
2090
2091 VMXReadCachedVMCS(VMX_VMCS32_GUEST_IDTR_LIMIT, &val);
2092 pCtx->idtr.cbIdt = val;
2093 VMXReadCachedVMCS(VMX_VMCS64_GUEST_IDTR_BASE, &val);
2094 pCtx->idtr.pIdt = val;
2095
2096 /* Real mode emulation using v86 mode. */
2097 if ( CPUMIsGuestInRealModeEx(pCtx)
2098 && pVM->hwaccm.s.vmx.pRealModeTSS)
2099 {
2100 /* Hide our emulation flags */
2101 pCtx->eflags.Bits.u1VM = 0;
2102
2103 /* Restore original IOPL setting as we always use 0. */
2104 pCtx->eflags.Bits.u2IOPL = pVCpu->hwaccm.s.vmx.RealMode.eflags.Bits.u2IOPL;
2105
2106 /* Force a TR resync every time in case we switch modes. */
2107 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_TR;
2108 }
2109 else
2110 {
2111 /* In real mode we have a fake TSS, so only sync it back when it's supposed to be valid. */
2112 VMX_READ_SELREG(TR, tr);
2113 }
2114
2115#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
2116 /* Save the possibly changed MSRs that we automatically restore and save during a world switch. */
2117 for (unsigned i = 0; i < pVCpu->hwaccm.s.vmx.cCachedMSRs; i++)
2118 {
2119 PVMXMSR pMsr = (PVMXMSR)pVCpu->hwaccm.s.vmx.pGuestMSR;
2120 pMsr += i;
2121
2122 switch (pMsr->u32IndexMSR)
2123 {
2124 case MSR_K8_LSTAR:
2125 pCtx->msrLSTAR = pMsr->u64Value;
2126 break;
2127 case MSR_K6_STAR:
2128 pCtx->msrSTAR = pMsr->u64Value;
2129 break;
2130 case MSR_K8_SF_MASK:
2131 pCtx->msrSFMASK = pMsr->u64Value;
2132 break;
2133 case MSR_K8_KERNEL_GS_BASE:
2134 pCtx->msrKERNELGSBASE = pMsr->u64Value;
2135 break;
2136 case MSR_K6_EFER:
2137 /* EFER can't be changed without causing a VM-exit. */
2138// Assert(pCtx->msrEFER == pMsr->u64Value);
2139 break;
2140 default:
2141 AssertFailed();
2142 return VERR_HM_UNEXPECTED_LD_ST_MSR;
2143 }
2144 }
2145#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
2146 return VINF_SUCCESS;
2147}
2148
2149/**
2150 * Dummy placeholder
2151 *
2152 * @param pVM The VM to operate on.
2153 * @param pVCpu The VMCPU to operate on.
2154 */
2155static void hmR0VmxSetupTLBDummy(PVM pVM, PVMCPU pVCpu)
2156{
2157 NOREF(pVM);
2158 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
2159 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2160 pVCpu->hwaccm.s.TlbShootdown.cPages = 0;
2161 return;
2162}
2163
2164/**
2165 * Setup the tagged TLB for EPT
2166 *
2167 * @returns VBox status code.
2168 * @param pVM The VM to operate on.
2169 * @param pVCpu The VMCPU to operate on.
2170 */
2171static void hmR0VmxSetupTLBEPT(PVM pVM, PVMCPU pVCpu)
2172{
2173 PHMGLOBLCPUINFO pCpu;
2174
2175 Assert(pVM->hwaccm.s.fNestedPaging);
2176 Assert(!pVM->hwaccm.s.vmx.fVPID);
2177
2178 /* Deal with tagged TLBs if VPID or EPT is supported. */
2179 pCpu = HWACCMR0GetCurrentCpu();
2180 /* Force a TLB flush for the first world switch if the current cpu differs from the one we ran on last. */
2181 /* Note that this can happen both for start and resume due to long jumps back to ring 3. */
2182 if ( pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu
2183 /* if the tlb flush count has changed, another VM has flushed the TLB of this cpu, so we can't use our current ASID anymore. */
2184 || pVCpu->hwaccm.s.cTLBFlushes != pCpu->cTLBFlushes)
2185 {
2186 /* Force a TLB flush on VM entry. */
2187 pVCpu->hwaccm.s.fForceTLBFlush = true;
2188 }
2189 /* Disabled because this has triggered every time I have suspended my
2190 * laptop with a VM running for the past three months or more. */
2191 // else
2192 // Assert(!pCpu->fFlushTLB);
2193
2194 /* Check for tlb shootdown flushes. */
2195 if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2196 pVCpu->hwaccm.s.fForceTLBFlush = true;
2197
2198 pVCpu->hwaccm.s.idLastCpu = pCpu->idCpu;
2199 pCpu->fFlushTLB = false;
2200
2201 if (pVCpu->hwaccm.s.fForceTLBFlush)
2202 {
2203 hmR0VmxFlushEPT(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushContext, 0);
2204 }
2205 else
2206 if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2207 {
2208 /* Deal with pending TLB shootdown actions which were queued when we were not executing code. */
2209 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTlbShootdown);
2210
2211 for (unsigned i=0;i<pVCpu->hwaccm.s.TlbShootdown.cPages;i++)
2212 {
2213 /* aTlbShootdownPages contains physical addresses in this case. */
2214 hmR0VmxFlushEPT(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushPage, pVCpu->hwaccm.s.TlbShootdown.aPages[i]);
2215 }
2216 }
2217 pVCpu->hwaccm.s.TlbShootdown.cPages= 0;
2218 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2219
2220#ifdef VBOX_WITH_STATISTICS
2221 if (pVCpu->hwaccm.s.fForceTLBFlush)
2222 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushTLBWorldSwitch);
2223 else
2224 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatNoFlushTLBWorldSwitch);
2225#endif
2226}
2227
2228#ifdef HWACCM_VTX_WITH_VPID
2229/**
2230 * Setup the tagged TLB for VPID
2231 *
2232 * @returns VBox status code.
2233 * @param pVM The VM to operate on.
2234 * @param pVCpu The VMCPU to operate on.
2235 */
2236static void hmR0VmxSetupTLBVPID(PVM pVM, PVMCPU pVCpu)
2237{
2238 PHMGLOBLCPUINFO pCpu;
2239
2240 Assert(pVM->hwaccm.s.vmx.fVPID);
2241 Assert(!pVM->hwaccm.s.fNestedPaging);
2242
2243 /* Deal with tagged TLBs if VPID or EPT is supported. */
2244 pCpu = HWACCMR0GetCurrentCpu();
2245 /* Force a TLB flush for the first world switch if the current cpu differs from the one we ran on last. */
2246 /* Note that this can happen both for start and resume due to long jumps back to ring 3. */
2247 if ( pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu
2248 /* if the tlb flush count has changed, another VM has flushed the TLB of this cpu, so we can't use our current ASID anymore. */
2249 || pVCpu->hwaccm.s.cTLBFlushes != pCpu->cTLBFlushes)
2250 {
2251 /* Force a TLB flush on VM entry. */
2252 pVCpu->hwaccm.s.fForceTLBFlush = true;
2253 }
2254 else
2255 Assert(!pCpu->fFlushTLB);
2256
2257 pVCpu->hwaccm.s.idLastCpu = pCpu->idCpu;
2258
2259 /* Check for tlb shootdown flushes. */
2260 if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2261 pVCpu->hwaccm.s.fForceTLBFlush = true;
2262
2263 /* Make sure we flush the TLB when required. Switch ASID to achieve the same thing, but without actually flushing the whole TLB (which is expensive). */
2264 if (pVCpu->hwaccm.s.fForceTLBFlush)
2265 {
2266 if ( ++pCpu->uCurrentASID >= pVM->hwaccm.s.uMaxASID
2267 || pCpu->fFlushTLB)
2268 {
2269 pCpu->fFlushTLB = false;
2270 pCpu->uCurrentASID = 1; /* start at 1; host uses 0 */
2271 pCpu->cTLBFlushes++;
2272 hmR0VmxFlushVPID(pVM, pVCpu, VMX_FLUSH_ALL_CONTEXTS, 0);
2273 }
2274 else
2275 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushASID);
2276
2277 pVCpu->hwaccm.s.fForceTLBFlush = false;
2278 pVCpu->hwaccm.s.cTLBFlushes = pCpu->cTLBFlushes;
2279 pVCpu->hwaccm.s.uCurrentASID = pCpu->uCurrentASID;
2280 }
2281 else
2282 {
2283 Assert(!pCpu->fFlushTLB);
2284 Assert(pVCpu->hwaccm.s.uCurrentASID && pCpu->uCurrentASID);
2285
2286 if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2287 {
2288 /* Deal with pending TLB shootdown actions which were queued when we were not executing code. */
2289 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTlbShootdown);
2290 for (unsigned i = 0; i < pVCpu->hwaccm.s.TlbShootdown.cPages; i++)
2291 hmR0VmxFlushVPID(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushPage, pVCpu->hwaccm.s.TlbShootdown.aPages[i]);
2292 }
2293 }
2294 pVCpu->hwaccm.s.TlbShootdown.cPages = 0;
2295 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2296
2297 AssertMsg(pVCpu->hwaccm.s.cTLBFlushes == pCpu->cTLBFlushes, ("Flush count mismatch for cpu %d (%x vs %x)\n", pCpu->idCpu, pVCpu->hwaccm.s.cTLBFlushes, pCpu->cTLBFlushes));
2298 AssertMsg(pCpu->uCurrentASID >= 1 && pCpu->uCurrentASID < pVM->hwaccm.s.uMaxASID, ("cpu%d uCurrentASID = %x\n", pCpu->idCpu, pCpu->uCurrentASID));
2299 AssertMsg(pVCpu->hwaccm.s.uCurrentASID >= 1 && pVCpu->hwaccm.s.uCurrentASID < pVM->hwaccm.s.uMaxASID, ("cpu%d VM uCurrentASID = %x\n", pCpu->idCpu, pVCpu->hwaccm.s.uCurrentASID));
2300
2301 int rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hwaccm.s.uCurrentASID);
2302 AssertRC(rc);
2303
2304 if (pVCpu->hwaccm.s.fForceTLBFlush)
2305 hmR0VmxFlushVPID(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushContext, 0);
2306
2307# ifdef VBOX_WITH_STATISTICS
2308 if (pVCpu->hwaccm.s.fForceTLBFlush)
2309 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushTLBWorldSwitch);
2310 else
2311 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatNoFlushTLBWorldSwitch);
2312# endif
2313}
2314#endif /* HWACCM_VTX_WITH_VPID */
2315
2316/**
2317 * Runs guest code in a VT-x VM.
2318 *
2319 * @returns VBox status code.
2320 * @param pVM The VM to operate on.
2321 * @param pVCpu The VMCPU to operate on.
2322 * @param pCtx Guest context
2323 */
2324VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2325{
2326 STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatEntry, x);
2327 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hwaccm.s.StatExit1);
2328 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hwaccm.s.StatExit2);
2329
2330 VBOXSTRICTRC rc = VINF_SUCCESS;
2331 int rc2;
2332 RTGCUINTREG val;
2333 RTGCUINTREG exitReason = (RTGCUINTREG)VMX_EXIT_INVALID;
2334 RTGCUINTREG instrError, cbInstr;
2335 RTGCUINTPTR exitQualification = 0;
2336 RTGCUINTPTR intInfo = 0; /* shut up buggy gcc 4 */
2337 RTGCUINTPTR errCode, instrInfo;
2338 bool fSetupTPRCaching = false;
2339 uint64_t u64OldLSTAR = 0;
2340 uint8_t u8LastTPR = 0;
2341 RTCCUINTREG uOldEFlags = ~(RTCCUINTREG)0;
2342 unsigned cResume = 0;
2343#ifdef VBOX_STRICT
2344 RTCPUID idCpuCheck;
2345 bool fWasInLongMode = false;
2346#endif
2347#ifdef VBOX_HIGH_RES_TIMERS_HACK_IN_RING0
2348 uint64_t u64LastTime = RTTimeMilliTS();
2349#endif
2350
2351 Assert(!(pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC) || (pVCpu->hwaccm.s.vmx.pbVAPIC && pVM->hwaccm.s.vmx.pAPIC));
2352
2353 /* Check if we need to use TPR shadowing. */
2354 if ( CPUMIsGuestInLongModeEx(pCtx)
2355 || ( ((pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC) || pVM->hwaccm.s.fTRPPatchingAllowed)
2356 && pVM->hwaccm.s.fHasIoApic)
2357 )
2358 {
2359 fSetupTPRCaching = true;
2360 }
2361
2362 Log2(("\nE"));
2363
2364#ifdef VBOX_STRICT
2365 {
2366 RTCCUINTREG val2;
2367
2368 rc2 = VMXReadVMCS(VMX_VMCS_CTRL_PIN_EXEC_CONTROLS, &val2);
2369 AssertRC(rc2);
2370 Log2(("VMX_VMCS_CTRL_PIN_EXEC_CONTROLS = %08x\n", val2));
2371
2372 /* allowed zero */
2373 if ((val2 & pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.disallowed0)
2374 Log(("Invalid VMX_VMCS_CTRL_PIN_EXEC_CONTROLS: zero\n"));
2375
2376 /* allowed one */
2377 if ((val2 & ~pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.allowed1) != 0)
2378 Log(("Invalid VMX_VMCS_CTRL_PIN_EXEC_CONTROLS: one\n"));
2379
2380 rc2 = VMXReadVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, &val2);
2381 AssertRC(rc2);
2382 Log2(("VMX_VMCS_CTRL_PROC_EXEC_CONTROLS = %08x\n", val2));
2383
2384 /* Must be set according to the MSR, but can be cleared in case of EPT. */
2385 if (pVM->hwaccm.s.fNestedPaging)
2386 val2 |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT
2387 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
2388 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
2389
2390 /* allowed zero */
2391 if ((val2 & pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.disallowed0)
2392 Log(("Invalid VMX_VMCS_CTRL_PROC_EXEC_CONTROLS: zero\n"));
2393
2394 /* allowed one */
2395 if ((val2 & ~pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1) != 0)
2396 Log(("Invalid VMX_VMCS_CTRL_PROC_EXEC_CONTROLS: one\n"));
2397
2398 rc2 = VMXReadVMCS(VMX_VMCS_CTRL_ENTRY_CONTROLS, &val2);
2399 AssertRC(rc2);
2400 Log2(("VMX_VMCS_CTRL_ENTRY_CONTROLS = %08x\n", val2));
2401
2402 /* allowed zero */
2403 if ((val2 & pVM->hwaccm.s.vmx.msr.vmx_entry.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_entry.n.disallowed0)
2404 Log(("Invalid VMX_VMCS_CTRL_ENTRY_CONTROLS: zero\n"));
2405
2406 /* allowed one */
2407 if ((val2 & ~pVM->hwaccm.s.vmx.msr.vmx_entry.n.allowed1) != 0)
2408 Log(("Invalid VMX_VMCS_CTRL_ENTRY_CONTROLS: one\n"));
2409
2410 rc2 = VMXReadVMCS(VMX_VMCS_CTRL_EXIT_CONTROLS, &val2);
2411 AssertRC(rc2);
2412 Log2(("VMX_VMCS_CTRL_EXIT_CONTROLS = %08x\n", val2));
2413
2414 /* allowed zero */
2415 if ((val2 & pVM->hwaccm.s.vmx.msr.vmx_exit.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_exit.n.disallowed0)
2416 Log(("Invalid VMX_VMCS_CTRL_EXIT_CONTROLS: zero\n"));
2417
2418 /* allowed one */
2419 if ((val2 & ~pVM->hwaccm.s.vmx.msr.vmx_exit.n.allowed1) != 0)
2420 Log(("Invalid VMX_VMCS_CTRL_EXIT_CONTROLS: one\n"));
2421 }
2422 fWasInLongMode = CPUMIsGuestInLongModeEx(pCtx);
2423#endif /* VBOX_STRICT */
2424
2425#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2426 pVCpu->hwaccm.s.vmx.VMCSCache.u64TimeEntry = RTTimeNanoTS();
2427#endif
2428
2429 /* We can jump to this point to resume execution after determining that a VM-exit is innocent.
2430 */
2431ResumeExecution:
2432 if (!STAM_REL_PROFILE_ADV_IS_RUNNING(&pVCpu->hwaccm.s.StatEntry))
2433 STAM_REL_PROFILE_ADV_STOP_START(&pVCpu->hwaccm.s.StatExit2, &pVCpu->hwaccm.s.StatEntry, x);
2434 AssertMsg(pVCpu->hwaccm.s.idEnteredCpu == RTMpCpuId(),
2435 ("Expected %d, I'm %d; cResume=%d exitReason=%RGv exitQualification=%RGv\n",
2436 (int)pVCpu->hwaccm.s.idEnteredCpu, (int)RTMpCpuId(), cResume, exitReason, exitQualification));
2437 Assert(!HWACCMR0SuspendPending());
2438 /* Not allowed to switch modes without reloading the host state (32->64 switcher)!! */
2439 Assert(fWasInLongMode == CPUMIsGuestInLongModeEx(pCtx));
2440
2441 /* Safety precaution; looping for too long here can have a very bad effect on the host */
2442 if (RT_UNLIKELY(++cResume > pVM->hwaccm.s.cMaxResumeLoops))
2443 {
2444 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitMaxResume);
2445 rc = VINF_EM_RAW_INTERRUPT;
2446 goto end;
2447 }
2448
2449 /* Check for irq inhibition due to instruction fusing (sti, mov ss). */
2450 if (VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
2451 {
2452 Log(("VM_FF_INHIBIT_INTERRUPTS at %RGv successor %RGv\n", (RTGCPTR)pCtx->rip, EMGetInhibitInterruptsPC(pVCpu)));
2453 if (pCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
2454 {
2455 /* Note: we intentionally don't clear VM_FF_INHIBIT_INTERRUPTS here.
2456 * Before we are able to execute this instruction in raw mode (iret to guest code) an external interrupt might
2457 * force a world switch again. Possibly allowing a guest interrupt to be dispatched in the process. This could
2458 * break the guest. Sounds very unlikely, but such timing sensitive problems are not as rare as you might think.
2459 */
2460 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2461 /* Irq inhibition is no longer active; clear the corresponding VMX state. */
2462 rc2 = VMXWriteVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, 0);
2463 AssertRC(rc2);
2464 }
2465 }
2466 else
2467 {
2468 /* Irq inhibition is no longer active; clear the corresponding VMX state. */
2469 rc2 = VMXWriteVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, 0);
2470 AssertRC(rc2);
2471 }
2472
2473#ifdef VBOX_HIGH_RES_TIMERS_HACK_IN_RING0
2474 if (RT_UNLIKELY((cResume & 0xf) == 0))
2475 {
2476 uint64_t u64CurTime = RTTimeMilliTS();
2477
2478 if (RT_UNLIKELY(u64CurTime > u64LastTime))
2479 {
2480 u64LastTime = u64CurTime;
2481 TMTimerPollVoid(pVM, pVCpu);
2482 }
2483 }
2484#endif
2485
2486 /* Check for pending actions that force us to go back to ring 3. */
2487 if ( VM_FF_ISPENDING(pVM, VM_FF_HWACCM_TO_R3_MASK | VM_FF_REQUEST | VM_FF_PGM_POOL_FLUSH_PENDING | VM_FF_PDM_DMA)
2488 || VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_HWACCM_TO_R3_MASK | VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL | VMCPU_FF_REQUEST))
2489 {
2490 /* Check if a sync operation is pending. */
2491 if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
2492 {
2493 rc = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4, VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
2494 if (rc != VINF_SUCCESS)
2495 {
2496 AssertRC(VBOXSTRICTRC_VAL(rc));
2497 Log(("Pending pool sync is forcing us back to ring 3; rc=%d\n", VBOXSTRICTRC_VAL(rc)));
2498 goto end;
2499 }
2500 }
2501
2502#ifdef DEBUG
2503 /* Intercept X86_XCPT_DB if stepping is enabled */
2504 if (!DBGFIsStepping(pVCpu))
2505#endif
2506 {
2507 if ( VM_FF_ISPENDING(pVM, VM_FF_HWACCM_TO_R3_MASK)
2508 || VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_HWACCM_TO_R3_MASK))
2509 {
2510 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatSwitchToR3);
2511 rc = RT_UNLIKELY(VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
2512 goto end;
2513 }
2514 }
2515
2516 /* Pending request packets might contain actions that need immediate attention, such as pending hardware interrupts. */
2517 if ( VM_FF_ISPENDING(pVM, VM_FF_REQUEST)
2518 || VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_REQUEST))
2519 {
2520 rc = VINF_EM_PENDING_REQUEST;
2521 goto end;
2522 }
2523
2524 /* Check if a pgm pool flush is in progress. */
2525 if (VM_FF_ISPENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
2526 {
2527 rc = VINF_PGM_POOL_FLUSH_PENDING;
2528 goto end;
2529 }
2530
2531 /* Check if DMA work is pending (2nd+ run). */
2532 if (VM_FF_ISPENDING(pVM, VM_FF_PDM_DMA) && cResume > 1)
2533 {
2534 rc = VINF_EM_RAW_TO_R3;
2535 goto end;
2536 }
2537 }
2538
2539#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
2540 /*
2541 * Exit to ring-3 preemption/work is pending.
2542 *
2543 * Interrupts are disabled before the call to make sure we don't miss any interrupt
2544 * that would flag preemption (IPI, timer tick, ++). (Would've been nice to do this
2545 * further down, but hmR0VmxCheckPendingInterrupt makes that impossible.)
2546 *
2547 * Note! Interrupts must be disabled done *before* we check for TLB flushes; TLB
2548 * shootdowns rely on this.
2549 */
2550 uOldEFlags = ASMIntDisableFlags();
2551 if (RTThreadPreemptIsPending(NIL_RTTHREAD))
2552 {
2553 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitPreemptPending);
2554 rc = VINF_EM_RAW_INTERRUPT;
2555 goto end;
2556 }
2557 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
2558#endif
2559
2560 /* When external interrupts are pending, we should exit the VM when IF is set. */
2561 /* Note! *After* VM_FF_INHIBIT_INTERRUPTS check!!! */
2562 rc = hmR0VmxCheckPendingInterrupt(pVM, pVCpu, pCtx);
2563 if (RT_FAILURE(rc))
2564 goto end;
2565
2566 /** @todo check timers?? */
2567
2568 /* TPR caching using CR8 is only available in 64 bits mode */
2569 /* Note the 32 bits exception for AMD (X86_CPUID_AMD_FEATURE_ECX_CR8L), but that appears missing in Intel CPUs */
2570 /* Note: we can't do this in LoadGuestState as PDMApicGetTPR can jump back to ring 3 (lock)!!!!! (no longer true) */
2571 /**
2572 * @todo query and update the TPR only when it could have been changed (mmio access & wrmsr (x2apic))
2573 */
2574 if (fSetupTPRCaching)
2575 {
2576 /* TPR caching in CR8 */
2577 bool fPending;
2578
2579 rc2 = PDMApicGetTPR(pVCpu, &u8LastTPR, &fPending);
2580 AssertRC(rc2);
2581 /* The TPR can be found at offset 0x80 in the APIC mmio page. */
2582 pVCpu->hwaccm.s.vmx.pbVAPIC[0x80] = u8LastTPR;
2583
2584 /* Two options here:
2585 * - external interrupt pending, but masked by the TPR value.
2586 * -> a CR8 update that lower the current TPR value should cause an exit
2587 * - no pending interrupts
2588 * -> We don't need to be explicitely notified. There are enough world switches for detecting pending interrupts.
2589 */
2590 rc = VMXWriteVMCS(VMX_VMCS_CTRL_TPR_THRESHOLD, (fPending) ? (u8LastTPR >> 4) : 0); /* cr8 bits 3-0 correspond to bits 7-4 of the task priority mmio register. */
2591 AssertRC(VBOXSTRICTRC_VAL(rc));
2592
2593 if (pVM->hwaccm.s.fTPRPatchingActive)
2594 {
2595 Assert(!CPUMIsGuestInLongModeEx(pCtx));
2596 /* Our patch code uses LSTAR for TPR caching. */
2597 pCtx->msrLSTAR = u8LastTPR;
2598
2599 if (fPending)
2600 {
2601 /* A TPR change could activate a pending interrupt, so catch lstar writes. */
2602 hmR0VmxSetMSRPermission(pVCpu, MSR_K8_LSTAR, true, false);
2603 }
2604 else
2605 {
2606 /* No interrupts are pending, so we don't need to be explicitely notified.
2607 * There are enough world switches for detecting pending interrupts.
2608 */
2609 hmR0VmxSetMSRPermission(pVCpu, MSR_K8_LSTAR, true, true);
2610 }
2611 }
2612 }
2613
2614#if defined(HWACCM_VTX_WITH_EPT) && defined(LOG_ENABLED)
2615 if ( pVM->hwaccm.s.fNestedPaging
2616# ifdef HWACCM_VTX_WITH_VPID
2617 || pVM->hwaccm.s.vmx.fVPID
2618# endif /* HWACCM_VTX_WITH_VPID */
2619 )
2620 {
2621 PHMGLOBLCPUINFO pCpu;
2622
2623 pCpu = HWACCMR0GetCurrentCpu();
2624 if ( pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu
2625 || pVCpu->hwaccm.s.cTLBFlushes != pCpu->cTLBFlushes)
2626 {
2627 if (pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu)
2628 LogFlow(("Force TLB flush due to rescheduling to a different cpu (%d vs %d)\n", pVCpu->hwaccm.s.idLastCpu, pCpu->idCpu));
2629 else
2630 LogFlow(("Force TLB flush due to changed TLB flush count (%x vs %x)\n", pVCpu->hwaccm.s.cTLBFlushes, pCpu->cTLBFlushes));
2631 }
2632 if (pCpu->fFlushTLB)
2633 LogFlow(("Force TLB flush: first time cpu %d is used -> flush\n", pCpu->idCpu));
2634 else
2635 if (pVCpu->hwaccm.s.fForceTLBFlush)
2636 LogFlow(("Manual TLB flush\n"));
2637 }
2638#endif
2639#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
2640 PGMRZDynMapFlushAutoSet(pVCpu);
2641#endif
2642
2643 /*
2644 * NOTE: DO NOT DO ANYTHING AFTER THIS POINT THAT MIGHT JUMP BACK TO RING 3!
2645 * (until the actual world switch)
2646 */
2647#ifdef VBOX_STRICT
2648 idCpuCheck = RTMpCpuId();
2649#endif
2650#ifdef LOG_ENABLED
2651 VMMR0LogFlushDisable(pVCpu);
2652#endif
2653 /* Save the host state first. */
2654 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_HOST_CONTEXT)
2655 {
2656 rc = VMXR0SaveHostState(pVM, pVCpu);
2657 if (RT_UNLIKELY(rc != VINF_SUCCESS))
2658 {
2659 VMMR0LogFlushEnable(pVCpu);
2660 goto end;
2661 }
2662 }
2663
2664 /* Load the guest state */
2665 if (!pVCpu->hwaccm.s.fContextUseFlags)
2666 {
2667 VMXR0LoadMinimalGuestState(pVM, pVCpu, pCtx);
2668 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatLoadMinimal);
2669 }
2670 else
2671 {
2672 rc = VMXR0LoadGuestState(pVM, pVCpu, pCtx);
2673 if (RT_UNLIKELY(rc != VINF_SUCCESS))
2674 {
2675 VMMR0LogFlushEnable(pVCpu);
2676 goto end;
2677 }
2678 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatLoadFull);
2679 }
2680
2681#ifndef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
2682 /* Disable interrupts to make sure a poke will interrupt execution.
2683 * This must be done *before* we check for TLB flushes; TLB shootdowns rely on this.
2684 */
2685 uOldEFlags = ASMIntDisableFlags();
2686 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
2687#endif
2688
2689 /* Non-register state Guest Context */
2690 /** @todo change me according to cpu state */
2691 rc2 = VMXWriteVMCS(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_CMS_GUEST_ACTIVITY_ACTIVE);
2692 AssertRC(rc2);
2693
2694 /* Set TLB flush state as checked until we return from the world switch. */
2695 ASMAtomicWriteBool(&pVCpu->hwaccm.s.fCheckedTLBFlush, true);
2696 /* Deal with tagged TLB setup and invalidation. */
2697 pVM->hwaccm.s.vmx.pfnSetupTaggedTLB(pVM, pVCpu);
2698
2699 /* Manual save and restore:
2700 * - General purpose registers except RIP, RSP
2701 *
2702 * Trashed:
2703 * - CR2 (we don't care)
2704 * - LDTR (reset to 0)
2705 * - DRx (presumably not changed at all)
2706 * - DR7 (reset to 0x400)
2707 * - EFLAGS (reset to RT_BIT(1); not relevant)
2708 *
2709 */
2710
2711 /* All done! Let's start VM execution. */
2712 STAM_PROFILE_ADV_STOP_START(&pVCpu->hwaccm.s.StatEntry, &pVCpu->hwaccm.s.StatInGC, x);
2713 Assert(idCpuCheck == RTMpCpuId());
2714
2715#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2716 pVCpu->hwaccm.s.vmx.VMCSCache.cResume = cResume;
2717 pVCpu->hwaccm.s.vmx.VMCSCache.u64TimeSwitch = RTTimeNanoTS();
2718#endif
2719
2720 /* Save the current TPR value in the LSTAR msr so our patches can access it. */
2721 if (pVM->hwaccm.s.fTPRPatchingActive)
2722 {
2723 Assert(pVM->hwaccm.s.fTPRPatchingActive);
2724 u64OldLSTAR = ASMRdMsr(MSR_K8_LSTAR);
2725 ASMWrMsr(MSR_K8_LSTAR, u8LastTPR);
2726 }
2727
2728 TMNotifyStartOfExecution(pVCpu);
2729#ifdef VBOX_WITH_KERNEL_USING_XMM
2730 rc = hwaccmR0VMXStartVMWrapXMM(pVCpu->hwaccm.s.fResumeVM, pCtx, &pVCpu->hwaccm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hwaccm.s.vmx.pfnStartVM);
2731#else
2732 rc = pVCpu->hwaccm.s.vmx.pfnStartVM(pVCpu->hwaccm.s.fResumeVM, pCtx, &pVCpu->hwaccm.s.vmx.VMCSCache, pVM, pVCpu);
2733#endif
2734 ASMAtomicWriteBool(&pVCpu->hwaccm.s.fCheckedTLBFlush, false);
2735 ASMAtomicIncU32(&pVCpu->hwaccm.s.cWorldSwitchExits);
2736 /* Possibly the last TSC value seen by the guest (too high) (only when we're in tsc offset mode). */
2737 if (!(pVCpu->hwaccm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT))
2738 TMCpuTickSetLastSeen(pVCpu, ASMReadTSC() + pVCpu->hwaccm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
2739
2740 TMNotifyEndOfExecution(pVCpu);
2741 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED);
2742 Assert(!(ASMGetFlags() & X86_EFL_IF));
2743
2744 /* Restore the host LSTAR msr if the guest could have changed it. */
2745 if (pVM->hwaccm.s.fTPRPatchingActive)
2746 {
2747 Assert(pVM->hwaccm.s.fTPRPatchingActive);
2748 pVCpu->hwaccm.s.vmx.pbVAPIC[0x80] = pCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
2749 ASMWrMsr(MSR_K8_LSTAR, u64OldLSTAR);
2750 }
2751
2752 STAM_PROFILE_ADV_STOP_START(&pVCpu->hwaccm.s.StatInGC, &pVCpu->hwaccm.s.StatExit1, x);
2753 ASMSetFlags(uOldEFlags);
2754#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
2755 uOldEFlags = ~(RTCCUINTREG)0;
2756#endif
2757
2758 AssertMsg(!pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries, ("pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries=%d\n", pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries));
2759
2760 /* In case we execute a goto ResumeExecution later on. */
2761 pVCpu->hwaccm.s.fResumeVM = true;
2762 pVCpu->hwaccm.s.fForceTLBFlush = false;
2763
2764 /*
2765 * !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
2766 * IMPORTANT: WE CAN'T DO ANY LOGGING OR OPERATIONS THAT CAN DO A LONGJMP BACK TO RING 3 *BEFORE* WE'VE SYNCED BACK (MOST OF) THE GUEST STATE
2767 * !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
2768 */
2769
2770 if (RT_UNLIKELY(rc != VINF_SUCCESS))
2771 {
2772 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx);
2773 VMMR0LogFlushEnable(pVCpu);
2774 goto end;
2775 }
2776
2777 /* Success. Query the guest state and figure out what has happened. */
2778
2779 /* Investigate why there was a VM-exit. */
2780 rc2 = VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_REASON, &exitReason);
2781 STAM_COUNTER_INC(&pVCpu->hwaccm.s.paStatExitReasonR0[exitReason & MASK_EXITREASON_STAT]);
2782
2783 exitReason &= 0xffff; /* bit 0-15 contain the exit code. */
2784 rc2 |= VMXReadCachedVMCS(VMX_VMCS32_RO_VM_INSTR_ERROR, &instrError);
2785 rc2 |= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &cbInstr);
2786 rc2 |= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &intInfo);
2787 /* might not be valid; depends on VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID. */
2788 rc2 |= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERRCODE, &errCode);
2789 rc2 |= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INSTR_INFO, &instrInfo);
2790 rc2 |= VMXReadCachedVMCS(VMX_VMCS_RO_EXIT_QUALIFICATION, &exitQualification);
2791 AssertRC(rc2);
2792
2793 /* Sync back the guest state */
2794 rc2 = VMXR0SaveGuestState(pVM, pVCpu, pCtx);
2795 AssertRC(rc2);
2796
2797 /* Note! NOW IT'S SAFE FOR LOGGING! */
2798 VMMR0LogFlushEnable(pVCpu);
2799 Log2(("Raw exit reason %08x\n", exitReason));
2800
2801 /* Check if an injected event was interrupted prematurely. */
2802 rc2 = VMXReadCachedVMCS(VMX_VMCS32_RO_IDT_INFO, &val);
2803 AssertRC(rc2);
2804 pVCpu->hwaccm.s.Event.intInfo = VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(val);
2805 if ( VMX_EXIT_INTERRUPTION_INFO_VALID(pVCpu->hwaccm.s.Event.intInfo)
2806 /* Ignore 'int xx' as they'll be restarted anyway. */
2807 && VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hwaccm.s.Event.intInfo) != VMX_EXIT_INTERRUPTION_INFO_TYPE_SW
2808 /* Ignore software exceptions (such as int3) as they'll reoccur when we restart the instruction anyway. */
2809 && VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hwaccm.s.Event.intInfo) != VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT)
2810 {
2811 Assert(!pVCpu->hwaccm.s.Event.fPending);
2812 pVCpu->hwaccm.s.Event.fPending = true;
2813 /* Error code present? */
2814 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVCpu->hwaccm.s.Event.intInfo))
2815 {
2816 rc2 = VMXReadCachedVMCS(VMX_VMCS32_RO_IDT_ERRCODE, &val);
2817 AssertRC(rc2);
2818 pVCpu->hwaccm.s.Event.errCode = val;
2819 Log(("Pending inject %RX64 at %RGv exit=%08x intInfo=%08x exitQualification=%RGv pending error=%RX64\n", pVCpu->hwaccm.s.Event.intInfo, (RTGCPTR)pCtx->rip, exitReason, intInfo, exitQualification, val));
2820 }
2821 else
2822 {
2823 Log(("Pending inject %RX64 at %RGv exit=%08x intInfo=%08x exitQualification=%RGv\n", pVCpu->hwaccm.s.Event.intInfo, (RTGCPTR)pCtx->rip, exitReason, intInfo, exitQualification));
2824 pVCpu->hwaccm.s.Event.errCode = 0;
2825 }
2826 }
2827#ifdef VBOX_STRICT
2828 else
2829 if ( VMX_EXIT_INTERRUPTION_INFO_VALID(pVCpu->hwaccm.s.Event.intInfo)
2830 /* Ignore software exceptions (such as int3) as they're reoccur when we restart the instruction anyway. */
2831 && VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hwaccm.s.Event.intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT)
2832 {
2833 Log(("Ignore pending inject %RX64 at %RGv exit=%08x intInfo=%08x exitQualification=%RGv\n", pVCpu->hwaccm.s.Event.intInfo, (RTGCPTR)pCtx->rip, exitReason, intInfo, exitQualification));
2834 }
2835
2836 if (exitReason == VMX_EXIT_ERR_INVALID_GUEST_STATE)
2837 HWACCMDumpRegs(pVM, pVCpu, pCtx);
2838#endif
2839
2840 Log2(("E%d: New EIP=%x:%RGv\n", (uint32_t)exitReason, pCtx->cs, (RTGCPTR)pCtx->rip));
2841 Log2(("Exit reason %d, exitQualification %RGv\n", (uint32_t)exitReason, exitQualification));
2842 Log2(("instrInfo=%d instrError=%d instr length=%d\n", (uint32_t)instrInfo, (uint32_t)instrError, (uint32_t)cbInstr));
2843 Log2(("Interruption error code %d\n", (uint32_t)errCode));
2844 Log2(("IntInfo = %08x\n", (uint32_t)intInfo));
2845
2846 /* Sync back the TPR if it was changed. */
2847 if ( fSetupTPRCaching
2848 && u8LastTPR != pVCpu->hwaccm.s.vmx.pbVAPIC[0x80])
2849 {
2850 rc2 = PDMApicSetTPR(pVCpu, pVCpu->hwaccm.s.vmx.pbVAPIC[0x80]);
2851 AssertRC(rc2);
2852 }
2853
2854 STAM_PROFILE_ADV_STOP_START(&pVCpu->hwaccm.s.StatExit1, &pVCpu->hwaccm.s.StatExit2, x);
2855
2856 /* Some cases don't need a complete resync of the guest CPU state; handle them here. */
2857 Assert(rc == VINF_SUCCESS); /* might consider VERR_IPE_UNINITIALIZED_STATUS here later... */
2858 switch (exitReason)
2859 {
2860 case VMX_EXIT_EXCEPTION: /* 0 Exception or non-maskable interrupt (NMI). */
2861 case VMX_EXIT_EXTERNAL_IRQ: /* 1 External interrupt. */
2862 {
2863 uint32_t vector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(intInfo);
2864
2865 if (!VMX_EXIT_INTERRUPTION_INFO_VALID(intInfo))
2866 {
2867 Assert(exitReason == VMX_EXIT_EXTERNAL_IRQ);
2868#if 0 //def VBOX_WITH_VMMR0_DISABLE_PREEMPTION
2869 if ( RTThreadPreemptIsPendingTrusty()
2870 && !RTThreadPreemptIsPending(NIL_RTTHREAD))
2871 goto ResumeExecution;
2872#endif
2873 /* External interrupt; leave to allow it to be dispatched again. */
2874 rc = VINF_EM_RAW_INTERRUPT;
2875 break;
2876 }
2877 STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2878 switch (VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo))
2879 {
2880 case VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI: /* Non-maskable interrupt. */
2881 /* External interrupt; leave to allow it to be dispatched again. */
2882 rc = VINF_EM_RAW_INTERRUPT;
2883 break;
2884
2885 case VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT: /* External hardware interrupt. */
2886 AssertFailed(); /* can't come here; fails the first check. */
2887 break;
2888
2889 case VMX_EXIT_INTERRUPTION_INFO_TYPE_DBEXCPT: /* Unknown why we get this type for #DB */
2890 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT: /* Software exception. (#BP or #OF) */
2891 Assert(vector == 1 || vector == 3 || vector == 4);
2892 /* no break */
2893 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT: /* Hardware exception. */
2894 Log2(("Hardware/software interrupt %d\n", vector));
2895 switch (vector)
2896 {
2897 case X86_XCPT_NM:
2898 {
2899 Log(("#NM fault at %RGv error code %x\n", (RTGCPTR)pCtx->rip, errCode));
2900
2901 /** @todo don't intercept #NM exceptions anymore when we've activated the guest FPU state. */
2902 /* If we sync the FPU/XMM state on-demand, then we can continue execution as if nothing has happened. */
2903 rc = CPUMR0LoadGuestFPU(pVM, pVCpu, pCtx);
2904 if (rc == VINF_SUCCESS)
2905 {
2906 Assert(CPUMIsGuestFPUStateActive(pVCpu));
2907
2908 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitShadowNM);
2909
2910 /* Continue execution. */
2911 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_CR0;
2912
2913 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2914 goto ResumeExecution;
2915 }
2916
2917 Log(("Forward #NM fault to the guest\n"));
2918 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestNM);
2919 rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, 0);
2920 AssertRC(rc2);
2921 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2922 goto ResumeExecution;
2923 }
2924
2925 case X86_XCPT_PF: /* Page fault */
2926 {
2927#ifdef DEBUG
2928 if (pVM->hwaccm.s.fNestedPaging)
2929 { /* A genuine pagefault.
2930 * Forward the trap to the guest by injecting the exception and resuming execution.
2931 */
2932 Log(("Guest page fault at %RGv cr2=%RGv error code %RGv rsp=%RGv\n", (RTGCPTR)pCtx->rip, exitQualification, errCode, (RTGCPTR)pCtx->rsp));
2933
2934 Assert(CPUMIsGuestInPagedProtectedModeEx(pCtx));
2935
2936 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestPF);
2937
2938 /* Now we must update CR2. */
2939 pCtx->cr2 = exitQualification;
2940 rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
2941 AssertRC(rc2);
2942
2943 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2944 goto ResumeExecution;
2945 }
2946#endif
2947 Assert(!pVM->hwaccm.s.fNestedPaging);
2948
2949#ifdef VBOX_HWACCM_WITH_GUEST_PATCHING
2950 /* Shortcut for APIC TPR reads and writes; 32 bits guests only */
2951 if ( pVM->hwaccm.s.fTRPPatchingAllowed
2952 && pVM->hwaccm.s.pGuestPatchMem
2953 && (exitQualification & 0xfff) == 0x080
2954 && !(errCode & X86_TRAP_PF_P) /* not present */
2955 && CPUMGetGuestCPL(pVCpu, CPUMCTX2CORE(pCtx)) == 0
2956 && !CPUMIsGuestInLongModeEx(pCtx)
2957 && pVM->hwaccm.s.cPatches < RT_ELEMENTS(pVM->hwaccm.s.aPatches))
2958 {
2959 RTGCPHYS GCPhysApicBase, GCPhys;
2960 PDMApicGetBase(pVM, &GCPhysApicBase); /* @todo cache this */
2961 GCPhysApicBase &= PAGE_BASE_GC_MASK;
2962
2963 rc = PGMGstGetPage(pVCpu, (RTGCPTR)exitQualification, NULL, &GCPhys);
2964 if ( rc == VINF_SUCCESS
2965 && GCPhys == GCPhysApicBase)
2966 {
2967 /* Only attempt to patch the instruction once. */
2968 PHWACCMTPRPATCH pPatch = (PHWACCMTPRPATCH)RTAvloU32Get(&pVM->hwaccm.s.PatchTree, (AVLOU32KEY)pCtx->eip);
2969 if (!pPatch)
2970 {
2971 rc = VINF_EM_HWACCM_PATCH_TPR_INSTR;
2972 break;
2973 }
2974 }
2975 }
2976#endif
2977
2978 Log2(("Page fault at %RGv error code %x\n", exitQualification, errCode));
2979 /* Exit qualification contains the linear address of the page fault. */
2980 TRPMAssertTrap(pVCpu, X86_XCPT_PF, TRPM_TRAP);
2981 TRPMSetErrorCode(pVCpu, errCode);
2982 TRPMSetFaultAddress(pVCpu, exitQualification);
2983
2984 /* Shortcut for APIC TPR reads and writes. */
2985 if ( (exitQualification & 0xfff) == 0x080
2986 && !(errCode & X86_TRAP_PF_P) /* not present */
2987 && fSetupTPRCaching
2988 && (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
2989 {
2990 RTGCPHYS GCPhysApicBase, GCPhys;
2991 PDMApicGetBase(pVM, &GCPhysApicBase); /* @todo cache this */
2992 GCPhysApicBase &= PAGE_BASE_GC_MASK;
2993
2994 rc = PGMGstGetPage(pVCpu, (RTGCPTR)exitQualification, NULL, &GCPhys);
2995 if ( rc == VINF_SUCCESS
2996 && GCPhys == GCPhysApicBase)
2997 {
2998 Log(("Enable VT-x virtual APIC access filtering\n"));
2999 rc2 = IOMMMIOMapMMIOHCPage(pVM, GCPhysApicBase, pVM->hwaccm.s.vmx.pAPICPhys, X86_PTE_RW | X86_PTE_P);
3000 AssertRC(rc2);
3001 }
3002 }
3003
3004 /* Forward it to our trap handler first, in case our shadow pages are out of sync. */
3005 rc = PGMTrap0eHandler(pVCpu, errCode, CPUMCTX2CORE(pCtx), (RTGCPTR)exitQualification);
3006 Log2(("PGMTrap0eHandler %RGv returned %Rrc\n", (RTGCPTR)pCtx->rip, VBOXSTRICTRC_VAL(rc)));
3007
3008 if (rc == VINF_SUCCESS)
3009 { /* We've successfully synced our shadow pages, so let's just continue execution. */
3010 Log2(("Shadow page fault at %RGv cr2=%RGv error code %x\n", (RTGCPTR)pCtx->rip, exitQualification ,errCode));
3011 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitShadowPF);
3012
3013 TRPMResetTrap(pVCpu);
3014 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3015 goto ResumeExecution;
3016 }
3017 else
3018 if (rc == VINF_EM_RAW_GUEST_TRAP)
3019 { /* A genuine pagefault.
3020 * Forward the trap to the guest by injecting the exception and resuming execution.
3021 */
3022 Log2(("Forward page fault to the guest\n"));
3023
3024 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestPF);
3025 /* The error code might have been changed. */
3026 errCode = TRPMGetErrorCode(pVCpu);
3027
3028 TRPMResetTrap(pVCpu);
3029
3030 /* Now we must update CR2. */
3031 pCtx->cr2 = exitQualification;
3032 rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
3033 AssertRC(rc2);
3034
3035 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3036 goto ResumeExecution;
3037 }
3038#ifdef VBOX_STRICT
3039 if (rc != VINF_EM_RAW_EMULATE_INSTR && rc != VINF_EM_RAW_EMULATE_IO_BLOCK)
3040 Log2(("PGMTrap0eHandler failed with %d\n", VBOXSTRICTRC_VAL(rc)));
3041#endif
3042 /* Need to go back to the recompiler to emulate the instruction. */
3043 TRPMResetTrap(pVCpu);
3044 break;
3045 }
3046
3047 case X86_XCPT_MF: /* Floating point exception. */
3048 {
3049 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestMF);
3050 if (!(pCtx->cr0 & X86_CR0_NE))
3051 {
3052 /* old style FPU error reporting needs some extra work. */
3053 /** @todo don't fall back to the recompiler, but do it manually. */
3054 rc = VINF_EM_RAW_EMULATE_INSTR;
3055 break;
3056 }
3057 Log(("Trap %x at %04X:%RGv\n", vector, pCtx->cs, (RTGCPTR)pCtx->rip));
3058 rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
3059 AssertRC(rc2);
3060
3061 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3062 goto ResumeExecution;
3063 }
3064
3065 case X86_XCPT_DB: /* Debug exception. */
3066 {
3067 uint64_t uDR6;
3068
3069 /* DR6, DR7.GD and IA32_DEBUGCTL.LBR are not updated yet.
3070 *
3071 * Exit qualification bits:
3072 * 3:0 B0-B3 which breakpoint condition was met
3073 * 12:4 Reserved (0)
3074 * 13 BD - debug register access detected
3075 * 14 BS - single step execution or branch taken
3076 * 63:15 Reserved (0)
3077 */
3078 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestDB);
3079
3080 /* Note that we don't support guest and host-initiated debugging at the same time. */
3081
3082 uDR6 = X86_DR6_INIT_VAL;
3083 uDR6 |= (exitQualification & (X86_DR6_B0|X86_DR6_B1|X86_DR6_B2|X86_DR6_B3|X86_DR6_BD|X86_DR6_BS));
3084 rc = DBGFRZTrap01Handler(pVM, pVCpu, CPUMCTX2CORE(pCtx), uDR6);
3085 if (rc == VINF_EM_RAW_GUEST_TRAP)
3086 {
3087 /* Update DR6 here. */
3088 pCtx->dr[6] = uDR6;
3089
3090 /* Resync DR6 if the debug state is active. */
3091 if (CPUMIsGuestDebugStateActive(pVCpu))
3092 ASMSetDR6(pCtx->dr[6]);
3093
3094 /* X86_DR7_GD will be cleared if drx accesses should be trapped inside the guest. */
3095 pCtx->dr[7] &= ~X86_DR7_GD;
3096
3097 /* Paranoia. */
3098 pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
3099 pCtx->dr[7] &= ~(RT_BIT(11) | RT_BIT(12) | RT_BIT(14) | RT_BIT(15)); /* must be zero */
3100 pCtx->dr[7] |= 0x400; /* must be one */
3101
3102 /* Resync DR7 */
3103 rc2 = VMXWriteVMCS64(VMX_VMCS64_GUEST_DR7, pCtx->dr[7]);
3104 AssertRC(rc2);
3105
3106 Log(("Trap %x (debug) at %RGv exit qualification %RX64 dr6=%x dr7=%x\n", vector, (RTGCPTR)pCtx->rip, exitQualification, (uint32_t)pCtx->dr[6], (uint32_t)pCtx->dr[7]));
3107 rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
3108 AssertRC(rc2);
3109
3110 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3111 goto ResumeExecution;
3112 }
3113 /* Return to ring 3 to deal with the debug exit code. */
3114 Log(("Debugger hardware BP at %04x:%RGv (rc=%Rrc)\n", pCtx->cs, pCtx->rip, VBOXSTRICTRC_VAL(rc)));
3115 break;
3116 }
3117
3118 case X86_XCPT_BP: /* Breakpoint. */
3119 {
3120 rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pCtx));
3121 if (rc == VINF_EM_RAW_GUEST_TRAP)
3122 {
3123 Log(("Guest #BP at %04x:%RGv\n", pCtx->cs, pCtx->rip));
3124 rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
3125 AssertRC(rc2);
3126 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3127 goto ResumeExecution;
3128 }
3129 if (rc == VINF_SUCCESS)
3130 {
3131 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3132 goto ResumeExecution;
3133 }
3134 Log(("Debugger BP at %04x:%RGv (rc=%Rrc)\n", pCtx->cs, pCtx->rip, VBOXSTRICTRC_VAL(rc)));
3135 break;
3136 }
3137
3138 case X86_XCPT_GP: /* General protection failure exception.*/
3139 {
3140 uint32_t cbOp;
3141 uint32_t cbSize;
3142 PDISCPUSTATE pDis = &pVCpu->hwaccm.s.DisState;
3143
3144 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestGP);
3145#ifdef VBOX_STRICT
3146 if ( !CPUMIsGuestInRealModeEx(pCtx)
3147 || !pVM->hwaccm.s.vmx.pRealModeTSS)
3148 {
3149 Log(("Trap %x at %04X:%RGv errorCode=%RGv\n", vector, pCtx->cs, (RTGCPTR)pCtx->rip, errCode));
3150 rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
3151 AssertRC(rc2);
3152 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3153 goto ResumeExecution;
3154 }
3155#endif
3156 Assert(CPUMIsGuestInRealModeEx(pCtx));
3157
3158 LogFlow(("Real mode X86_XCPT_GP instruction emulation at %x:%RGv\n", pCtx->cs, (RTGCPTR)pCtx->rip));
3159
3160 rc2 = EMInterpretDisasOne(pVM, pVCpu, CPUMCTX2CORE(pCtx), pDis, &cbOp);
3161 if (RT_SUCCESS(rc2))
3162 {
3163 bool fUpdateRIP = true;
3164
3165 rc = VINF_SUCCESS;
3166 Assert(cbOp == pDis->opsize);
3167 switch (pDis->pCurInstr->opcode)
3168 {
3169 case OP_CLI:
3170 pCtx->eflags.Bits.u1IF = 0;
3171 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCli);
3172 break;
3173
3174 case OP_STI:
3175 pCtx->eflags.Bits.u1IF = 1;
3176 EMSetInhibitInterruptsPC(pVCpu, pCtx->rip + pDis->opsize);
3177 Assert(VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
3178 rc2 = VMXWriteVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
3179 AssertRC(rc2);
3180 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitSti);
3181 break;
3182
3183 case OP_HLT:
3184 fUpdateRIP = false;
3185 rc = VINF_EM_HALT;
3186 pCtx->rip += pDis->opsize;
3187 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitHlt);
3188 break;
3189
3190 case OP_POPF:
3191 {
3192 RTGCPTR GCPtrStack;
3193 uint32_t cbParm;
3194 uint32_t uMask;
3195 X86EFLAGS eflags;
3196
3197 if (pDis->prefix & PREFIX_OPSIZE)
3198 {
3199 cbParm = 4;
3200 uMask = 0xffffffff;
3201 }
3202 else
3203 {
3204 cbParm = 2;
3205 uMask = 0xffff;
3206 }
3207
3208 rc2 = SELMToFlatEx(pVM, DIS_SELREG_SS, CPUMCTX2CORE(pCtx), pCtx->esp & uMask, 0, &GCPtrStack);
3209 if (RT_FAILURE(rc2))
3210 {
3211 rc = VERR_EM_INTERPRETER;
3212 break;
3213 }
3214 eflags.u = 0;
3215 rc2 = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &eflags.u, cbParm);
3216 if (RT_FAILURE(rc2))
3217 {
3218 rc = VERR_EM_INTERPRETER;
3219 break;
3220 }
3221 LogFlow(("POPF %x -> %RGv mask=%x\n", eflags.u, pCtx->rsp, uMask));
3222 pCtx->eflags.u = (pCtx->eflags.u & ~(X86_EFL_POPF_BITS & uMask)) | (eflags.u & X86_EFL_POPF_BITS & uMask);
3223 /* RF cleared when popped in real mode; see pushf description in AMD manual. */
3224 pCtx->eflags.Bits.u1RF = 0;
3225 pCtx->esp += cbParm;
3226 pCtx->esp &= uMask;
3227
3228 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitPopf);
3229 break;
3230 }
3231
3232 case OP_PUSHF:
3233 {
3234 RTGCPTR GCPtrStack;
3235 uint32_t cbParm;
3236 uint32_t uMask;
3237 X86EFLAGS eflags;
3238
3239 if (pDis->prefix & PREFIX_OPSIZE)
3240 {
3241 cbParm = 4;
3242 uMask = 0xffffffff;
3243 }
3244 else
3245 {
3246 cbParm = 2;
3247 uMask = 0xffff;
3248 }
3249
3250 rc2 = SELMToFlatEx(pVM, DIS_SELREG_SS, CPUMCTX2CORE(pCtx), (pCtx->esp - cbParm) & uMask, 0, &GCPtrStack);
3251 if (RT_FAILURE(rc2))
3252 {
3253 rc = VERR_EM_INTERPRETER;
3254 break;
3255 }
3256 eflags = pCtx->eflags;
3257 /* RF & VM cleared when pushed in real mode; see pushf description in AMD manual. */
3258 eflags.Bits.u1RF = 0;
3259 eflags.Bits.u1VM = 0;
3260
3261 rc2 = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &eflags.u, cbParm);
3262 if (RT_FAILURE(rc2))
3263 {
3264 rc = VERR_EM_INTERPRETER;
3265 break;
3266 }
3267 LogFlow(("PUSHF %x -> %RGv\n", eflags.u, GCPtrStack));
3268 pCtx->esp -= cbParm;
3269 pCtx->esp &= uMask;
3270 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitPushf);
3271 break;
3272 }
3273
3274 case OP_IRET:
3275 {
3276 RTGCPTR GCPtrStack;
3277 uint32_t uMask = 0xffff;
3278 uint16_t aIretFrame[3];
3279
3280 if (pDis->prefix & (PREFIX_OPSIZE | PREFIX_ADDRSIZE))
3281 {
3282 rc = VERR_EM_INTERPRETER;
3283 break;
3284 }
3285
3286 rc2 = SELMToFlatEx(pVM, DIS_SELREG_SS, CPUMCTX2CORE(pCtx), pCtx->esp & uMask, 0, &GCPtrStack);
3287 if (RT_FAILURE(rc2))
3288 {
3289 rc = VERR_EM_INTERPRETER;
3290 break;
3291 }
3292 rc2 = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
3293 if (RT_FAILURE(rc2))
3294 {
3295 rc = VERR_EM_INTERPRETER;
3296 break;
3297 }
3298 pCtx->ip = aIretFrame[0];
3299 pCtx->cs = aIretFrame[1];
3300 pCtx->csHid.u64Base = pCtx->cs << 4;
3301 pCtx->eflags.u = (pCtx->eflags.u & ~(X86_EFL_POPF_BITS & uMask)) | (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
3302 pCtx->sp += sizeof(aIretFrame);
3303
3304 LogFlow(("iret to %04x:%x\n", pCtx->cs, pCtx->ip));
3305 fUpdateRIP = false;
3306 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIret);
3307 break;
3308 }
3309
3310 case OP_INT:
3311 {
3312 uint32_t intInfo2;
3313
3314 LogFlow(("Realmode: INT %x\n", pDis->param1.parval & 0xff));
3315 intInfo2 = pDis->param1.parval & 0xff;
3316 intInfo2 |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
3317 intInfo2 |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
3318
3319 rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo2, cbOp, 0);
3320 AssertRC(VBOXSTRICTRC_VAL(rc));
3321 fUpdateRIP = false;
3322 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInt);
3323 break;
3324 }
3325
3326 case OP_INTO:
3327 {
3328 if (pCtx->eflags.Bits.u1OF)
3329 {
3330 uint32_t intInfo2;
3331
3332 LogFlow(("Realmode: INTO\n"));
3333 intInfo2 = X86_XCPT_OF;
3334 intInfo2 |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
3335 intInfo2 |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
3336
3337 rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo2, cbOp, 0);
3338 AssertRC(VBOXSTRICTRC_VAL(rc));
3339 fUpdateRIP = false;
3340 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInt);
3341 }
3342 break;
3343 }
3344
3345 case OP_INT3:
3346 {
3347 uint32_t intInfo2;
3348
3349 LogFlow(("Realmode: INT 3\n"));
3350 intInfo2 = 3;
3351 intInfo2 |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
3352 intInfo2 |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
3353
3354 rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo2, cbOp, 0);
3355 AssertRC(VBOXSTRICTRC_VAL(rc));
3356 fUpdateRIP = false;
3357 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInt);
3358 break;
3359 }
3360
3361 default:
3362 rc = EMInterpretInstructionCPU(pVM, pVCpu, pDis, CPUMCTX2CORE(pCtx), 0, EMCODETYPE_SUPERVISOR, &cbSize);
3363 break;
3364 }
3365
3366 if (rc == VINF_SUCCESS)
3367 {
3368 if (fUpdateRIP)
3369 pCtx->rip += cbOp; /* Move on to the next instruction. */
3370
3371 /* lidt, lgdt can end up here. In the future crx changes as well. Just reload the whole context to be done with it. */
3372 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_ALL;
3373
3374 /* Only resume if successful. */
3375 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3376 goto ResumeExecution;
3377 }
3378 }
3379 else
3380 rc = VERR_EM_INTERPRETER;
3381
3382 AssertMsg(rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_EM_HALT, ("Unexpected rc=%Rrc\n", VBOXSTRICTRC_VAL(rc)));
3383 break;
3384 }
3385
3386#ifdef VBOX_STRICT
3387 case X86_XCPT_XF: /* SIMD exception. */
3388 case X86_XCPT_DE: /* Divide error. */
3389 case X86_XCPT_UD: /* Unknown opcode exception. */
3390 case X86_XCPT_SS: /* Stack segment exception. */
3391 case X86_XCPT_NP: /* Segment not present exception. */
3392 {
3393 switch(vector)
3394 {
3395 case X86_XCPT_DE:
3396 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestDE);
3397 break;
3398 case X86_XCPT_UD:
3399 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestUD);
3400 break;
3401 case X86_XCPT_SS:
3402 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestSS);
3403 break;
3404 case X86_XCPT_NP:
3405 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestNP);
3406 break;
3407 }
3408
3409 Log(("Trap %x at %04X:%RGv\n", vector, pCtx->cs, (RTGCPTR)pCtx->rip));
3410 rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
3411 AssertRC(rc2);
3412
3413 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3414 goto ResumeExecution;
3415 }
3416#endif
3417 default:
3418 if ( CPUMIsGuestInRealModeEx(pCtx)
3419 && pVM->hwaccm.s.vmx.pRealModeTSS)
3420 {
3421 Log(("Real Mode Trap %x at %04x:%04X error code %x\n", vector, pCtx->cs, pCtx->eip, errCode));
3422 rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
3423 AssertRC(VBOXSTRICTRC_VAL(rc)); /* Strict RC check below. */
3424
3425 /* Go back to ring 3 in case of a triple fault. */
3426 if ( vector == X86_XCPT_DF
3427 && rc == VINF_EM_RESET)
3428 break;
3429
3430 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3431 goto ResumeExecution;
3432 }
3433 AssertMsgFailed(("Unexpected vm-exit caused by exception %x\n", vector));
3434 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
3435 break;
3436 } /* switch (vector) */
3437
3438 break;
3439
3440 default:
3441 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_CODE;
3442 AssertMsgFailed(("Unexpected interruption code %x\n", intInfo));
3443 break;
3444 }
3445
3446 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3447 break;
3448 }
3449
3450 case VMX_EXIT_EPT_VIOLATION: /* 48 EPT violation. An attempt to access memory with a guest-physical address was disallowed by the configuration of the EPT paging structures. */
3451 {
3452 RTGCPHYS GCPhys;
3453
3454 Assert(pVM->hwaccm.s.fNestedPaging);
3455
3456 rc2 = VMXReadVMCS64(VMX_VMCS_EXIT_PHYS_ADDR_FULL, &GCPhys);
3457 AssertRC(rc2);
3458 Assert(((exitQualification >> 7) & 3) != 2);
3459
3460 /* Determine the kind of violation. */
3461 errCode = 0;
3462 if (exitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
3463 errCode |= X86_TRAP_PF_ID;
3464
3465 if (exitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
3466 errCode |= X86_TRAP_PF_RW;
3467
3468 /* If the page is present, then it's a page level protection fault. */
3469 if (exitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
3470 {
3471 errCode |= X86_TRAP_PF_P;
3472 }
3473 else
3474 {
3475 /* Shortcut for APIC TPR reads and writes. */
3476 if ( (GCPhys & 0xfff) == 0x080
3477 && GCPhys > 0x1000000 /* to skip VGA frame buffer accesses */
3478 && fSetupTPRCaching
3479 && (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
3480 {
3481 RTGCPHYS GCPhysApicBase;
3482 PDMApicGetBase(pVM, &GCPhysApicBase); /* @todo cache this */
3483 GCPhysApicBase &= PAGE_BASE_GC_MASK;
3484 if (GCPhys == GCPhysApicBase + 0x80)
3485 {
3486 Log(("Enable VT-x virtual APIC access filtering\n"));
3487 rc2 = IOMMMIOMapMMIOHCPage(pVM, GCPhysApicBase, pVM->hwaccm.s.vmx.pAPICPhys, X86_PTE_RW | X86_PTE_P);
3488 AssertRC(rc2);
3489 }
3490 }
3491 }
3492 Log(("EPT Page fault %x at %RGp error code %x\n", (uint32_t)exitQualification, GCPhys, errCode));
3493
3494 /* GCPhys contains the guest physical address of the page fault. */
3495 TRPMAssertTrap(pVCpu, X86_XCPT_PF, TRPM_TRAP);
3496 TRPMSetErrorCode(pVCpu, errCode);
3497 TRPMSetFaultAddress(pVCpu, GCPhys);
3498
3499 /* Handle the pagefault trap for the nested shadow table. */
3500 rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, errCode, CPUMCTX2CORE(pCtx), GCPhys);
3501 Log2(("PGMR0Trap0eHandlerNestedPaging %RGv returned %Rrc\n", (RTGCPTR)pCtx->rip, VBOXSTRICTRC_VAL(rc)));
3502 if (rc == VINF_SUCCESS)
3503 { /* We've successfully synced our shadow pages, so let's just continue execution. */
3504 Log2(("Shadow page fault at %RGv cr2=%RGp error code %x\n", (RTGCPTR)pCtx->rip, exitQualification , errCode));
3505 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitReasonNPF);
3506
3507 TRPMResetTrap(pVCpu);
3508 goto ResumeExecution;
3509 }
3510
3511#ifdef VBOX_STRICT
3512 if (rc != VINF_EM_RAW_EMULATE_INSTR)
3513 LogFlow(("PGMTrap0eHandlerNestedPaging failed with %d\n", VBOXSTRICTRC_VAL(rc)));
3514#endif
3515 /* Need to go back to the recompiler to emulate the instruction. */
3516 TRPMResetTrap(pVCpu);
3517 break;
3518 }
3519
3520 case VMX_EXIT_EPT_MISCONFIG:
3521 {
3522 RTGCPHYS GCPhys;
3523
3524 Assert(pVM->hwaccm.s.fNestedPaging);
3525
3526 rc2 = VMXReadVMCS64(VMX_VMCS_EXIT_PHYS_ADDR_FULL, &GCPhys);
3527 AssertRC(rc2);
3528 Log(("VMX_EXIT_EPT_MISCONFIG for %RGp\n", GCPhys));
3529
3530 /* Shortcut for APIC TPR reads and writes. */
3531 if ( (GCPhys & 0xfff) == 0x080
3532 && GCPhys > 0x1000000 /* to skip VGA frame buffer accesses */
3533 && fSetupTPRCaching
3534 && (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
3535 {
3536 RTGCPHYS GCPhysApicBase;
3537 PDMApicGetBase(pVM, &GCPhysApicBase); /* @todo cache this */
3538 GCPhysApicBase &= PAGE_BASE_GC_MASK;
3539 if (GCPhys == GCPhysApicBase + 0x80)
3540 {
3541 Log(("Enable VT-x virtual APIC access filtering\n"));
3542 rc2 = IOMMMIOMapMMIOHCPage(pVM, GCPhysApicBase, pVM->hwaccm.s.vmx.pAPICPhys, X86_PTE_RW | X86_PTE_P);
3543 AssertRC(rc2);
3544 }
3545 }
3546
3547 rc = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pCtx), GCPhys, UINT32_MAX);
3548 if (rc == VINF_SUCCESS)
3549 {
3550 Log2(("PGMR0Trap0eHandlerNPMisconfig(,,,%RGp) at %RGv -> resume\n", GCPhys, (RTGCPTR)pCtx->rip));
3551 goto ResumeExecution;
3552 }
3553
3554 Log2(("PGMR0Trap0eHandlerNPMisconfig(,,,%RGp) at %RGv -> %Rrc\n", GCPhys, (RTGCPTR)pCtx->rip, VBOXSTRICTRC_VAL(rc)));
3555 break;
3556 }
3557
3558 case VMX_EXIT_IRQ_WINDOW: /* 7 Interrupt window. */
3559 /* Clear VM-exit on IF=1 change. */
3560 LogFlow(("VMX_EXIT_IRQ_WINDOW %RGv pending=%d IF=%d\n", (RTGCPTR)pCtx->rip, VMCPU_FF_ISPENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC|VMCPU_FF_INTERRUPT_PIC)), pCtx->eflags.Bits.u1IF));
3561 pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_IRQ_WINDOW_EXIT;
3562 rc2 = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
3563 AssertRC(rc2);
3564 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIrqWindow);
3565 goto ResumeExecution; /* we check for pending guest interrupts there */
3566
3567 case VMX_EXIT_WBINVD: /* 54 Guest software attempted to execute WBINVD. (conditional) */
3568 case VMX_EXIT_INVD: /* 13 Guest software attempted to execute INVD. (unconditional) */
3569 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInvd);
3570 /* Skip instruction and continue directly. */
3571 pCtx->rip += cbInstr;
3572 /* Continue execution.*/
3573 goto ResumeExecution;
3574
3575 case VMX_EXIT_CPUID: /* 10 Guest software attempted to execute CPUID. */
3576 {
3577 Log2(("VMX: Cpuid %x\n", pCtx->eax));
3578 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCpuid);
3579 rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pCtx));
3580 if (rc == VINF_SUCCESS)
3581 {
3582 /* Update EIP and continue execution. */
3583 Assert(cbInstr == 2);
3584 pCtx->rip += cbInstr;
3585 goto ResumeExecution;
3586 }
3587 AssertMsgFailed(("EMU: cpuid failed with %Rrc\n", VBOXSTRICTRC_VAL(rc)));
3588 rc = VINF_EM_RAW_EMULATE_INSTR;
3589 break;
3590 }
3591
3592 case VMX_EXIT_RDPMC: /* 15 Guest software attempted to execute RDPMC. */
3593 {
3594 Log2(("VMX: Rdpmc %x\n", pCtx->ecx));
3595 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitRdpmc);
3596 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pCtx));
3597 if (rc == VINF_SUCCESS)
3598 {
3599 /* Update EIP and continue execution. */
3600 Assert(cbInstr == 2);
3601 pCtx->rip += cbInstr;
3602 goto ResumeExecution;
3603 }
3604 rc = VINF_EM_RAW_EMULATE_INSTR;
3605 break;
3606 }
3607
3608 case VMX_EXIT_RDTSC: /* 16 Guest software attempted to execute RDTSC. */
3609 {
3610 Log2(("VMX: Rdtsc\n"));
3611 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitRdtsc);
3612 rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pCtx));
3613 if (rc == VINF_SUCCESS)
3614 {
3615 /* Update EIP and continue execution. */
3616 Assert(cbInstr == 2);
3617 pCtx->rip += cbInstr;
3618 goto ResumeExecution;
3619 }
3620 rc = VINF_EM_RAW_EMULATE_INSTR;
3621 break;
3622 }
3623
3624 case VMX_EXIT_INVPG: /* 14 Guest software attempted to execute INVPG. */
3625 {
3626 Log2(("VMX: invlpg\n"));
3627 Assert(!pVM->hwaccm.s.fNestedPaging);
3628
3629 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInvpg);
3630 rc = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pCtx), exitQualification);
3631 if (rc == VINF_SUCCESS)
3632 {
3633 /* Update EIP and continue execution. */
3634 pCtx->rip += cbInstr;
3635 goto ResumeExecution;
3636 }
3637 AssertMsg(rc == VERR_EM_INTERPRETER, ("EMU: invlpg %RGv failed with %Rrc\n", exitQualification, VBOXSTRICTRC_VAL(rc)));
3638 break;
3639 }
3640
3641 case VMX_EXIT_MONITOR: /* 39 Guest software attempted to execute MONITOR. */
3642 {
3643 Log2(("VMX: monitor\n"));
3644
3645 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitMonitor);
3646 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pCtx));
3647 if (rc == VINF_SUCCESS)
3648 {
3649 /* Update EIP and continue execution. */
3650 pCtx->rip += cbInstr;
3651 goto ResumeExecution;
3652 }
3653 AssertMsg(rc == VERR_EM_INTERPRETER, ("EMU: monitor failed with %Rrc\n", VBOXSTRICTRC_VAL(rc)));
3654 break;
3655 }
3656
3657 case VMX_EXIT_WRMSR: /* 32 WRMSR. Guest software attempted to execute WRMSR. */
3658 /* When an interrupt is pending, we'll let MSR_K8_LSTAR writes fault in our TPR patch code. */
3659 if ( pVM->hwaccm.s.fTPRPatchingActive
3660 && pCtx->ecx == MSR_K8_LSTAR)
3661 {
3662 Assert(!CPUMIsGuestInLongModeEx(pCtx));
3663 if ((pCtx->eax & 0xff) != u8LastTPR)
3664 {
3665 Log(("VMX: Faulting MSR_K8_LSTAR write with new TPR value %x\n", pCtx->eax & 0xff));
3666
3667 /* Our patch code uses LSTAR for TPR caching. */
3668 rc2 = PDMApicSetTPR(pVCpu, pCtx->eax & 0xff);
3669 AssertRC(rc2);
3670 }
3671
3672 /* Skip the instruction and continue. */
3673 pCtx->rip += cbInstr; /* wrmsr = [0F 30] */
3674
3675 /* Only resume if successful. */
3676 goto ResumeExecution;
3677 }
3678 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_MSR;
3679 /* no break */
3680 case VMX_EXIT_RDMSR: /* 31 RDMSR. Guest software attempted to execute RDMSR. */
3681 {
3682 uint32_t cbSize;
3683
3684 STAM_COUNTER_INC((exitReason == VMX_EXIT_RDMSR) ? &pVCpu->hwaccm.s.StatExitRdmsr : &pVCpu->hwaccm.s.StatExitWrmsr);
3685
3686 /* Note: the intel manual claims there's a REX version of RDMSR that's slightly different, so we play safe by completely disassembling the instruction. */
3687 Log2(("VMX: %s\n", (exitReason == VMX_EXIT_RDMSR) ? "rdmsr" : "wrmsr"));
3688 rc = EMInterpretInstruction(pVM, pVCpu, CPUMCTX2CORE(pCtx), 0, &cbSize);
3689 if (rc == VINF_SUCCESS)
3690 {
3691 /* EIP has been updated already. */
3692
3693 /* Only resume if successful. */
3694 goto ResumeExecution;
3695 }
3696 AssertMsg(rc == VERR_EM_INTERPRETER, ("EMU: %s failed with %Rrc\n", (exitReason == VMX_EXIT_RDMSR) ? "rdmsr" : "wrmsr", VBOXSTRICTRC_VAL(rc)));
3697 break;
3698 }
3699
3700 case VMX_EXIT_CRX_MOVE: /* 28 Control-register accesses. */
3701 {
3702 STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatExit2Sub2, y2);
3703
3704 switch (VMX_EXIT_QUALIFICATION_CRX_ACCESS(exitQualification))
3705 {
3706 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE:
3707 Log2(("VMX: %RGv mov cr%d, x\n", (RTGCPTR)pCtx->rip, VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification)));
3708 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification)]);
3709 rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
3710 VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification),
3711 VMX_EXIT_QUALIFICATION_CRX_GENREG(exitQualification));
3712
3713 switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification))
3714 {
3715 case 0:
3716 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_CR0 | HWACCM_CHANGED_GUEST_CR3;
3717 break;
3718 case 2:
3719 break;
3720 case 3:
3721 Assert(!pVM->hwaccm.s.fNestedPaging || !CPUMIsGuestInPagedProtectedModeEx(pCtx));
3722 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_CR3;
3723 break;
3724 case 4:
3725 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_CR4;
3726 break;
3727 case 8:
3728 /* CR8 contains the APIC TPR */
3729 Assert(!(pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
3730 break;
3731
3732 default:
3733 AssertFailed();
3734 break;
3735 }
3736 break;
3737
3738 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ:
3739 Log2(("VMX: mov x, crx\n"));
3740 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification)]);
3741
3742 Assert(!pVM->hwaccm.s.fNestedPaging || !CPUMIsGuestInPagedProtectedModeEx(pCtx) || VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification) != USE_REG_CR3);
3743
3744 /* CR8 reads only cause an exit when the TPR shadow feature isn't present. */
3745 Assert(VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification) != 8 || !(pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
3746
3747 rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
3748 VMX_EXIT_QUALIFICATION_CRX_GENREG(exitQualification),
3749 VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification));
3750 break;
3751
3752 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS:
3753 Log2(("VMX: clts\n"));
3754 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCLTS);
3755 rc = EMInterpretCLTS(pVM, pVCpu);
3756 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_CR0;
3757 break;
3758
3759 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW:
3760 Log2(("VMX: lmsw %x\n", VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(exitQualification)));
3761 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitLMSW);
3762 rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(exitQualification));
3763 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_CR0;
3764 break;
3765 }
3766
3767 /* Update EIP if no error occurred. */
3768 if (RT_SUCCESS(rc))
3769 pCtx->rip += cbInstr;
3770
3771 if (rc == VINF_SUCCESS)
3772 {
3773 /* Only resume if successful. */
3774 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub2, y2);
3775 goto ResumeExecution;
3776 }
3777 Assert(rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_PGM_SYNC_CR3);
3778 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub2, y2);
3779 break;
3780 }
3781
3782 case VMX_EXIT_DRX_MOVE: /* 29 Debug-register accesses. */
3783 {
3784 if ( !DBGFIsStepping(pVCpu)
3785 && !CPUMIsHyperDebugStateActive(pVCpu))
3786 {
3787 /* Disable drx move intercepts. */
3788 pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
3789 rc2 = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
3790 AssertRC(rc2);
3791
3792 /* Save the host and load the guest debug state. */
3793 rc2 = CPUMR0LoadGuestDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
3794 AssertRC(rc2);
3795
3796#ifdef LOG_ENABLED
3797 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
3798 Log(("VMX_EXIT_DRX_MOVE: write DR%d genreg %d\n", VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification), VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification)));
3799 else
3800 Log(("VMX_EXIT_DRX_MOVE: read DR%d\n", VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification)));
3801#endif
3802
3803#ifdef VBOX_WITH_STATISTICS
3804 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatDRxContextSwitch);
3805 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
3806 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxWrite);
3807 else
3808 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxRead);
3809#endif
3810
3811 goto ResumeExecution;
3812 }
3813
3814 /** @todo clear VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT after the first time and restore drx registers afterwards */
3815 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
3816 {
3817 Log2(("VMX: mov drx%d, genreg%d\n", VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification), VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification)));
3818 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxWrite);
3819 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
3820 VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification),
3821 VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification));
3822 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_DEBUG;
3823 Log2(("DR7=%08x\n", pCtx->dr[7]));
3824 }
3825 else
3826 {
3827 Log2(("VMX: mov x, drx\n"));
3828 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxRead);
3829 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
3830 VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification),
3831 VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification));
3832 }
3833 /* Update EIP if no error occurred. */
3834 if (RT_SUCCESS(rc))
3835 pCtx->rip += cbInstr;
3836
3837 if (rc == VINF_SUCCESS)
3838 {
3839 /* Only resume if successful. */
3840 goto ResumeExecution;
3841 }
3842 Assert(rc == VERR_EM_INTERPRETER);
3843 break;
3844 }
3845
3846 /* Note: We'll get a #GP if the IO instruction isn't allowed (IOPL or TSS bitmap); no need to double check. */
3847 case VMX_EXIT_PORT_IO: /* 30 I/O instruction. */
3848 {
3849 STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3850 uint32_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(exitQualification);
3851 uint32_t uPort;
3852 bool fIOWrite = (VMX_EXIT_QUALIFICATION_IO_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
3853
3854 /** @todo necessary to make the distinction? */
3855 if (VMX_EXIT_QUALIFICATION_IO_ENCODING(exitQualification) == VMX_EXIT_QUALIFICATION_IO_ENCODING_DX)
3856 {
3857 uPort = pCtx->edx & 0xffff;
3858 }
3859 else
3860 uPort = VMX_EXIT_QUALIFICATION_IO_PORT(exitQualification); /* Immediate encoding. */
3861
3862 /* paranoia */
3863 if (RT_UNLIKELY(uIOWidth == 2 || uIOWidth >= 4))
3864 {
3865 rc = fIOWrite ? VINF_IOM_HC_IOPORT_WRITE : VINF_IOM_HC_IOPORT_READ;
3866 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3867 break;
3868 }
3869
3870 uint32_t cbSize = g_aIOSize[uIOWidth];
3871
3872 if (VMX_EXIT_QUALIFICATION_IO_STRING(exitQualification))
3873 {
3874 /* ins/outs */
3875 PDISCPUSTATE pDis = &pVCpu->hwaccm.s.DisState;
3876
3877 /* Disassemble manually to deal with segment prefixes. */
3878 /** @todo VMX_VMCS_EXIT_GUEST_LINEAR_ADDR contains the flat pointer operand of the instruction. */
3879 /** @todo VMX_VMCS32_RO_EXIT_INSTR_INFO also contains segment prefix info. */
3880 rc2 = EMInterpretDisasOne(pVM, pVCpu, CPUMCTX2CORE(pCtx), pDis, NULL);
3881 if (RT_SUCCESS(rc))
3882 {
3883 if (fIOWrite)
3884 {
3885 Log2(("IOMInterpretOUTSEx %RGv %x size=%d\n", (RTGCPTR)pCtx->rip, uPort, cbSize));
3886 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIOStringWrite);
3887 rc = IOMInterpretOUTSEx(pVM, CPUMCTX2CORE(pCtx), uPort, pDis->prefix, pDis->addrmode, cbSize);
3888 }
3889 else
3890 {
3891 Log2(("IOMInterpretINSEx %RGv %x size=%d\n", (RTGCPTR)pCtx->rip, uPort, cbSize));
3892 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIOStringRead);
3893 rc = IOMInterpretINSEx(pVM, CPUMCTX2CORE(pCtx), uPort, pDis->prefix, pDis->addrmode, cbSize);
3894 }
3895 }
3896 else
3897 rc = VINF_EM_RAW_EMULATE_INSTR;
3898 }
3899 else
3900 {
3901 /* normal in/out */
3902 uint32_t uAndVal = g_aIOOpAnd[uIOWidth];
3903
3904 Assert(!VMX_EXIT_QUALIFICATION_IO_REP(exitQualification));
3905
3906 if (fIOWrite)
3907 {
3908 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIOWrite);
3909 rc = IOMIOPortWrite(pVM, uPort, pCtx->eax & uAndVal, cbSize);
3910 if (rc == VINF_IOM_HC_IOPORT_WRITE)
3911 HWACCMR0SavePendingIOPortWrite(pVCpu, pCtx->rip, pCtx->rip + cbInstr, uPort, uAndVal, cbSize);
3912 }
3913 else
3914 {
3915 uint32_t u32Val = 0;
3916
3917 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIORead);
3918 rc = IOMIOPortRead(pVM, uPort, &u32Val, cbSize);
3919 if (IOM_SUCCESS(rc))
3920 {
3921 /* Write back to the EAX register. */
3922 pCtx->eax = (pCtx->eax & ~uAndVal) | (u32Val & uAndVal);
3923 }
3924 else
3925 if (rc == VINF_IOM_HC_IOPORT_READ)
3926 HWACCMR0SavePendingIOPortRead(pVCpu, pCtx->rip, pCtx->rip + cbInstr, uPort, uAndVal, cbSize);
3927 }
3928 }
3929 /*
3930 * Handled the I/O return codes.
3931 * (The unhandled cases end up with rc == VINF_EM_RAW_EMULATE_INSTR.)
3932 */
3933 if (IOM_SUCCESS(rc))
3934 {
3935 /* Update EIP and continue execution. */
3936 pCtx->rip += cbInstr;
3937 if (RT_LIKELY(rc == VINF_SUCCESS))
3938 {
3939 /* If any IO breakpoints are armed, then we should check if a debug trap needs to be generated. */
3940 if (pCtx->dr[7] & X86_DR7_ENABLED_MASK)
3941 {
3942 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatDRxIOCheck);
3943 for (unsigned i=0;i<4;i++)
3944 {
3945 unsigned uBPLen = g_aIOSize[X86_DR7_GET_LEN(pCtx->dr[7], i)];
3946
3947 if ( (uPort >= pCtx->dr[i] && uPort < pCtx->dr[i] + uBPLen)
3948 && (pCtx->dr[7] & (X86_DR7_L(i) | X86_DR7_G(i)))
3949 && (pCtx->dr[7] & X86_DR7_RW(i, X86_DR7_RW_IO)) == X86_DR7_RW(i, X86_DR7_RW_IO))
3950 {
3951 uint64_t uDR6;
3952
3953 Assert(CPUMIsGuestDebugStateActive(pVCpu));
3954
3955 uDR6 = ASMGetDR6();
3956
3957 /* Clear all breakpoint status flags and set the one we just hit. */
3958 uDR6 &= ~(X86_DR6_B0|X86_DR6_B1|X86_DR6_B2|X86_DR6_B3);
3959 uDR6 |= (uint64_t)RT_BIT(i);
3960
3961 /* Note: AMD64 Architecture Programmer's Manual 13.1:
3962 * Bits 15:13 of the DR6 register is never cleared by the processor and must be cleared by software after
3963 * the contents have been read.
3964 */
3965 ASMSetDR6(uDR6);
3966
3967 /* X86_DR7_GD will be cleared if drx accesses should be trapped inside the guest. */
3968 pCtx->dr[7] &= ~X86_DR7_GD;
3969
3970 /* Paranoia. */
3971 pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
3972 pCtx->dr[7] &= ~(RT_BIT(11) | RT_BIT(12) | RT_BIT(14) | RT_BIT(15)); /* must be zero */
3973 pCtx->dr[7] |= 0x400; /* must be one */
3974
3975 /* Resync DR7 */
3976 rc2 = VMXWriteVMCS64(VMX_VMCS64_GUEST_DR7, pCtx->dr[7]);
3977 AssertRC(rc2);
3978
3979 /* Construct inject info. */
3980 intInfo = X86_XCPT_DB;
3981 intInfo |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
3982 intInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
3983
3984 Log(("Inject IO debug trap at %RGv\n", (RTGCPTR)pCtx->rip));
3985 rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), 0, 0);
3986 AssertRC(rc2);
3987
3988 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3989 goto ResumeExecution;
3990 }
3991 }
3992 }
3993 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3994 goto ResumeExecution;
3995 }
3996 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3997 break;
3998 }
3999
4000#ifdef VBOX_STRICT
4001 if (rc == VINF_IOM_HC_IOPORT_READ)
4002 Assert(!fIOWrite);
4003 else if (rc == VINF_IOM_HC_IOPORT_WRITE)
4004 Assert(fIOWrite);
4005 else
4006 AssertMsg(RT_FAILURE(rc) || rc == VINF_EM_RAW_EMULATE_INSTR || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rc)));
4007#endif
4008 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
4009 break;
4010 }
4011
4012 case VMX_EXIT_TPR: /* 43 TPR below threshold. Guest software executed MOV to CR8. */
4013 LogFlow(("VMX_EXIT_TPR\n"));
4014 /* RIP is already set to the next instruction and the TPR has been synced back. Just resume. */
4015 goto ResumeExecution;
4016
4017 case VMX_EXIT_APIC_ACCESS: /* 44 APIC access. Guest software attempted to access memory at a physical address on the APIC-access page. */
4018 {
4019 LogFlow(("VMX_EXIT_APIC_ACCESS\n"));
4020 unsigned uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(exitQualification);
4021
4022 switch(uAccessType)
4023 {
4024 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
4025 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
4026 {
4027 RTGCPHYS GCPhys;
4028 PDMApicGetBase(pVM, &GCPhys);
4029 GCPhys &= PAGE_BASE_GC_MASK;
4030 GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(exitQualification);
4031
4032 LogFlow(("Apic access at %RGp\n", GCPhys));
4033 rc = IOMMMIOPhysHandler(pVM, (uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW, CPUMCTX2CORE(pCtx), GCPhys);
4034 if (rc == VINF_SUCCESS)
4035 goto ResumeExecution; /* rip already updated */
4036 break;
4037 }
4038
4039 default:
4040 rc = VINF_EM_RAW_EMULATE_INSTR;
4041 break;
4042 }
4043 break;
4044 }
4045
4046 case VMX_EXIT_PREEMPTION_TIMER: /* 52 VMX-preemption timer expired. The preemption timer counted down to zero. */
4047 if (!TMTimerPollBool(pVM, pVCpu))
4048 goto ResumeExecution;
4049 rc = VINF_EM_RAW_TIMER_PENDING;
4050 break;
4051
4052 default:
4053 /* The rest is handled after syncing the entire CPU state. */
4054 break;
4055 }
4056
4057 /* Note: the guest state isn't entirely synced back at this stage. */
4058
4059 /* Investigate why there was a VM-exit. (part 2) */
4060 switch (exitReason)
4061 {
4062 case VMX_EXIT_EXCEPTION: /* 0 Exception or non-maskable interrupt (NMI). */
4063 case VMX_EXIT_EXTERNAL_IRQ: /* 1 External interrupt. */
4064 case VMX_EXIT_EPT_VIOLATION:
4065 case VMX_EXIT_EPT_MISCONFIG: /* 49 EPT misconfig is used by the PGM/MMIO optimizations. */
4066 case VMX_EXIT_PREEMPTION_TIMER: /* 52 VMX-preemption timer expired. The preemption timer counted down to zero. */
4067 /* Already handled above. */
4068 break;
4069
4070 case VMX_EXIT_TRIPLE_FAULT: /* 2 Triple fault. */
4071 rc = VINF_EM_RESET; /* Triple fault equals a reset. */
4072 break;
4073
4074 case VMX_EXIT_INIT_SIGNAL: /* 3 INIT signal. */
4075 case VMX_EXIT_SIPI: /* 4 Start-up IPI (SIPI). */
4076 rc = VINF_EM_RAW_INTERRUPT;
4077 AssertFailed(); /* Can't happen. Yet. */
4078 break;
4079
4080 case VMX_EXIT_IO_SMI_IRQ: /* 5 I/O system-management interrupt (SMI). */
4081 case VMX_EXIT_SMI_IRQ: /* 6 Other SMI. */
4082 rc = VINF_EM_RAW_INTERRUPT;
4083 AssertFailed(); /* Can't happen afaik. */
4084 break;
4085
4086 case VMX_EXIT_TASK_SWITCH: /* 9 Task switch: too complicated to emulate, so fall back to the recompiler */
4087 Log(("VMX_EXIT_TASK_SWITCH: exit=%RX64\n", exitQualification));
4088 if ( (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(exitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
4089 && pVCpu->hwaccm.s.Event.fPending)
4090 {
4091 /* Caused by an injected interrupt. */
4092 pVCpu->hwaccm.s.Event.fPending = false;
4093
4094 Log(("VMX_EXIT_TASK_SWITCH: reassert trap %d\n", VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVCpu->hwaccm.s.Event.intInfo)));
4095 Assert(!VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVCpu->hwaccm.s.Event.intInfo));
4096 rc2 = TRPMAssertTrap(pVCpu, VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVCpu->hwaccm.s.Event.intInfo), TRPM_HARDWARE_INT);
4097 AssertRC(rc2);
4098 }
4099 /* else Exceptions and software interrupts can just be restarted. */
4100 rc = VERR_EM_INTERPRETER;
4101 break;
4102
4103 case VMX_EXIT_HLT: /* 12 Guest software attempted to execute HLT. */
4104 /** Check if external interrupts are pending; if so, don't switch back. */
4105 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitHlt);
4106 pCtx->rip++; /* skip hlt */
4107 if (EMShouldContinueAfterHalt(pVCpu, pCtx))
4108 goto ResumeExecution;
4109
4110 rc = VINF_EM_HALT;
4111 break;
4112
4113 case VMX_EXIT_MWAIT: /* 36 Guest software executed MWAIT. */
4114 Log2(("VMX: mwait\n"));
4115 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitMwait);
4116 rc = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pCtx));
4117 if ( rc == VINF_EM_HALT
4118 || rc == VINF_SUCCESS)
4119 {
4120 /* Update EIP and continue execution. */
4121 pCtx->rip += cbInstr;
4122
4123 /** Check if external interrupts are pending; if so, don't switch back. */
4124 if ( rc == VINF_SUCCESS
4125 || ( rc == VINF_EM_HALT
4126 && EMShouldContinueAfterHalt(pVCpu, pCtx))
4127 )
4128 goto ResumeExecution;
4129 }
4130 AssertMsg(rc == VERR_EM_INTERPRETER || rc == VINF_EM_HALT, ("EMU: mwait failed with %Rrc\n", VBOXSTRICTRC_VAL(rc)));
4131 break;
4132
4133 case VMX_EXIT_RSM: /* 17 Guest software attempted to execute RSM in SMM. */
4134 AssertFailed(); /* can't happen. */
4135 rc = VERR_EM_INTERPRETER;
4136 break;
4137
4138 case VMX_EXIT_VMCALL: /* 18 Guest software executed VMCALL. */
4139 case VMX_EXIT_VMCLEAR: /* 19 Guest software executed VMCLEAR. */
4140 case VMX_EXIT_VMLAUNCH: /* 20 Guest software executed VMLAUNCH. */
4141 case VMX_EXIT_VMPTRLD: /* 21 Guest software executed VMPTRLD. */
4142 case VMX_EXIT_VMPTRST: /* 22 Guest software executed VMPTRST. */
4143 case VMX_EXIT_VMREAD: /* 23 Guest software executed VMREAD. */
4144 case VMX_EXIT_VMRESUME: /* 24 Guest software executed VMRESUME. */
4145 case VMX_EXIT_VMWRITE: /* 25 Guest software executed VMWRITE. */
4146 case VMX_EXIT_VMXOFF: /* 26 Guest software executed VMXOFF. */
4147 case VMX_EXIT_VMXON: /* 27 Guest software executed VMXON. */
4148 /** @todo inject #UD immediately */
4149 rc = VERR_EM_INTERPRETER;
4150 break;
4151
4152 case VMX_EXIT_CPUID: /* 10 Guest software attempted to execute CPUID. */
4153 case VMX_EXIT_RDTSC: /* 16 Guest software attempted to execute RDTSC. */
4154 case VMX_EXIT_INVPG: /* 14 Guest software attempted to execute INVPG. */
4155 case VMX_EXIT_CRX_MOVE: /* 28 Control-register accesses. */
4156 case VMX_EXIT_DRX_MOVE: /* 29 Debug-register accesses. */
4157 case VMX_EXIT_PORT_IO: /* 30 I/O instruction. */
4158 case VMX_EXIT_RDPMC: /* 15 Guest software attempted to execute RDPMC. */
4159 /* already handled above */
4160 AssertMsg( rc == VINF_PGM_CHANGE_MODE
4161 || rc == VINF_EM_RAW_INTERRUPT
4162 || rc == VERR_EM_INTERPRETER
4163 || rc == VINF_EM_RAW_EMULATE_INSTR
4164 || rc == VINF_PGM_SYNC_CR3
4165 || rc == VINF_IOM_HC_IOPORT_READ
4166 || rc == VINF_IOM_HC_IOPORT_WRITE
4167 || rc == VINF_EM_RAW_GUEST_TRAP
4168 || rc == VINF_TRPM_XCPT_DISPATCHED
4169 || rc == VINF_EM_RESCHEDULE_REM,
4170 ("rc = %d\n", VBOXSTRICTRC_VAL(rc)));
4171 break;
4172
4173 case VMX_EXIT_TPR: /* 43 TPR below threshold. Guest software executed MOV to CR8. */
4174 case VMX_EXIT_APIC_ACCESS: /* 44 APIC access. Guest software attempted to access memory at a physical address on the APIC-access page. */
4175 case VMX_EXIT_RDMSR: /* 31 RDMSR. Guest software attempted to execute RDMSR. */
4176 case VMX_EXIT_WRMSR: /* 32 WRMSR. Guest software attempted to execute WRMSR. */
4177 case VMX_EXIT_PAUSE: /* 40 Guest software attempted to execute PAUSE. */
4178 case VMX_EXIT_MONITOR: /* 39 Guest software attempted to execute MONITOR. */
4179 /* Note: If we decide to emulate them here, then we must sync the MSRs that could have been changed (sysenter, fs/gs base)!!! */
4180 rc = VERR_EM_INTERPRETER;
4181 break;
4182
4183 case VMX_EXIT_IRQ_WINDOW: /* 7 Interrupt window. */
4184 Assert(rc == VINF_EM_RAW_INTERRUPT);
4185 break;
4186
4187 case VMX_EXIT_ERR_INVALID_GUEST_STATE: /* 33 VM-entry failure due to invalid guest state. */
4188 {
4189#ifdef VBOX_STRICT
4190 RTCCUINTREG val2 = 0;
4191
4192 Log(("VMX_EXIT_ERR_INVALID_GUEST_STATE\n"));
4193
4194 VMXReadVMCS(VMX_VMCS64_GUEST_RIP, &val2);
4195 Log(("Old eip %RGv new %RGv\n", (RTGCPTR)pCtx->rip, (RTGCPTR)val2));
4196
4197 VMXReadVMCS(VMX_VMCS64_GUEST_CR0, &val2);
4198 Log(("VMX_VMCS_GUEST_CR0 %RX64\n", (uint64_t)val2));
4199
4200 VMXReadVMCS(VMX_VMCS64_GUEST_CR3, &val2);
4201 Log(("VMX_VMCS_GUEST_CR3 %RX64\n", (uint64_t)val2));
4202
4203 VMXReadVMCS(VMX_VMCS64_GUEST_CR4, &val2);
4204 Log(("VMX_VMCS_GUEST_CR4 %RX64\n", (uint64_t)val2));
4205
4206 VMXReadVMCS(VMX_VMCS_GUEST_RFLAGS, &val2);
4207 Log(("VMX_VMCS_GUEST_RFLAGS %08x\n", val2));
4208
4209 VMX_LOG_SELREG(CS, "CS", val2);
4210 VMX_LOG_SELREG(DS, "DS", val2);
4211 VMX_LOG_SELREG(ES, "ES", val2);
4212 VMX_LOG_SELREG(FS, "FS", val2);
4213 VMX_LOG_SELREG(GS, "GS", val2);
4214 VMX_LOG_SELREG(SS, "SS", val2);
4215 VMX_LOG_SELREG(TR, "TR", val2);
4216 VMX_LOG_SELREG(LDTR, "LDTR", val2);
4217
4218 VMXReadVMCS(VMX_VMCS64_GUEST_GDTR_BASE, &val2);
4219 Log(("VMX_VMCS_GUEST_GDTR_BASE %RX64\n", (uint64_t)val2));
4220 VMXReadVMCS(VMX_VMCS64_GUEST_IDTR_BASE, &val2);
4221 Log(("VMX_VMCS_GUEST_IDTR_BASE %RX64\n", (uint64_t)val2));
4222#endif /* VBOX_STRICT */
4223 rc = VERR_VMX_INVALID_GUEST_STATE;
4224 break;
4225 }
4226
4227 case VMX_EXIT_ERR_MSR_LOAD: /* 34 VM-entry failure due to MSR loading. */
4228 case VMX_EXIT_ERR_MACHINE_CHECK: /* 41 VM-entry failure due to machine-check. */
4229 default:
4230 rc = VERR_VMX_UNEXPECTED_EXIT_CODE;
4231 AssertMsgFailed(("Unexpected exit code %d\n", exitReason)); /* Can't happen. */
4232 break;
4233
4234 }
4235end:
4236
4237 /* We now going back to ring-3, so clear the action flag. */
4238 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
4239
4240 /* Signal changes for the recompiler. */
4241 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR | CPUM_CHANGED_LDTR | CPUM_CHANGED_GDTR | CPUM_CHANGED_IDTR | CPUM_CHANGED_TR | CPUM_CHANGED_HIDDEN_SEL_REGS);
4242
4243 /* If we executed vmlaunch/vmresume and an external irq was pending, then we don't have to do a full sync the next time. */
4244 if ( exitReason == VMX_EXIT_EXTERNAL_IRQ
4245 && !VMX_EXIT_INTERRUPTION_INFO_VALID(intInfo))
4246 {
4247 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatPendingHostIrq);
4248 /* On the next entry we'll only sync the host context. */
4249 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_HOST_CONTEXT;
4250 }
4251 else
4252 {
4253 /* On the next entry we'll sync everything. */
4254 /** @todo we can do better than this */
4255 /* Not in the VINF_PGM_CHANGE_MODE though! */
4256 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_ALL;
4257 }
4258
4259 /* translate into a less severe return code */
4260 if (rc == VERR_EM_INTERPRETER)
4261 rc = VINF_EM_RAW_EMULATE_INSTR;
4262 else
4263 /* Try to extract more information about what might have gone wrong here. */
4264 if (rc == VERR_VMX_INVALID_VMCS_PTR)
4265 {
4266 VMXGetActivateVMCS(&pVCpu->hwaccm.s.vmx.lasterror.u64VMCSPhys);
4267 pVCpu->hwaccm.s.vmx.lasterror.ulVMCSRevision = *(uint32_t *)pVCpu->hwaccm.s.vmx.pvVMCS;
4268 pVCpu->hwaccm.s.vmx.lasterror.idEnteredCpu = pVCpu->hwaccm.s.idEnteredCpu;
4269 pVCpu->hwaccm.s.vmx.lasterror.idCurrentCpu = RTMpCpuId();
4270 }
4271
4272 /* Just set the correct state here instead of trying to catch every goto above. */
4273 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED, VMCPUSTATE_STARTED_EXEC);
4274
4275#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
4276 /* Restore interrupts if we exitted after disabling them. */
4277 if (uOldEFlags != ~(RTCCUINTREG)0)
4278 ASMSetFlags(uOldEFlags);
4279#endif
4280
4281 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2, x);
4282 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit1, x);
4283 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatEntry, x);
4284 Log2(("X"));
4285 return VBOXSTRICTRC_TODO(rc);
4286}
4287
4288
4289/**
4290 * Enters the VT-x session
4291 *
4292 * @returns VBox status code.
4293 * @param pVM The VM to operate on.
4294 * @param pVCpu The VMCPU to operate on.
4295 * @param pCpu CPU info struct
4296 */
4297VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBLCPUINFO pCpu)
4298{
4299 Assert(pVM->hwaccm.s.vmx.fSupported);
4300 NOREF(pCpu);
4301
4302 unsigned cr4 = ASMGetCR4();
4303 if (!(cr4 & X86_CR4_VMXE))
4304 {
4305 AssertMsgFailed(("X86_CR4_VMXE should be set!\n"));
4306 return VERR_VMX_X86_CR4_VMXE_CLEARED;
4307 }
4308
4309 /* Activate the VM Control Structure. */
4310 int rc = VMXActivateVMCS(pVCpu->hwaccm.s.vmx.HCPhysVMCS);
4311 if (RT_FAILURE(rc))
4312 return rc;
4313
4314 pVCpu->hwaccm.s.fResumeVM = false;
4315 return VINF_SUCCESS;
4316}
4317
4318
4319/**
4320 * Leaves the VT-x session
4321 *
4322 * @returns VBox status code.
4323 * @param pVM The VM to operate on.
4324 * @param pVCpu The VMCPU to operate on.
4325 * @param pCtx CPU context
4326 */
4327VMMR0DECL(int) VMXR0Leave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4328{
4329 Assert(pVM->hwaccm.s.vmx.fSupported);
4330
4331#ifdef DEBUG
4332 if (CPUMIsHyperDebugStateActive(pVCpu))
4333 {
4334 CPUMR0LoadHostDebugState(pVM, pVCpu);
4335 Assert(pVCpu->hwaccm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT);
4336 }
4337 else
4338#endif
4339 /* Save the guest debug state if necessary. */
4340 if (CPUMIsGuestDebugStateActive(pVCpu))
4341 {
4342 CPUMR0SaveGuestDebugState(pVM, pVCpu, pCtx, true /* save DR6 */);
4343
4344 /* Enable drx move intercepts again. */
4345 pVCpu->hwaccm.s.vmx.proc_ctls |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
4346 int rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
4347 AssertRC(rc);
4348
4349 /* Resync the debug registers the next time. */
4350 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_DEBUG;
4351 }
4352 else
4353 Assert(pVCpu->hwaccm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT);
4354
4355 /* Clear VM Control Structure. Marking it inactive, clearing implementation specific data and writing back VMCS data to memory. */
4356 int rc = VMXClearVMCS(pVCpu->hwaccm.s.vmx.HCPhysVMCS);
4357 AssertRC(rc);
4358
4359 return VINF_SUCCESS;
4360}
4361
4362/**
4363 * Flush the TLB (EPT)
4364 *
4365 * @returns VBox status code.
4366 * @param pVM The VM to operate on.
4367 * @param pVCpu The VM CPU to operate on.
4368 * @param enmFlush Type of flush
4369 * @param GCPhys Physical address of the page to flush
4370 */
4371static void hmR0VmxFlushEPT(PVM pVM, PVMCPU pVCpu, VMX_FLUSH enmFlush, RTGCPHYS GCPhys)
4372{
4373 uint64_t descriptor[2];
4374
4375 LogFlow(("hmR0VmxFlushEPT %d %RGv\n", enmFlush, GCPhys));
4376 Assert(pVM->hwaccm.s.fNestedPaging);
4377 descriptor[0] = pVCpu->hwaccm.s.vmx.GCPhysEPTP;
4378 descriptor[1] = GCPhys;
4379 int rc = VMXR0InvEPT(enmFlush, &descriptor[0]);
4380 AssertRC(rc);
4381}
4382
4383#ifdef HWACCM_VTX_WITH_VPID
4384/**
4385 * Flush the TLB (EPT)
4386 *
4387 * @returns VBox status code.
4388 * @param pVM The VM to operate on.
4389 * @param pVCpu The VM CPU to operate on.
4390 * @param enmFlush Type of flush
4391 * @param GCPtr Virtual address of the page to flush
4392 */
4393static void hmR0VmxFlushVPID(PVM pVM, PVMCPU pVCpu, VMX_FLUSH enmFlush, RTGCPTR GCPtr)
4394{
4395#if HC_ARCH_BITS == 32
4396 /* If we get a flush in 64 bits guest mode, then force a full TLB flush. Invvpid probably takes only 32 bits addresses. (@todo) */
4397 if ( CPUMIsGuestInLongMode(pVCpu)
4398 && !VMX_IS_64BIT_HOST_MODE())
4399 {
4400 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
4401 }
4402 else
4403#endif
4404 {
4405 uint64_t descriptor[2];
4406
4407 Assert(pVM->hwaccm.s.vmx.fVPID);
4408 descriptor[0] = pVCpu->hwaccm.s.uCurrentASID;
4409 descriptor[1] = GCPtr;
4410 int rc = VMXR0InvVPID(enmFlush, &descriptor[0]); NOREF(rc);
4411 AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvVPID %x %x %RGv failed with %d\n", enmFlush, pVCpu->hwaccm.s.uCurrentASID, GCPtr, rc));
4412 }
4413}
4414#endif /* HWACCM_VTX_WITH_VPID */
4415
4416/**
4417 * Invalidates a guest page
4418 *
4419 * @returns VBox status code.
4420 * @param pVM The VM to operate on.
4421 * @param pVCpu The VM CPU to operate on.
4422 * @param GCVirt Page to invalidate
4423 */
4424VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
4425{
4426 bool fFlushPending = VMCPU_FF_ISSET(pVCpu, VMCPU_FF_TLB_FLUSH);
4427
4428 Log2(("VMXR0InvalidatePage %RGv\n", GCVirt));
4429
4430 /* Only relevant if we want to use VPID.
4431 * In the nested paging case we still see such calls, but
4432 * can safely ignore them. (e.g. after cr3 updates)
4433 */
4434#ifdef HWACCM_VTX_WITH_VPID
4435 /* Skip it if a TLB flush is already pending. */
4436 if ( !fFlushPending
4437 && pVM->hwaccm.s.vmx.fVPID)
4438 hmR0VmxFlushVPID(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushPage, GCVirt);
4439#endif /* HWACCM_VTX_WITH_VPID */
4440
4441 return VINF_SUCCESS;
4442}
4443
4444/**
4445 * Invalidates a guest page by physical address
4446 *
4447 * NOTE: Assumes the current instruction references this physical page though a virtual address!!
4448 *
4449 * @returns VBox status code.
4450 * @param pVM The VM to operate on.
4451 * @param pVCpu The VM CPU to operate on.
4452 * @param GCPhys Page to invalidate
4453 */
4454VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
4455{
4456 bool fFlushPending = VMCPU_FF_ISSET(pVCpu, VMCPU_FF_TLB_FLUSH);
4457
4458 Assert(pVM->hwaccm.s.fNestedPaging);
4459
4460 LogFlow(("VMXR0InvalidatePhysPage %RGp\n", GCPhys));
4461
4462 /* Skip it if a TLB flush is already pending. */
4463 if (!fFlushPending)
4464 hmR0VmxFlushEPT(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushPage, GCPhys);
4465
4466 return VINF_SUCCESS;
4467}
4468
4469/**
4470 * Report world switch error and dump some useful debug info
4471 *
4472 * @param pVM The VM to operate on.
4473 * @param pVCpu The VMCPU to operate on.
4474 * @param rc Return code
4475 * @param pCtx Current CPU context (not updated)
4476 */
4477static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, VBOXSTRICTRC rc, PCPUMCTX pCtx)
4478{
4479 NOREF(pVM);
4480
4481 switch (VBOXSTRICTRC_VAL(rc))
4482 {
4483 case VERR_VMX_INVALID_VMXON_PTR:
4484 AssertFailed();
4485 break;
4486
4487 case VERR_VMX_UNABLE_TO_START_VM:
4488 case VERR_VMX_UNABLE_TO_RESUME_VM:
4489 {
4490 int rc2;
4491 RTCCUINTREG exitReason, instrError;
4492
4493 rc2 = VMXReadVMCS(VMX_VMCS32_RO_EXIT_REASON, &exitReason);
4494 rc2 |= VMXReadVMCS(VMX_VMCS32_RO_VM_INSTR_ERROR, &instrError);
4495 AssertRC(rc2);
4496 if (rc2 == VINF_SUCCESS)
4497 {
4498 Log(("Unable to start/resume VM for reason: %x. Instruction error %x\n", (uint32_t)exitReason, (uint32_t)instrError));
4499 Log(("Current stack %08x\n", &rc2));
4500
4501 pVCpu->hwaccm.s.vmx.lasterror.ulInstrError = instrError;
4502 pVCpu->hwaccm.s.vmx.lasterror.ulExitReason = exitReason;
4503
4504#ifdef VBOX_STRICT
4505 RTGDTR gdtr;
4506 PCX86DESCHC pDesc;
4507 RTCCUINTREG val;
4508
4509 ASMGetGDTR(&gdtr);
4510
4511 VMXReadVMCS(VMX_VMCS64_GUEST_RIP, &val);
4512 Log(("Old eip %RGv new %RGv\n", (RTGCPTR)pCtx->rip, (RTGCPTR)val));
4513 VMXReadVMCS(VMX_VMCS_CTRL_PIN_EXEC_CONTROLS, &val);
4514 Log(("VMX_VMCS_CTRL_PIN_EXEC_CONTROLS %08x\n", val));
4515 VMXReadVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, &val);
4516 Log(("VMX_VMCS_CTRL_PROC_EXEC_CONTROLS %08x\n", val));
4517 VMXReadVMCS(VMX_VMCS_CTRL_ENTRY_CONTROLS, &val);
4518 Log(("VMX_VMCS_CTRL_ENTRY_CONTROLS %08x\n", val));
4519 VMXReadVMCS(VMX_VMCS_CTRL_EXIT_CONTROLS, &val);
4520 Log(("VMX_VMCS_CTRL_EXIT_CONTROLS %08x\n", val));
4521
4522 VMXReadVMCS(VMX_VMCS_HOST_CR0, &val);
4523 Log(("VMX_VMCS_HOST_CR0 %08x\n", val));
4524
4525 VMXReadVMCS(VMX_VMCS_HOST_CR3, &val);
4526 Log(("VMX_VMCS_HOST_CR3 %08x\n", val));
4527
4528 VMXReadVMCS(VMX_VMCS_HOST_CR4, &val);
4529 Log(("VMX_VMCS_HOST_CR4 %08x\n", val));
4530
4531 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_CS, &val);
4532 Log(("VMX_VMCS_HOST_FIELD_CS %08x\n", val));
4533
4534 VMXReadVMCS(VMX_VMCS_GUEST_RFLAGS, &val);
4535 Log(("VMX_VMCS_GUEST_RFLAGS %08x\n", val));
4536
4537 if (val < gdtr.cbGdt)
4538 {
4539 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
4540 HWACCMR0DumpDescriptor(pDesc, val, "CS: ");
4541 }
4542
4543 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_DS, &val);
4544 Log(("VMX_VMCS_HOST_FIELD_DS %08x\n", val));
4545 if (val < gdtr.cbGdt)
4546 {
4547 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
4548 HWACCMR0DumpDescriptor(pDesc, val, "DS: ");
4549 }
4550
4551 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_ES, &val);
4552 Log(("VMX_VMCS_HOST_FIELD_ES %08x\n", val));
4553 if (val < gdtr.cbGdt)
4554 {
4555 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
4556 HWACCMR0DumpDescriptor(pDesc, val, "ES: ");
4557 }
4558
4559 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_FS, &val);
4560 Log(("VMX_VMCS16_HOST_FIELD_FS %08x\n", val));
4561 if (val < gdtr.cbGdt)
4562 {
4563 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
4564 HWACCMR0DumpDescriptor(pDesc, val, "FS: ");
4565 }
4566
4567 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_GS, &val);
4568 Log(("VMX_VMCS16_HOST_FIELD_GS %08x\n", val));
4569 if (val < gdtr.cbGdt)
4570 {
4571 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
4572 HWACCMR0DumpDescriptor(pDesc, val, "GS: ");
4573 }
4574
4575 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_SS, &val);
4576 Log(("VMX_VMCS16_HOST_FIELD_SS %08x\n", val));
4577 if (val < gdtr.cbGdt)
4578 {
4579 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
4580 HWACCMR0DumpDescriptor(pDesc, val, "SS: ");
4581 }
4582
4583 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_TR, &val);
4584 Log(("VMX_VMCS16_HOST_FIELD_TR %08x\n", val));
4585 if (val < gdtr.cbGdt)
4586 {
4587 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
4588 HWACCMR0DumpDescriptor(pDesc, val, "TR: ");
4589 }
4590
4591 VMXReadVMCS(VMX_VMCS_HOST_TR_BASE, &val);
4592 Log(("VMX_VMCS_HOST_TR_BASE %RHv\n", val));
4593
4594 VMXReadVMCS(VMX_VMCS_HOST_GDTR_BASE, &val);
4595 Log(("VMX_VMCS_HOST_GDTR_BASE %RHv\n", val));
4596 VMXReadVMCS(VMX_VMCS_HOST_IDTR_BASE, &val);
4597 Log(("VMX_VMCS_HOST_IDTR_BASE %RHv\n", val));
4598
4599 VMXReadVMCS(VMX_VMCS32_HOST_SYSENTER_CS, &val);
4600 Log(("VMX_VMCS_HOST_SYSENTER_CS %08x\n", val));
4601
4602 VMXReadVMCS(VMX_VMCS_HOST_SYSENTER_EIP, &val);
4603 Log(("VMX_VMCS_HOST_SYSENTER_EIP %RHv\n", val));
4604
4605 VMXReadVMCS(VMX_VMCS_HOST_SYSENTER_ESP, &val);
4606 Log(("VMX_VMCS_HOST_SYSENTER_ESP %RHv\n", val));
4607
4608 VMXReadVMCS(VMX_VMCS_HOST_RSP, &val);
4609 Log(("VMX_VMCS_HOST_RSP %RHv\n", val));
4610 VMXReadVMCS(VMX_VMCS_HOST_RIP, &val);
4611 Log(("VMX_VMCS_HOST_RIP %RHv\n", val));
4612
4613# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4614 if (VMX_IS_64BIT_HOST_MODE())
4615 {
4616 Log(("MSR_K6_EFER = %RX64\n", ASMRdMsr(MSR_K6_EFER)));
4617 Log(("MSR_K6_STAR = %RX64\n", ASMRdMsr(MSR_K6_STAR)));
4618 Log(("MSR_K8_LSTAR = %RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
4619 Log(("MSR_K8_CSTAR = %RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
4620 Log(("MSR_K8_SF_MASK = %RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
4621 }
4622# endif
4623#endif /* VBOX_STRICT */
4624 }
4625 break;
4626 }
4627
4628 default:
4629 /* impossible */
4630 AssertMsgFailed(("%Rrc (%#x)\n", VBOXSTRICTRC_VAL(rc), VBOXSTRICTRC_VAL(rc)));
4631 break;
4632 }
4633}
4634
4635#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4636
4637/**
4638 * Prepares for and executes VMLAUNCH (64 bits guest mode)
4639 *
4640 * @returns VBox status code
4641 * @param fResume vmlauch/vmresume
4642 * @param pCtx Guest context
4643 * @param pCache VMCS cache
4644 * @param pVM The VM to operate on.
4645 * @param pVCpu The VMCPU to operate on.
4646 */
4647DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
4648{
4649 uint32_t aParam[6];
4650 PHMGLOBLCPUINFO pCpu;
4651 RTHCPHYS HCPhysCpuPage;
4652 int rc;
4653
4654 pCpu = HWACCMR0GetCurrentCpu();
4655 HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4656
4657#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4658 pCache->uPos = 1;
4659 pCache->interPD = PGMGetInterPaeCR3(pVM);
4660 pCache->pSwitcher = (uint64_t)pVM->hwaccm.s.pfnHost32ToGuest64R0;
4661#endif
4662
4663#ifdef DEBUG
4664 pCache->TestIn.HCPhysCpuPage= 0;
4665 pCache->TestIn.HCPhysVMCS = 0;
4666 pCache->TestIn.pCache = 0;
4667 pCache->TestOut.HCPhysVMCS = 0;
4668 pCache->TestOut.pCache = 0;
4669 pCache->TestOut.pCtx = 0;
4670 pCache->TestOut.eflags = 0;
4671#endif
4672
4673 aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
4674 aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
4675 aParam[2] = (uint32_t)(pVCpu->hwaccm.s.vmx.HCPhysVMCS); /* Param 2: VMCS physical address - Lo. */
4676 aParam[3] = (uint32_t)(pVCpu->hwaccm.s.vmx.HCPhysVMCS >> 32); /* Param 2: VMCS physical address - Hi. */
4677 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hwaccm.s.vmx.VMCSCache);
4678 aParam[5] = 0;
4679
4680#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4681 pCtx->dr[4] = pVM->hwaccm.s.vmx.pScratchPhys + 16 + 8;
4682 *(uint32_t *)(pVM->hwaccm.s.vmx.pScratch + 16 + 8) = 1;
4683#endif
4684 rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, pVM->hwaccm.s.pfnVMXGCStartVM64, 6, &aParam[0]);
4685
4686#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4687 Assert(*(uint32_t *)(pVM->hwaccm.s.vmx.pScratch + 16 + 8) == 5);
4688 Assert(pCtx->dr[4] == 10);
4689 *(uint32_t *)(pVM->hwaccm.s.vmx.pScratch + 16 + 8) = 0xff;
4690#endif
4691
4692#ifdef DEBUG
4693 AssertMsg(pCache->TestIn.HCPhysCpuPage== HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
4694 AssertMsg(pCache->TestIn.HCPhysVMCS == pVCpu->hwaccm.s.vmx.HCPhysVMCS, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVMCS, pVCpu->hwaccm.s.vmx.HCPhysVMCS));
4695 AssertMsg(pCache->TestIn.HCPhysVMCS == pCache->TestOut.HCPhysVMCS, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVMCS, pCache->TestOut.HCPhysVMCS));
4696 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache, pCache->TestOut.pCache));
4697 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hwaccm.s.vmx.VMCSCache), ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hwaccm.s.vmx.VMCSCache)));
4698 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx, pCache->TestOut.pCtx));
4699 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
4700#endif
4701 return rc;
4702}
4703
4704# ifdef VBOX_STRICT
4705
4706static bool hmR0VmxIsValidReadField(uint32_t idxField)
4707{
4708 switch(idxField)
4709 {
4710 case VMX_VMCS64_GUEST_RIP:
4711 case VMX_VMCS64_GUEST_RSP:
4712 case VMX_VMCS_GUEST_RFLAGS:
4713 case VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE:
4714 case VMX_VMCS_CTRL_CR0_READ_SHADOW:
4715 case VMX_VMCS64_GUEST_CR0:
4716 case VMX_VMCS_CTRL_CR4_READ_SHADOW:
4717 case VMX_VMCS64_GUEST_CR4:
4718 case VMX_VMCS64_GUEST_DR7:
4719 case VMX_VMCS32_GUEST_SYSENTER_CS:
4720 case VMX_VMCS64_GUEST_SYSENTER_EIP:
4721 case VMX_VMCS64_GUEST_SYSENTER_ESP:
4722 case VMX_VMCS32_GUEST_GDTR_LIMIT:
4723 case VMX_VMCS64_GUEST_GDTR_BASE:
4724 case VMX_VMCS32_GUEST_IDTR_LIMIT:
4725 case VMX_VMCS64_GUEST_IDTR_BASE:
4726 case VMX_VMCS16_GUEST_FIELD_CS:
4727 case VMX_VMCS32_GUEST_CS_LIMIT:
4728 case VMX_VMCS64_GUEST_CS_BASE:
4729 case VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS:
4730 case VMX_VMCS16_GUEST_FIELD_DS:
4731 case VMX_VMCS32_GUEST_DS_LIMIT:
4732 case VMX_VMCS64_GUEST_DS_BASE:
4733 case VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS:
4734 case VMX_VMCS16_GUEST_FIELD_ES:
4735 case VMX_VMCS32_GUEST_ES_LIMIT:
4736 case VMX_VMCS64_GUEST_ES_BASE:
4737 case VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS:
4738 case VMX_VMCS16_GUEST_FIELD_FS:
4739 case VMX_VMCS32_GUEST_FS_LIMIT:
4740 case VMX_VMCS64_GUEST_FS_BASE:
4741 case VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS:
4742 case VMX_VMCS16_GUEST_FIELD_GS:
4743 case VMX_VMCS32_GUEST_GS_LIMIT:
4744 case VMX_VMCS64_GUEST_GS_BASE:
4745 case VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS:
4746 case VMX_VMCS16_GUEST_FIELD_SS:
4747 case VMX_VMCS32_GUEST_SS_LIMIT:
4748 case VMX_VMCS64_GUEST_SS_BASE:
4749 case VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS:
4750 case VMX_VMCS16_GUEST_FIELD_LDTR:
4751 case VMX_VMCS32_GUEST_LDTR_LIMIT:
4752 case VMX_VMCS64_GUEST_LDTR_BASE:
4753 case VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS:
4754 case VMX_VMCS16_GUEST_FIELD_TR:
4755 case VMX_VMCS32_GUEST_TR_LIMIT:
4756 case VMX_VMCS64_GUEST_TR_BASE:
4757 case VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS:
4758 case VMX_VMCS32_RO_EXIT_REASON:
4759 case VMX_VMCS32_RO_VM_INSTR_ERROR:
4760 case VMX_VMCS32_RO_EXIT_INSTR_LENGTH:
4761 case VMX_VMCS32_RO_EXIT_INTERRUPTION_ERRCODE:
4762 case VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO:
4763 case VMX_VMCS32_RO_EXIT_INSTR_INFO:
4764 case VMX_VMCS_RO_EXIT_QUALIFICATION:
4765 case VMX_VMCS32_RO_IDT_INFO:
4766 case VMX_VMCS32_RO_IDT_ERRCODE:
4767 case VMX_VMCS64_GUEST_CR3:
4768 case VMX_VMCS_EXIT_PHYS_ADDR_FULL:
4769 return true;
4770 }
4771 return false;
4772}
4773
4774static bool hmR0VmxIsValidWriteField(uint32_t idxField)
4775{
4776 switch(idxField)
4777 {
4778 case VMX_VMCS64_GUEST_LDTR_BASE:
4779 case VMX_VMCS64_GUEST_TR_BASE:
4780 case VMX_VMCS64_GUEST_GDTR_BASE:
4781 case VMX_VMCS64_GUEST_IDTR_BASE:
4782 case VMX_VMCS64_GUEST_SYSENTER_EIP:
4783 case VMX_VMCS64_GUEST_SYSENTER_ESP:
4784 case VMX_VMCS64_GUEST_CR0:
4785 case VMX_VMCS64_GUEST_CR4:
4786 case VMX_VMCS64_GUEST_CR3:
4787 case VMX_VMCS64_GUEST_DR7:
4788 case VMX_VMCS64_GUEST_RIP:
4789 case VMX_VMCS64_GUEST_RSP:
4790 case VMX_VMCS64_GUEST_CS_BASE:
4791 case VMX_VMCS64_GUEST_DS_BASE:
4792 case VMX_VMCS64_GUEST_ES_BASE:
4793 case VMX_VMCS64_GUEST_FS_BASE:
4794 case VMX_VMCS64_GUEST_GS_BASE:
4795 case VMX_VMCS64_GUEST_SS_BASE:
4796 return true;
4797 }
4798 return false;
4799}
4800
4801# endif /* VBOX_STRICT */
4802
4803/**
4804 * Executes the specified handler in 64 mode
4805 *
4806 * @returns VBox status code.
4807 * @param pVM The VM to operate on.
4808 * @param pVCpu The VMCPU to operate on.
4809 * @param pCtx Guest context
4810 * @param pfnHandler RC handler
4811 * @param cbParam Number of parameters
4812 * @param paParam Array of 32 bits parameters
4813 */
4814VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, RTRCPTR pfnHandler, uint32_t cbParam, uint32_t *paParam)
4815{
4816 int rc, rc2;
4817 PHMGLOBLCPUINFO pCpu;
4818 RTHCPHYS HCPhysCpuPage;
4819 RTHCUINTREG uOldEFlags;
4820
4821 AssertReturn(pVM->hwaccm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
4822 Assert(pfnHandler);
4823 Assert(pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hwaccm.s.vmx.VMCSCache.Write.aField));
4824 Assert(pVCpu->hwaccm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hwaccm.s.vmx.VMCSCache.Read.aField));
4825
4826#ifdef VBOX_STRICT
4827 for (unsigned i=0;i<pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries;i++)
4828 Assert(hmR0VmxIsValidWriteField(pVCpu->hwaccm.s.vmx.VMCSCache.Write.aField[i]));
4829
4830 for (unsigned i=0;i<pVCpu->hwaccm.s.vmx.VMCSCache.Read.cValidEntries;i++)
4831 Assert(hmR0VmxIsValidReadField(pVCpu->hwaccm.s.vmx.VMCSCache.Read.aField[i]));
4832#endif
4833
4834 /* Disable interrupts. */
4835 uOldEFlags = ASMIntDisableFlags();
4836
4837 pCpu = HWACCMR0GetCurrentCpu();
4838 HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4839
4840 /* Clear VM Control Structure. Marking it inactive, clearing implementation specific data and writing back VMCS data to memory. */
4841 VMXClearVMCS(pVCpu->hwaccm.s.vmx.HCPhysVMCS);
4842
4843 /* Leave VMX Root Mode. */
4844 VMXDisable();
4845
4846 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4847
4848 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
4849 CPUMSetHyperEIP(pVCpu, pfnHandler);
4850 for (int i=(int)cbParam-1;i>=0;i--)
4851 CPUMPushHyper(pVCpu, paParam[i]);
4852
4853 STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatWorldSwitch3264, z);
4854 /* Call switcher. */
4855 rc = pVM->hwaccm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
4856 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatWorldSwitch3264, z);
4857
4858 /* Make sure the VMX instructions don't cause #UD faults. */
4859 ASMSetCR4(ASMGetCR4() | X86_CR4_VMXE);
4860
4861 /* Enter VMX Root Mode */
4862 rc2 = VMXEnable(HCPhysCpuPage);
4863 if (RT_FAILURE(rc2))
4864 {
4865 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4866 ASMSetFlags(uOldEFlags);
4867 return VERR_VMX_VMXON_FAILED;
4868 }
4869
4870 rc2 = VMXActivateVMCS(pVCpu->hwaccm.s.vmx.HCPhysVMCS);
4871 AssertRC(rc2);
4872 Assert(!(ASMGetFlags() & X86_EFL_IF));
4873 ASMSetFlags(uOldEFlags);
4874 return rc;
4875}
4876
4877#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
4878
4879
4880#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
4881/**
4882 * Executes VMWRITE
4883 *
4884 * @returns VBox status code
4885 * @param pVCpu The VMCPU to operate on.
4886 * @param idxField VMCS index
4887 * @param u64Val 16, 32 or 64 bits value
4888 */
4889VMMR0DECL(int) VMXWriteVMCS64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4890{
4891 int rc;
4892
4893 switch (idxField)
4894 {
4895 case VMX_VMCS_CTRL_TSC_OFFSET_FULL:
4896 case VMX_VMCS_CTRL_IO_BITMAP_A_FULL:
4897 case VMX_VMCS_CTRL_IO_BITMAP_B_FULL:
4898 case VMX_VMCS_CTRL_MSR_BITMAP_FULL:
4899 case VMX_VMCS_CTRL_VMEXIT_MSR_STORE_FULL:
4900 case VMX_VMCS_CTRL_VMEXIT_MSR_LOAD_FULL:
4901 case VMX_VMCS_CTRL_VMENTRY_MSR_LOAD_FULL:
4902 case VMX_VMCS_CTRL_VAPIC_PAGEADDR_FULL:
4903 case VMX_VMCS_CTRL_APIC_ACCESSADDR_FULL:
4904 case VMX_VMCS_GUEST_LINK_PTR_FULL:
4905 case VMX_VMCS_GUEST_PDPTR0_FULL:
4906 case VMX_VMCS_GUEST_PDPTR1_FULL:
4907 case VMX_VMCS_GUEST_PDPTR2_FULL:
4908 case VMX_VMCS_GUEST_PDPTR3_FULL:
4909 case VMX_VMCS_GUEST_DEBUGCTL_FULL:
4910 case VMX_VMCS_GUEST_EFER_FULL:
4911 case VMX_VMCS_CTRL_EPTP_FULL:
4912 /* These fields consist of two parts, which are both writable in 32 bits mode. */
4913 rc = VMXWriteVMCS32(idxField, u64Val);
4914 rc |= VMXWriteVMCS32(idxField + 1, (uint32_t)(u64Val >> 32ULL));
4915 AssertRC(rc);
4916 return rc;
4917
4918 case VMX_VMCS64_GUEST_LDTR_BASE:
4919 case VMX_VMCS64_GUEST_TR_BASE:
4920 case VMX_VMCS64_GUEST_GDTR_BASE:
4921 case VMX_VMCS64_GUEST_IDTR_BASE:
4922 case VMX_VMCS64_GUEST_SYSENTER_EIP:
4923 case VMX_VMCS64_GUEST_SYSENTER_ESP:
4924 case VMX_VMCS64_GUEST_CR0:
4925 case VMX_VMCS64_GUEST_CR4:
4926 case VMX_VMCS64_GUEST_CR3:
4927 case VMX_VMCS64_GUEST_DR7:
4928 case VMX_VMCS64_GUEST_RIP:
4929 case VMX_VMCS64_GUEST_RSP:
4930 case VMX_VMCS64_GUEST_CS_BASE:
4931 case VMX_VMCS64_GUEST_DS_BASE:
4932 case VMX_VMCS64_GUEST_ES_BASE:
4933 case VMX_VMCS64_GUEST_FS_BASE:
4934 case VMX_VMCS64_GUEST_GS_BASE:
4935 case VMX_VMCS64_GUEST_SS_BASE:
4936 /* Queue a 64 bits value as we can't set it in 32 bits host mode. */
4937 if (u64Val >> 32ULL)
4938 rc = VMXWriteCachedVMCSEx(pVCpu, idxField, u64Val);
4939 else
4940 rc = VMXWriteVMCS32(idxField, (uint32_t)u64Val);
4941
4942 return rc;
4943
4944 default:
4945 AssertMsgFailed(("Unexpected field %x\n", idxField));
4946 return VERR_INVALID_PARAMETER;
4947 }
4948}
4949
4950/**
4951 * Cache VMCS writes for performance reasons (Darwin) and for running 64 bits guests on 32 bits hosts.
4952 *
4953 * @param pVCpu The VMCPU to operate on.
4954 * @param idxField VMCS field
4955 * @param u64Val Value
4956 */
4957VMMR0DECL(int) VMXWriteCachedVMCSEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4958{
4959 PVMCSCACHE pCache = &pVCpu->hwaccm.s.vmx.VMCSCache;
4960
4961 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1, ("entries=%x\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
4962
4963 /* Make sure there are no duplicates. */
4964 for (unsigned i=0;i<pCache->Write.cValidEntries;i++)
4965 {
4966 if (pCache->Write.aField[i] == idxField)
4967 {
4968 pCache->Write.aFieldVal[i] = u64Val;
4969 return VINF_SUCCESS;
4970 }
4971 }
4972
4973 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
4974 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
4975 pCache->Write.cValidEntries++;
4976 return VINF_SUCCESS;
4977}
4978
4979#endif /* HC_ARCH_BITS == 32 && !VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0 */
4980
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette