VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HWVMXR0.cpp@ 39402

Last change on this file since 39402 was 39402, checked in by vboxsync, 13 years ago

VMM: don't use generic IPE status codes, use specific ones. Part 1.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Id
File size: 200.3 KB
Line 
1/* $Id: HWVMXR0.cpp 39402 2011-11-23 16:25:04Z vboxsync $ */
2/** @file
3 * HM VMX (VT-x) - Host Context Ring 0.
4 */
5
6/*
7 * Copyright (C) 2006-2011 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18
19/*******************************************************************************
20* Header Files *
21*******************************************************************************/
22#define LOG_GROUP LOG_GROUP_HWACCM
23#include <iprt/asm-amd64-x86.h>
24#include <VBox/vmm/hwaccm.h>
25#include <VBox/vmm/pgm.h>
26#include <VBox/vmm/dbgf.h>
27#include <VBox/vmm/selm.h>
28#include <VBox/vmm/iom.h>
29#include <VBox/vmm/rem.h>
30#include <VBox/vmm/tm.h>
31#include "HWACCMInternal.h"
32#include <VBox/vmm/vm.h>
33#include <VBox/vmm/pdmapi.h>
34#include <VBox/err.h>
35#include <VBox/log.h>
36#include <iprt/assert.h>
37#include <iprt/param.h>
38#include <iprt/string.h>
39#include <iprt/time.h>
40#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
41# include <iprt/thread.h>
42#endif
43#include <iprt/x86.h>
44#include "HWVMXR0.h"
45
46/*******************************************************************************
47* Defined Constants And Macros *
48*******************************************************************************/
49#if defined(RT_ARCH_AMD64)
50# define VMX_IS_64BIT_HOST_MODE() (true)
51#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
52# define VMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
53#else
54# define VMX_IS_64BIT_HOST_MODE() (false)
55#endif
56
57/*******************************************************************************
58* Global Variables *
59*******************************************************************************/
60/* IO operation lookup arrays. */
61static uint32_t const g_aIOSize[4] = {1, 2, 0, 4};
62static uint32_t const g_aIOOpAnd[4] = {0xff, 0xffff, 0, 0xffffffff};
63
64#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
65/** See HWACCMR0A.asm. */
66extern "C" uint32_t g_fVMXIs64bitHost;
67#endif
68
69/*******************************************************************************
70* Local Functions *
71*******************************************************************************/
72static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, VBOXSTRICTRC rc, PCPUMCTX pCtx);
73static void hmR0VmxSetupTLBEPT(PVM pVM, PVMCPU pVCpu);
74static void hmR0VmxSetupTLBVPID(PVM pVM, PVMCPU pVCpu);
75static void hmR0VmxSetupTLBDummy(PVM pVM, PVMCPU pVCpu);
76static void hmR0VmxFlushEPT(PVM pVM, PVMCPU pVCpu, VMX_FLUSH enmFlush, RTGCPHYS GCPhys);
77static void hmR0VmxFlushVPID(PVM pVM, PVMCPU pVCpu, VMX_FLUSH enmFlush, RTGCPTR GCPtr);
78static void hmR0VmxUpdateExceptionBitmap(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
79static void hmR0VmxSetMSRPermission(PVMCPU pVCpu, unsigned ulMSR, bool fRead, bool fWrite);
80
81
82static void hmR0VmxCheckError(PVM pVM, PVMCPU pVCpu, int rc)
83{
84 if (rc == VERR_VMX_GENERIC)
85 {
86 RTCCUINTREG instrError;
87
88 VMXReadVMCS(VMX_VMCS32_RO_VM_INSTR_ERROR, &instrError);
89 pVCpu->hwaccm.s.vmx.lasterror.ulInstrError = instrError;
90 }
91 pVM->hwaccm.s.lLastError = rc;
92}
93
94/**
95 * Sets up and activates VT-x on the current CPU
96 *
97 * @returns VBox status code.
98 * @param pCpu CPU info struct
99 * @param pVM The VM to operate on. (can be NULL after a resume!!)
100 * @param pvCpuPage Pointer to the global cpu page.
101 * @param HCPhysCpuPage Physical address of the global cpu page.
102 */
103VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBLCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
104{
105 AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
106 AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
107 NOREF(pCpu);
108
109 if (pVM)
110 {
111 /* Set revision dword at the beginning of the VMXON structure. */
112 *(uint32_t *)pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hwaccm.s.vmx.msr.vmx_basic_info);
113 }
114
115 /** @todo we should unmap the two pages from the virtual address space in order to prevent accidental corruption.
116 * (which can have very bad consequences!!!)
117 */
118
119 if (ASMGetCR4() & X86_CR4_VMXE)
120 return VERR_VMX_IN_VMX_ROOT_MODE;
121
122 /* Make sure the VMX instructions don't cause #UD faults. */
123 ASMSetCR4(ASMGetCR4() | X86_CR4_VMXE);
124
125 /* Enter VMX Root Mode. */
126 int rc = VMXEnable(HCPhysCpuPage);
127 if (RT_FAILURE(rc))
128 {
129 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
130 return VERR_VMX_VMXON_FAILED;
131 }
132 return VINF_SUCCESS;
133}
134
135/**
136 * Deactivates VT-x on the current CPU
137 *
138 * @returns VBox status code.
139 * @param pCpu CPU info struct
140 * @param pvCpuPage Pointer to the global cpu page.
141 * @param HCPhysCpuPage Physical address of the global cpu page.
142 */
143VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBLCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
144{
145 AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
146 AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
147 NOREF(pCpu);
148
149 /* If we're somehow not in VMX root mode, then we shouldn't dare leaving it. */
150 if (!(ASMGetCR4() & X86_CR4_VMXE))
151 return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
152
153 /* Leave VMX Root Mode. */
154 VMXDisable();
155
156 /* And clear the X86_CR4_VMXE bit. */
157 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
158 return VINF_SUCCESS;
159}
160
161/**
162 * Does Ring-0 per VM VT-x init.
163 *
164 * @returns VBox status code.
165 * @param pVM The VM to operate on.
166 */
167VMMR0DECL(int) VMXR0InitVM(PVM pVM)
168{
169 int rc;
170
171#ifdef LOG_ENABLED
172 SUPR0Printf("VMXR0InitVM %x\n", pVM);
173#endif
174
175 pVM->hwaccm.s.vmx.pMemObjAPIC = NIL_RTR0MEMOBJ;
176
177 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
178 {
179 /* Allocate one page for the APIC physical page (serves for filtering accesses). */
180 rc = RTR0MemObjAllocCont(&pVM->hwaccm.s.vmx.pMemObjAPIC, PAGE_SIZE, true /* executable R0 mapping */);
181 AssertRC(rc);
182 if (RT_FAILURE(rc))
183 return rc;
184
185 pVM->hwaccm.s.vmx.pAPIC = (uint8_t *)RTR0MemObjAddress(pVM->hwaccm.s.vmx.pMemObjAPIC);
186 pVM->hwaccm.s.vmx.pAPICPhys = RTR0MemObjGetPagePhysAddr(pVM->hwaccm.s.vmx.pMemObjAPIC, 0);
187 ASMMemZero32(pVM->hwaccm.s.vmx.pAPIC, PAGE_SIZE);
188 }
189 else
190 {
191 pVM->hwaccm.s.vmx.pMemObjAPIC = 0;
192 pVM->hwaccm.s.vmx.pAPIC = 0;
193 pVM->hwaccm.s.vmx.pAPICPhys = 0;
194 }
195
196#ifdef VBOX_WITH_CRASHDUMP_MAGIC
197 {
198 rc = RTR0MemObjAllocCont(&pVM->hwaccm.s.vmx.pMemObjScratch, PAGE_SIZE, true /* executable R0 mapping */);
199 AssertRC(rc);
200 if (RT_FAILURE(rc))
201 return rc;
202
203 pVM->hwaccm.s.vmx.pScratch = (uint8_t *)RTR0MemObjAddress(pVM->hwaccm.s.vmx.pMemObjScratch);
204 pVM->hwaccm.s.vmx.pScratchPhys = RTR0MemObjGetPagePhysAddr(pVM->hwaccm.s.vmx.pMemObjScratch, 0);
205
206 ASMMemZero32(pVM->hwaccm.s.vmx.pScratch, PAGE_SIZE);
207 strcpy((char *)pVM->hwaccm.s.vmx.pScratch, "SCRATCH Magic");
208 *(uint64_t *)(pVM->hwaccm.s.vmx.pScratch + 16) = UINT64_C(0xDEADBEEFDEADBEEF);
209 }
210#endif
211
212 /* Allocate VMCBs for all guest CPUs. */
213 for (VMCPUID i = 0; i < pVM->cCpus; i++)
214 {
215 PVMCPU pVCpu = &pVM->aCpus[i];
216
217 pVCpu->hwaccm.s.vmx.hMemObjVMCS = NIL_RTR0MEMOBJ;
218
219 /* Allocate one page for the VM control structure (VMCS). */
220 rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.hMemObjVMCS, PAGE_SIZE, true /* executable R0 mapping */);
221 AssertRC(rc);
222 if (RT_FAILURE(rc))
223 return rc;
224
225 pVCpu->hwaccm.s.vmx.pvVMCS = RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.hMemObjVMCS);
226 pVCpu->hwaccm.s.vmx.HCPhysVMCS = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.hMemObjVMCS, 0);
227 ASMMemZeroPage(pVCpu->hwaccm.s.vmx.pvVMCS);
228
229 pVCpu->hwaccm.s.vmx.cr0_mask = 0;
230 pVCpu->hwaccm.s.vmx.cr4_mask = 0;
231
232 /* Allocate one page for the virtual APIC page for TPR caching. */
233 rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.hMemObjVAPIC, PAGE_SIZE, true /* executable R0 mapping */);
234 AssertRC(rc);
235 if (RT_FAILURE(rc))
236 return rc;
237
238 pVCpu->hwaccm.s.vmx.pbVAPIC = (uint8_t *)RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.hMemObjVAPIC);
239 pVCpu->hwaccm.s.vmx.HCPhysVAPIC = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.hMemObjVAPIC, 0);
240 ASMMemZeroPage(pVCpu->hwaccm.s.vmx.pbVAPIC);
241
242 /* Allocate the MSR bitmap if this feature is supported. */
243 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
244 {
245 rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap, PAGE_SIZE, true /* executable R0 mapping */);
246 AssertRC(rc);
247 if (RT_FAILURE(rc))
248 return rc;
249
250 pVCpu->hwaccm.s.vmx.pMSRBitmap = (uint8_t *)RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap);
251 pVCpu->hwaccm.s.vmx.pMSRBitmapPhys = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap, 0);
252 memset(pVCpu->hwaccm.s.vmx.pMSRBitmap, 0xff, PAGE_SIZE);
253 }
254
255#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
256 /* Allocate one page for the guest MSR load area (for preloading guest MSRs during the world switch). */
257 rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.pMemObjGuestMSR, PAGE_SIZE, true /* executable R0 mapping */);
258 AssertRC(rc);
259 if (RT_FAILURE(rc))
260 return rc;
261
262 pVCpu->hwaccm.s.vmx.pGuestMSR = (uint8_t *)RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.pMemObjGuestMSR);
263 pVCpu->hwaccm.s.vmx.pGuestMSRPhys = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.pMemObjGuestMSR, 0);
264 memset(pVCpu->hwaccm.s.vmx.pGuestMSR, 0, PAGE_SIZE);
265
266 /* Allocate one page for the host MSR load area (for restoring host MSRs after the world switch back). */
267 rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.pMemObjHostMSR, PAGE_SIZE, true /* executable R0 mapping */);
268 AssertRC(rc);
269 if (RT_FAILURE(rc))
270 return rc;
271
272 pVCpu->hwaccm.s.vmx.pHostMSR = (uint8_t *)RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.pMemObjHostMSR);
273 pVCpu->hwaccm.s.vmx.pHostMSRPhys = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.pMemObjHostMSR, 0);
274 memset(pVCpu->hwaccm.s.vmx.pHostMSR, 0, PAGE_SIZE);
275#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
276
277 /* Current guest paging mode. */
278 pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode = PGMMODE_REAL;
279
280#ifdef LOG_ENABLED
281 SUPR0Printf("VMXR0InitVM %x VMCS=%x (%x)\n", pVM, pVCpu->hwaccm.s.vmx.pvVMCS, (uint32_t)pVCpu->hwaccm.s.vmx.HCPhysVMCS);
282#endif
283 }
284
285 return VINF_SUCCESS;
286}
287
288/**
289 * Does Ring-0 per VM VT-x termination.
290 *
291 * @returns VBox status code.
292 * @param pVM The VM to operate on.
293 */
294VMMR0DECL(int) VMXR0TermVM(PVM pVM)
295{
296 for (VMCPUID i = 0; i < pVM->cCpus; i++)
297 {
298 PVMCPU pVCpu = &pVM->aCpus[i];
299
300 if (pVCpu->hwaccm.s.vmx.hMemObjVMCS != NIL_RTR0MEMOBJ)
301 {
302 RTR0MemObjFree(pVCpu->hwaccm.s.vmx.hMemObjVMCS, false);
303 pVCpu->hwaccm.s.vmx.hMemObjVMCS = NIL_RTR0MEMOBJ;
304 pVCpu->hwaccm.s.vmx.pvVMCS = 0;
305 pVCpu->hwaccm.s.vmx.HCPhysVMCS = 0;
306 }
307 if (pVCpu->hwaccm.s.vmx.hMemObjVAPIC != NIL_RTR0MEMOBJ)
308 {
309 RTR0MemObjFree(pVCpu->hwaccm.s.vmx.hMemObjVAPIC, false);
310 pVCpu->hwaccm.s.vmx.hMemObjVAPIC = NIL_RTR0MEMOBJ;
311 pVCpu->hwaccm.s.vmx.pbVAPIC = 0;
312 pVCpu->hwaccm.s.vmx.HCPhysVAPIC = 0;
313 }
314 if (pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap != NIL_RTR0MEMOBJ)
315 {
316 RTR0MemObjFree(pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap, false);
317 pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap = NIL_RTR0MEMOBJ;
318 pVCpu->hwaccm.s.vmx.pMSRBitmap = 0;
319 pVCpu->hwaccm.s.vmx.pMSRBitmapPhys = 0;
320 }
321#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
322 if (pVCpu->hwaccm.s.vmx.pMemObjHostMSR != NIL_RTR0MEMOBJ)
323 {
324 RTR0MemObjFree(pVCpu->hwaccm.s.vmx.pMemObjHostMSR, false);
325 pVCpu->hwaccm.s.vmx.pMemObjHostMSR = NIL_RTR0MEMOBJ;
326 pVCpu->hwaccm.s.vmx.pHostMSR = 0;
327 pVCpu->hwaccm.s.vmx.pHostMSRPhys = 0;
328 }
329 if (pVCpu->hwaccm.s.vmx.pMemObjGuestMSR != NIL_RTR0MEMOBJ)
330 {
331 RTR0MemObjFree(pVCpu->hwaccm.s.vmx.pMemObjGuestMSR, false);
332 pVCpu->hwaccm.s.vmx.pMemObjGuestMSR = NIL_RTR0MEMOBJ;
333 pVCpu->hwaccm.s.vmx.pGuestMSR = 0;
334 pVCpu->hwaccm.s.vmx.pGuestMSRPhys = 0;
335 }
336#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
337 }
338 if (pVM->hwaccm.s.vmx.pMemObjAPIC != NIL_RTR0MEMOBJ)
339 {
340 RTR0MemObjFree(pVM->hwaccm.s.vmx.pMemObjAPIC, false);
341 pVM->hwaccm.s.vmx.pMemObjAPIC = NIL_RTR0MEMOBJ;
342 pVM->hwaccm.s.vmx.pAPIC = 0;
343 pVM->hwaccm.s.vmx.pAPICPhys = 0;
344 }
345#ifdef VBOX_WITH_CRASHDUMP_MAGIC
346 if (pVM->hwaccm.s.vmx.pMemObjScratch != NIL_RTR0MEMOBJ)
347 {
348 ASMMemZero32(pVM->hwaccm.s.vmx.pScratch, PAGE_SIZE);
349 RTR0MemObjFree(pVM->hwaccm.s.vmx.pMemObjScratch, false);
350 pVM->hwaccm.s.vmx.pMemObjScratch = NIL_RTR0MEMOBJ;
351 pVM->hwaccm.s.vmx.pScratch = 0;
352 pVM->hwaccm.s.vmx.pScratchPhys = 0;
353 }
354#endif
355 return VINF_SUCCESS;
356}
357
358/**
359 * Sets up VT-x for the specified VM
360 *
361 * @returns VBox status code.
362 * @param pVM The VM to operate on.
363 */
364VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
365{
366 int rc = VINF_SUCCESS;
367 uint32_t val;
368
369 AssertReturn(pVM, VERR_INVALID_PARAMETER);
370
371 for (VMCPUID i = 0; i < pVM->cCpus; i++)
372 {
373 PVMCPU pVCpu = &pVM->aCpus[i];
374
375 AssertPtr(pVCpu->hwaccm.s.vmx.pvVMCS);
376
377 /* Set revision dword at the beginning of the VMCS structure. */
378 *(uint32_t *)pVCpu->hwaccm.s.vmx.pvVMCS = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hwaccm.s.vmx.msr.vmx_basic_info);
379
380 /* Clear VM Control Structure. */
381 Log(("HCPhysVMCS = %RHp\n", pVCpu->hwaccm.s.vmx.HCPhysVMCS));
382 rc = VMXClearVMCS(pVCpu->hwaccm.s.vmx.HCPhysVMCS);
383 if (RT_FAILURE(rc))
384 goto vmx_end;
385
386 /* Activate the VM Control Structure. */
387 rc = VMXActivateVMCS(pVCpu->hwaccm.s.vmx.HCPhysVMCS);
388 if (RT_FAILURE(rc))
389 goto vmx_end;
390
391 /* VMX_VMCS_CTRL_PIN_EXEC_CONTROLS
392 * Set required bits to one and zero according to the MSR capabilities.
393 */
394 val = pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.disallowed0;
395 /* External and non-maskable interrupts cause VM-exits. */
396 val |= VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_EXT_INT_EXIT | VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_NMI_EXIT;
397 /* enable the preemption timer. */
398 if (pVM->hwaccm.s.vmx.fUsePreemptTimer)
399 val |= VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_PREEMPT_TIMER;
400 val &= pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.allowed1;
401
402 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PIN_EXEC_CONTROLS, val);
403 AssertRC(rc);
404
405 /* VMX_VMCS_CTRL_PROC_EXEC_CONTROLS
406 * Set required bits to one and zero according to the MSR capabilities.
407 */
408 val = pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.disallowed0;
409 /* Program which event cause VM-exits and which features we want to use. */
410 val = val | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_HLT_EXIT
411 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_TSC_OFFSET
412 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT
413 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_UNCOND_IO_EXIT
414 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDPMC_EXIT
415 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_EXIT
416 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MWAIT_EXIT; /* don't execute mwait or else we'll idle inside the guest (host thinks the cpu load is high) */
417
418 /* Without nested paging we should intercept invlpg and cr3 mov instructions. */
419 if (!pVM->hwaccm.s.fNestedPaging)
420 val |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT
421 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
422 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
423
424 /* Note: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MWAIT_EXIT might cause a vmlaunch failure with an invalid control fields error. (combined with some other exit reasons) */
425 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
426 {
427 /* CR8 reads from the APIC shadow page; writes cause an exit is they lower the TPR below the threshold */
428 val |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW;
429 Assert(pVM->hwaccm.s.vmx.pAPIC);
430 }
431 else
432 /* Exit on CR8 reads & writes in case the TPR shadow feature isn't present. */
433 val |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_STORE_EXIT | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_LOAD_EXIT;
434
435 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
436 {
437 Assert(pVCpu->hwaccm.s.vmx.pMSRBitmapPhys);
438 val |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS;
439 }
440
441 /* We will use the secondary control if it's present. */
442 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
443
444 /* Mask away the bits that the CPU doesn't support */
445 /** @todo make sure they don't conflict with the above requirements. */
446 val &= pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1;
447 pVCpu->hwaccm.s.vmx.proc_ctls = val;
448
449 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, val);
450 AssertRC(rc);
451
452 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
453 {
454 /* VMX_VMCS_CTRL_PROC_EXEC_CONTROLS2
455 * Set required bits to one and zero according to the MSR capabilities.
456 */
457 val = pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0;
458 val |= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT;
459
460#ifdef HWACCM_VTX_WITH_EPT
461 if (pVM->hwaccm.s.fNestedPaging)
462 val |= VMX_VMCS_CTRL_PROC_EXEC2_EPT;
463#endif /* HWACCM_VTX_WITH_EPT */
464#ifdef HWACCM_VTX_WITH_VPID
465 else
466 if (pVM->hwaccm.s.vmx.fVPID)
467 val |= VMX_VMCS_CTRL_PROC_EXEC2_VPID;
468#endif /* HWACCM_VTX_WITH_VPID */
469
470 if (pVM->hwaccm.s.fHasIoApic)
471 val |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC;
472
473 if (pVM->hwaccm.s.vmx.fUnrestrictedGuest)
474 val |= VMX_VMCS_CTRL_PROC_EXEC2_REAL_MODE;
475
476 /* Mask away the bits that the CPU doesn't support */
477 /** @todo make sure they don't conflict with the above requirements. */
478 val &= pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1;
479 pVCpu->hwaccm.s.vmx.proc_ctls2 = val;
480 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS2, val);
481 AssertRC(rc);
482 }
483
484 /* VMX_VMCS_CTRL_CR3_TARGET_COUNT
485 * Set required bits to one and zero according to the MSR capabilities.
486 */
487 rc = VMXWriteVMCS(VMX_VMCS_CTRL_CR3_TARGET_COUNT, 0);
488 AssertRC(rc);
489
490 /* Forward all exception except #NM & #PF to the guest.
491 * We always need to check pagefaults since our shadow page table can be out of sync.
492 * And we always lazily sync the FPU & XMM state.
493 */
494
495 /** @todo Possible optimization:
496 * Keep the FPU and XMM state current in the EM thread. That way there's no need to
497 * lazily sync anything, but the downside is that we can't use the FPU stack or XMM
498 * registers ourselves of course.
499 *
500 * Note: only possible if the current state is actually ours (X86_CR0_TS flag)
501 */
502
503 /* Don't filter page faults; all of them should cause a switch. */
504 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PAGEFAULT_ERROR_MASK, 0);
505 rc |= VMXWriteVMCS(VMX_VMCS_CTRL_PAGEFAULT_ERROR_MATCH, 0);
506 AssertRC(rc);
507
508 /* Init TSC offset to zero. */
509 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_TSC_OFFSET_FULL, 0);
510 AssertRC(rc);
511
512 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_IO_BITMAP_A_FULL, 0);
513 AssertRC(rc);
514
515 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_IO_BITMAP_B_FULL, 0);
516 AssertRC(rc);
517
518 /* Set the MSR bitmap address. */
519 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
520 {
521 Assert(pVCpu->hwaccm.s.vmx.pMSRBitmapPhys);
522
523 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_MSR_BITMAP_FULL, pVCpu->hwaccm.s.vmx.pMSRBitmapPhys);
524 AssertRC(rc);
525
526 /* Allow the guest to directly modify these MSRs; they are restored and saved automatically. */
527 hmR0VmxSetMSRPermission(pVCpu, MSR_IA32_SYSENTER_CS, true, true);
528 hmR0VmxSetMSRPermission(pVCpu, MSR_IA32_SYSENTER_ESP, true, true);
529 hmR0VmxSetMSRPermission(pVCpu, MSR_IA32_SYSENTER_EIP, true, true);
530 hmR0VmxSetMSRPermission(pVCpu, MSR_K8_LSTAR, true, true);
531 hmR0VmxSetMSRPermission(pVCpu, MSR_K6_STAR, true, true);
532 hmR0VmxSetMSRPermission(pVCpu, MSR_K8_SF_MASK, true, true);
533 hmR0VmxSetMSRPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, true, true);
534 hmR0VmxSetMSRPermission(pVCpu, MSR_K8_GS_BASE, true, true);
535 hmR0VmxSetMSRPermission(pVCpu, MSR_K8_FS_BASE, true, true);
536 }
537
538#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
539 /* Set the guest & host MSR load/store physical addresses. */
540 Assert(pVCpu->hwaccm.s.vmx.pGuestMSRPhys);
541 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_VMENTRY_MSR_LOAD_FULL, pVCpu->hwaccm.s.vmx.pGuestMSRPhys);
542 AssertRC(rc);
543 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_VMEXIT_MSR_STORE_FULL, pVCpu->hwaccm.s.vmx.pGuestMSRPhys);
544 AssertRC(rc);
545
546 Assert(pVCpu->hwaccm.s.vmx.pHostMSRPhys);
547 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_VMEXIT_MSR_LOAD_FULL, pVCpu->hwaccm.s.vmx.pHostMSRPhys);
548 AssertRC(rc);
549#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
550
551 rc = VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
552 AssertRC(rc);
553
554 rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_MSR_STORE_COUNT, 0);
555 AssertRC(rc);
556
557 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
558 {
559 Assert(pVM->hwaccm.s.vmx.pMemObjAPIC);
560 /* Optional */
561 rc = VMXWriteVMCS(VMX_VMCS_CTRL_TPR_THRESHOLD, 0);
562 rc |= VMXWriteVMCS64(VMX_VMCS_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hwaccm.s.vmx.HCPhysVAPIC);
563
564 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
565 rc |= VMXWriteVMCS64(VMX_VMCS_CTRL_APIC_ACCESSADDR_FULL, pVM->hwaccm.s.vmx.pAPICPhys);
566
567 AssertRC(rc);
568 }
569
570 /* Set link pointer to -1. Not currently used. */
571 rc = VMXWriteVMCS64(VMX_VMCS_GUEST_LINK_PTR_FULL, 0xFFFFFFFFFFFFFFFFULL);
572 AssertRC(rc);
573
574 /* Clear VM Control Structure. Marking it inactive, clearing implementation specific data and writing back VMCS data to memory. */
575 rc = VMXClearVMCS(pVCpu->hwaccm.s.vmx.HCPhysVMCS);
576 AssertRC(rc);
577
578 /* Configure the VMCS read cache. */
579 PVMCSCACHE pCache = &pVCpu->hwaccm.s.vmx.VMCSCache;
580
581 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_RIP);
582 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_RSP);
583 VMXSetupCachedReadVMCS(pCache, VMX_VMCS_GUEST_RFLAGS);
584 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE);
585 VMXSetupCachedReadVMCS(pCache, VMX_VMCS_CTRL_CR0_READ_SHADOW);
586 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_CR0);
587 VMXSetupCachedReadVMCS(pCache, VMX_VMCS_CTRL_CR4_READ_SHADOW);
588 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_CR4);
589 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_DR7);
590 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_SYSENTER_CS);
591 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_SYSENTER_EIP);
592 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_SYSENTER_ESP);
593 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_GDTR_LIMIT);
594 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_GDTR_BASE);
595 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_IDTR_LIMIT);
596 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_IDTR_BASE);
597
598 VMX_SETUP_SELREG(ES, pCache);
599 VMX_SETUP_SELREG(SS, pCache);
600 VMX_SETUP_SELREG(CS, pCache);
601 VMX_SETUP_SELREG(DS, pCache);
602 VMX_SETUP_SELREG(FS, pCache);
603 VMX_SETUP_SELREG(GS, pCache);
604 VMX_SETUP_SELREG(LDTR, pCache);
605 VMX_SETUP_SELREG(TR, pCache);
606
607 /* Status code VMCS reads. */
608 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_REASON);
609 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_VM_INSTR_ERROR);
610 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INSTR_LENGTH);
611 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INTERRUPTION_ERRCODE);
612 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO);
613 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INSTR_INFO);
614 VMXSetupCachedReadVMCS(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
615 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_IDT_INFO);
616 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_IDT_ERRCODE);
617
618 if (pVM->hwaccm.s.fNestedPaging)
619 {
620 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_CR3);
621 VMXSetupCachedReadVMCS(pCache, VMX_VMCS_EXIT_PHYS_ADDR_FULL);
622 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
623 }
624 else
625 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
626 } /* for each VMCPU */
627
628 /* Choose the right TLB setup function. */
629 if (pVM->hwaccm.s.fNestedPaging)
630 {
631 pVM->hwaccm.s.vmx.pfnSetupTaggedTLB = hmR0VmxSetupTLBEPT;
632
633 /* Default values for flushing. */
634 pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_ALL_CONTEXTS;
635 pVM->hwaccm.s.vmx.enmFlushContext = VMX_FLUSH_ALL_CONTEXTS;
636
637 /* If the capabilities specify we can do more, then make use of it. */
638 if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVEPT_CAPS_INDIV)
639 pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_PAGE;
640 else
641 if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVEPT_CAPS_CONTEXT)
642 pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_SINGLE_CONTEXT;
643
644 if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVEPT_CAPS_CONTEXT)
645 pVM->hwaccm.s.vmx.enmFlushContext = VMX_FLUSH_SINGLE_CONTEXT;
646 }
647#ifdef HWACCM_VTX_WITH_VPID
648 else
649 if (pVM->hwaccm.s.vmx.fVPID)
650 {
651 pVM->hwaccm.s.vmx.pfnSetupTaggedTLB = hmR0VmxSetupTLBVPID;
652
653 /* Default values for flushing. */
654 pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_ALL_CONTEXTS;
655 pVM->hwaccm.s.vmx.enmFlushContext = VMX_FLUSH_ALL_CONTEXTS;
656
657 /* If the capabilities specify we can do more, then make use of it. */
658 if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_INDIV)
659 pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_PAGE;
660 else
661 if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_CONTEXT)
662 pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_SINGLE_CONTEXT;
663
664 if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_CONTEXT)
665 pVM->hwaccm.s.vmx.enmFlushContext = VMX_FLUSH_SINGLE_CONTEXT;
666 }
667#endif /* HWACCM_VTX_WITH_VPID */
668 else
669 pVM->hwaccm.s.vmx.pfnSetupTaggedTLB = hmR0VmxSetupTLBDummy;
670
671vmx_end:
672 hmR0VmxCheckError(pVM, &pVM->aCpus[0], rc);
673 return rc;
674}
675
676/**
677 * Sets the permission bits for the specified MSR
678 *
679 * @param pVCpu The VMCPU to operate on.
680 * @param ulMSR MSR value
681 * @param fRead Reading allowed/disallowed
682 * @param fWrite Writing allowed/disallowed
683 */
684static void hmR0VmxSetMSRPermission(PVMCPU pVCpu, unsigned ulMSR, bool fRead, bool fWrite)
685{
686 unsigned ulBit;
687 uint8_t *pMSRBitmap = (uint8_t *)pVCpu->hwaccm.s.vmx.pMSRBitmap;
688
689 /* Layout:
690 * 0x000 - 0x3ff - Low MSR read bits
691 * 0x400 - 0x7ff - High MSR read bits
692 * 0x800 - 0xbff - Low MSR write bits
693 * 0xc00 - 0xfff - High MSR write bits
694 */
695 if (ulMSR <= 0x00001FFF)
696 {
697 /* Pentium-compatible MSRs */
698 ulBit = ulMSR;
699 }
700 else
701 if ( ulMSR >= 0xC0000000
702 && ulMSR <= 0xC0001FFF)
703 {
704 /* AMD Sixth Generation x86 Processor MSRs */
705 ulBit = (ulMSR - 0xC0000000);
706 pMSRBitmap += 0x400;
707 }
708 else
709 {
710 AssertFailed();
711 return;
712 }
713
714 Assert(ulBit <= 0x1fff);
715 if (fRead)
716 ASMBitClear(pMSRBitmap, ulBit);
717 else
718 ASMBitSet(pMSRBitmap, ulBit);
719
720 if (fWrite)
721 ASMBitClear(pMSRBitmap + 0x800, ulBit);
722 else
723 ASMBitSet(pMSRBitmap + 0x800, ulBit);
724}
725
726
727/**
728 * Injects an event (trap or external interrupt)
729 *
730 * @returns VBox status code. Note that it may return VINF_EM_RESET to
731 * indicate a triple fault when injecting X86_XCPT_DF.
732 *
733 * @param pVM The VM to operate on.
734 * @param pVCpu The VMCPU to operate on.
735 * @param pCtx CPU Context
736 * @param intInfo VMX interrupt info
737 * @param cbInstr Opcode length of faulting instruction
738 * @param errCode Error code (optional)
739 */
740static int hmR0VmxInjectEvent(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, uint32_t intInfo, uint32_t cbInstr, uint32_t errCode)
741{
742 int rc;
743 uint32_t iGate = VMX_EXIT_INTERRUPTION_INFO_VECTOR(intInfo);
744
745#ifdef VBOX_WITH_STATISTICS
746 STAM_COUNTER_INC(&pVCpu->hwaccm.s.paStatInjectedIrqsR0[iGate & MASK_INJECT_IRQ_STAT]);
747#endif
748
749#ifdef VBOX_STRICT
750 if (iGate == 0xE)
751 LogFlow(("hmR0VmxInjectEvent: Injecting interrupt %d at %RGv error code=%08x CR2=%RGv intInfo=%08x\n", iGate, (RTGCPTR)pCtx->rip, errCode, pCtx->cr2, intInfo));
752 else
753 if (iGate < 0x20)
754 LogFlow(("hmR0VmxInjectEvent: Injecting interrupt %d at %RGv error code=%08x\n", iGate, (RTGCPTR)pCtx->rip, errCode));
755 else
756 {
757 LogFlow(("INJ-EI: %x at %RGv\n", iGate, (RTGCPTR)pCtx->rip));
758 Assert(VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW || !VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
759 Assert(VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW || pCtx->eflags.u32 & X86_EFL_IF);
760 }
761#endif
762
763 if ( CPUMIsGuestInRealModeEx(pCtx)
764 && pVM->hwaccm.s.vmx.pRealModeTSS)
765 {
766 RTGCPHYS GCPhysHandler;
767 uint16_t offset, ip;
768 RTSEL sel;
769
770 /* Injecting events doesn't work right with real mode emulation.
771 * (#GP if we try to inject external hardware interrupts)
772 * Inject the interrupt or trap directly instead.
773 *
774 * ASSUMES no access handlers for the bits we read or write below (should be safe).
775 */
776 Log(("Manual interrupt/trap '%x' inject (real mode)\n", iGate));
777
778 /* Check if the interrupt handler is present. */
779 if (iGate * 4 + 3 > pCtx->idtr.cbIdt)
780 {
781 Log(("IDT cbIdt violation\n"));
782 if (iGate != X86_XCPT_DF)
783 {
784 uint32_t intInfo2;
785
786 intInfo2 = (iGate == X86_XCPT_GP) ? (uint32_t)X86_XCPT_DF : iGate;
787 intInfo2 |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
788 intInfo2 |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
789 intInfo2 |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
790
791 return hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo2, 0, 0 /* no error code according to the Intel docs */);
792 }
793 Log(("Triple fault -> reset the VM!\n"));
794 return VINF_EM_RESET;
795 }
796 if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW
797 || iGate == 3 /* Both #BP and #OF point to the instruction after. */
798 || iGate == 4)
799 {
800 ip = pCtx->ip + cbInstr;
801 }
802 else
803 ip = pCtx->ip;
804
805 /* Read the selector:offset pair of the interrupt handler. */
806 GCPhysHandler = (RTGCPHYS)pCtx->idtr.pIdt + iGate * 4;
807 rc = PGMPhysSimpleReadGCPhys(pVM, &offset, GCPhysHandler, sizeof(offset)); AssertRC(rc);
808 rc = PGMPhysSimpleReadGCPhys(pVM, &sel, GCPhysHandler + 2, sizeof(sel)); AssertRC(rc);
809
810 LogFlow(("IDT handler %04X:%04X\n", sel, offset));
811
812 /* Construct the stack frame. */
813 /** @todo should check stack limit. */
814 pCtx->sp -= 2;
815 LogFlow(("ss:sp %04X:%04X eflags=%x\n", pCtx->ss, pCtx->sp, pCtx->eflags.u));
816 rc = PGMPhysSimpleWriteGCPhys(pVM, pCtx->ssHid.u64Base + pCtx->sp, &pCtx->eflags, sizeof(uint16_t)); AssertRC(rc);
817 pCtx->sp -= 2;
818 LogFlow(("ss:sp %04X:%04X cs=%x\n", pCtx->ss, pCtx->sp, pCtx->cs));
819 rc = PGMPhysSimpleWriteGCPhys(pVM, pCtx->ssHid.u64Base + pCtx->sp, &pCtx->cs, sizeof(uint16_t)); AssertRC(rc);
820 pCtx->sp -= 2;
821 LogFlow(("ss:sp %04X:%04X ip=%x\n", pCtx->ss, pCtx->sp, ip));
822 rc = PGMPhysSimpleWriteGCPhys(pVM, pCtx->ssHid.u64Base + pCtx->sp, &ip, sizeof(ip)); AssertRC(rc);
823
824 /* Update the CPU state for executing the handler. */
825 pCtx->rip = offset;
826 pCtx->cs = sel;
827 pCtx->csHid.u64Base = sel << 4;
828 pCtx->eflags.u &= ~(X86_EFL_IF|X86_EFL_TF|X86_EFL_RF|X86_EFL_AC);
829
830 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_SEGMENT_REGS;
831 return VINF_SUCCESS;
832 }
833
834 /* Set event injection state. */
835 rc = VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_IRQ_INFO, intInfo | (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT));
836
837 rc |= VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
838 rc |= VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_EXCEPTION_ERRCODE, errCode);
839
840 AssertRC(rc);
841 return rc;
842}
843
844
845/**
846 * Checks for pending guest interrupts and injects them
847 *
848 * @returns VBox status code.
849 * @param pVM The VM to operate on.
850 * @param pVCpu The VMCPU to operate on.
851 * @param pCtx CPU Context
852 */
853static int hmR0VmxCheckPendingInterrupt(PVM pVM, PVMCPU pVCpu, CPUMCTX *pCtx)
854{
855 int rc;
856
857 /* Dispatch any pending interrupts. (injected before, but a VM exit occurred prematurely) */
858 if (pVCpu->hwaccm.s.Event.fPending)
859 {
860 Log(("CPU%d: Reinjecting event %RX64 %08x at %RGv cr2=%RX64\n", pVCpu->idCpu, pVCpu->hwaccm.s.Event.intInfo, pVCpu->hwaccm.s.Event.errCode, (RTGCPTR)pCtx->rip, pCtx->cr2));
861 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatIntReinject);
862 rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, pVCpu->hwaccm.s.Event.intInfo, 0, pVCpu->hwaccm.s.Event.errCode);
863 AssertRC(rc);
864
865 pVCpu->hwaccm.s.Event.fPending = false;
866 return VINF_SUCCESS;
867 }
868
869 /* If an active trap is already pending, then we must forward it first! */
870 if (!TRPMHasTrap(pVCpu))
871 {
872 if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI))
873 {
874 RTGCUINTPTR intInfo;
875
876 Log(("CPU%d: injecting #NMI\n", pVCpu->idCpu));
877
878 intInfo = X86_XCPT_NMI;
879 intInfo |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
880 intInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
881
882 rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo, 0, 0);
883 AssertRC(rc);
884
885 return VINF_SUCCESS;
886 }
887
888 /* @todo SMI interrupts. */
889
890 /* When external interrupts are pending, we should exit the VM when IF is set. */
891 if (VMCPU_FF_ISPENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC|VMCPU_FF_INTERRUPT_PIC)))
892 {
893 if (!(pCtx->eflags.u32 & X86_EFL_IF))
894 {
895 if (!(pVCpu->hwaccm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_IRQ_WINDOW_EXIT))
896 {
897 LogFlow(("Enable irq window exit!\n"));
898 pVCpu->hwaccm.s.vmx.proc_ctls |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_IRQ_WINDOW_EXIT;
899 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
900 AssertRC(rc);
901 }
902 /* else nothing to do but wait */
903 }
904 else
905 if (!VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
906 {
907 uint8_t u8Interrupt;
908
909 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
910 Log(("CPU%d: Dispatch interrupt: u8Interrupt=%x (%d) rc=%Rrc cs:rip=%04X:%RGv\n", pVCpu->idCpu, u8Interrupt, u8Interrupt, rc, pCtx->cs, (RTGCPTR)pCtx->rip));
911 if (RT_SUCCESS(rc))
912 {
913 rc = TRPMAssertTrap(pVCpu, u8Interrupt, TRPM_HARDWARE_INT);
914 AssertRC(rc);
915 }
916 else
917 {
918 /* Can only happen in rare cases where a pending interrupt is cleared behind our back */
919 Assert(!VMCPU_FF_ISPENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC|VMCPU_FF_INTERRUPT_PIC)));
920 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatSwitchGuestIrq);
921 /* Just continue */
922 }
923 }
924 else
925 Log(("Pending interrupt blocked at %RGv by VM_FF_INHIBIT_INTERRUPTS!!\n", (RTGCPTR)pCtx->rip));
926 }
927 }
928
929#ifdef VBOX_STRICT
930 if (TRPMHasTrap(pVCpu))
931 {
932 uint8_t u8Vector;
933 rc = TRPMQueryTrapAll(pVCpu, &u8Vector, 0, 0, 0);
934 AssertRC(rc);
935 }
936#endif
937
938 if ( (pCtx->eflags.u32 & X86_EFL_IF)
939 && (!VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
940 && TRPMHasTrap(pVCpu)
941 )
942 {
943 uint8_t u8Vector;
944 TRPMEVENT enmType;
945 RTGCUINTPTR intInfo;
946 RTGCUINT errCode;
947
948 /* If a new event is pending, then dispatch it now. */
949 rc = TRPMQueryTrapAll(pVCpu, &u8Vector, &enmType, &errCode, 0);
950 AssertRC(rc);
951 Assert(pCtx->eflags.Bits.u1IF == 1 || enmType == TRPM_TRAP);
952 Assert(enmType != TRPM_SOFTWARE_INT);
953
954 /* Clear the pending trap. */
955 rc = TRPMResetTrap(pVCpu);
956 AssertRC(rc);
957
958 intInfo = u8Vector;
959 intInfo |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
960
961 if (enmType == TRPM_TRAP)
962 {
963 switch (u8Vector) {
964 case 8:
965 case 10:
966 case 11:
967 case 12:
968 case 13:
969 case 14:
970 case 17:
971 /* Valid error codes. */
972 intInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
973 break;
974 default:
975 break;
976 }
977 if (u8Vector == X86_XCPT_BP || u8Vector == X86_XCPT_OF)
978 intInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
979 else
980 intInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
981 }
982 else
983 intInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
984
985 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatIntInject);
986 rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo, 0, errCode);
987 AssertRC(rc);
988 } /* if (interrupts can be dispatched) */
989
990 return VINF_SUCCESS;
991}
992
993/**
994 * Save the host state
995 *
996 * @returns VBox status code.
997 * @param pVM The VM to operate on.
998 * @param pVCpu The VMCPU to operate on.
999 */
1000VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
1001{
1002 int rc = VINF_SUCCESS;
1003 NOREF(pVM);
1004
1005 /*
1006 * Host CPU Context
1007 */
1008 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_HOST_CONTEXT)
1009 {
1010 RTIDTR idtr;
1011 RTGDTR gdtr;
1012 RTSEL SelTR;
1013 PCX86DESCHC pDesc;
1014 uintptr_t trBase;
1015 RTSEL cs;
1016 RTSEL ss;
1017 uint64_t cr3;
1018
1019 /* Control registers */
1020 rc = VMXWriteVMCS(VMX_VMCS_HOST_CR0, ASMGetCR0());
1021#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1022 if (VMX_IS_64BIT_HOST_MODE())
1023 {
1024 cr3 = hwaccmR0Get64bitCR3();
1025 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_CR3, cr3);
1026 }
1027 else
1028#endif
1029 {
1030 cr3 = ASMGetCR3();
1031 rc |= VMXWriteVMCS(VMX_VMCS_HOST_CR3, cr3);
1032 }
1033 rc |= VMXWriteVMCS(VMX_VMCS_HOST_CR4, ASMGetCR4());
1034 AssertRC(rc);
1035 Log2(("VMX_VMCS_HOST_CR0 %08x\n", ASMGetCR0()));
1036 Log2(("VMX_VMCS_HOST_CR3 %08RX64\n", cr3));
1037 Log2(("VMX_VMCS_HOST_CR4 %08x\n", ASMGetCR4()));
1038
1039 /* Selector registers. */
1040#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1041 if (VMX_IS_64BIT_HOST_MODE())
1042 {
1043 cs = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
1044 ss = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
1045 }
1046 else
1047 {
1048 /* sysenter loads LDT cs & ss, VMX doesn't like this. Load the GDT ones (safe). */
1049 cs = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
1050 ss = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
1051 }
1052#else
1053 cs = ASMGetCS();
1054 ss = ASMGetSS();
1055#endif
1056 Assert(!(cs & X86_SEL_LDT)); Assert((cs & X86_SEL_RPL) == 0);
1057 Assert(!(ss & X86_SEL_LDT)); Assert((ss & X86_SEL_RPL) == 0);
1058 rc = VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_CS, cs);
1059 /* Note: VMX is (again) very picky about the RPL of the selectors here; we'll restore them manually. */
1060 rc |= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_DS, 0);
1061 rc |= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_ES, 0);
1062#if HC_ARCH_BITS == 32
1063 if (!VMX_IS_64BIT_HOST_MODE())
1064 {
1065 rc |= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_FS, 0);
1066 rc |= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_GS, 0);
1067 }
1068#endif
1069 rc |= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_SS, ss);
1070 SelTR = ASMGetTR();
1071 rc |= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_TR, SelTR);
1072 AssertRC(rc);
1073 Log2(("VMX_VMCS_HOST_FIELD_CS %08x (%08x)\n", cs, ASMGetSS()));
1074 Log2(("VMX_VMCS_HOST_FIELD_DS 00000000 (%08x)\n", ASMGetDS()));
1075 Log2(("VMX_VMCS_HOST_FIELD_ES 00000000 (%08x)\n", ASMGetES()));
1076 Log2(("VMX_VMCS_HOST_FIELD_FS 00000000 (%08x)\n", ASMGetFS()));
1077 Log2(("VMX_VMCS_HOST_FIELD_GS 00000000 (%08x)\n", ASMGetGS()));
1078 Log2(("VMX_VMCS_HOST_FIELD_SS %08x (%08x)\n", ss, ASMGetSS()));
1079 Log2(("VMX_VMCS_HOST_FIELD_TR %08x\n", ASMGetTR()));
1080
1081 /* GDTR & IDTR */
1082#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1083 if (VMX_IS_64BIT_HOST_MODE())
1084 {
1085 X86XDTR64 gdtr64, idtr64;
1086 hwaccmR0Get64bitGDTRandIDTR(&gdtr64, &idtr64);
1087 rc = VMXWriteVMCS64(VMX_VMCS_HOST_GDTR_BASE, gdtr64.uAddr);
1088 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_IDTR_BASE, gdtr64.uAddr);
1089 AssertRC(rc);
1090 Log2(("VMX_VMCS_HOST_GDTR_BASE %RX64\n", gdtr64.uAddr));
1091 Log2(("VMX_VMCS_HOST_IDTR_BASE %RX64\n", idtr64.uAddr));
1092 gdtr.cbGdt = gdtr64.cb;
1093 gdtr.pGdt = (uintptr_t)gdtr64.uAddr;
1094 }
1095 else
1096#endif
1097 {
1098 ASMGetGDTR(&gdtr);
1099 rc = VMXWriteVMCS(VMX_VMCS_HOST_GDTR_BASE, gdtr.pGdt);
1100 ASMGetIDTR(&idtr);
1101 rc |= VMXWriteVMCS(VMX_VMCS_HOST_IDTR_BASE, idtr.pIdt);
1102 AssertRC(rc);
1103 Log2(("VMX_VMCS_HOST_GDTR_BASE %RHv\n", gdtr.pGdt));
1104 Log2(("VMX_VMCS_HOST_IDTR_BASE %RHv\n", idtr.pIdt));
1105 }
1106
1107 /* Save the base address of the TR selector. */
1108 if (SelTR > gdtr.cbGdt)
1109 {
1110 AssertMsgFailed(("Invalid TR selector %x. GDTR.cbGdt=%x\n", SelTR, gdtr.cbGdt));
1111 return VERR_VMX_INVALID_HOST_STATE;
1112 }
1113
1114 pDesc = (PCX86DESCHC)(gdtr.pGdt + (SelTR & X86_SEL_MASK));
1115#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1116 if (VMX_IS_64BIT_HOST_MODE())
1117 {
1118 uint64_t trBase64 = X86DESC64_BASE(*(PX86DESC64)pDesc);
1119 rc = VMXWriteVMCS64(VMX_VMCS_HOST_TR_BASE, trBase64);
1120 Log2(("VMX_VMCS_HOST_TR_BASE %RX64\n", trBase64));
1121 AssertRC(rc);
1122 }
1123 else
1124#endif
1125 {
1126#if HC_ARCH_BITS == 64
1127 trBase = X86DESC64_BASE(*pDesc);
1128#else
1129 trBase = X86DESC_BASE(*pDesc);
1130#endif
1131 rc = VMXWriteVMCS(VMX_VMCS_HOST_TR_BASE, trBase);
1132 AssertRC(rc);
1133 Log2(("VMX_VMCS_HOST_TR_BASE %RHv\n", trBase));
1134 }
1135
1136 /* FS and GS base. */
1137#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1138 if (VMX_IS_64BIT_HOST_MODE())
1139 {
1140 Log2(("MSR_K8_FS_BASE = %RX64\n", ASMRdMsr(MSR_K8_FS_BASE)));
1141 Log2(("MSR_K8_GS_BASE = %RX64\n", ASMRdMsr(MSR_K8_GS_BASE)));
1142 rc = VMXWriteVMCS64(VMX_VMCS_HOST_FS_BASE, ASMRdMsr(MSR_K8_FS_BASE));
1143 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_GS_BASE, ASMRdMsr(MSR_K8_GS_BASE));
1144 }
1145#endif
1146 AssertRC(rc);
1147
1148 /* Sysenter MSRs. */
1149 /** @todo expensive!! */
1150 rc = VMXWriteVMCS(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
1151 Log2(("VMX_VMCS_HOST_SYSENTER_CS %08x\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_CS)));
1152#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1153 if (VMX_IS_64BIT_HOST_MODE())
1154 {
1155 Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_EIP)));
1156 Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_ESP)));
1157 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
1158 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
1159 }
1160 else
1161 {
1162 rc |= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
1163 rc |= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
1164 Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP)));
1165 Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP)));
1166 }
1167#elif HC_ARCH_BITS == 32
1168 rc |= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
1169 rc |= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
1170 Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP)));
1171 Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP)));
1172#else
1173 Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_EIP)));
1174 Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_ESP)));
1175 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
1176 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
1177#endif
1178 AssertRC(rc);
1179
1180#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
1181 /* Store all host MSRs in the VM-Exit load area, so they will be reloaded after the world switch back to the host. */
1182 PVMXMSR pMsr = (PVMXMSR)pVCpu->hwaccm.s.vmx.pHostMSR;
1183 unsigned idxMsr = 0;
1184
1185 /* EFER MSR present? */
1186 if (ASMCpuId_EDX(0x80000001) & (X86_CPUID_AMD_FEATURE_EDX_NX|X86_CPUID_AMD_FEATURE_EDX_LONG_MODE))
1187 {
1188 if (ASMCpuId_EDX(0x80000001) & X86_CPUID_AMD_FEATURE_EDX_SEP)
1189 {
1190 pMsr->u32IndexMSR = MSR_K6_STAR;
1191 pMsr->u32Reserved = 0;
1192 pMsr->u64Value = ASMRdMsr(MSR_K6_STAR); /* legacy syscall eip, cs & ss */
1193 pMsr++; idxMsr++;
1194 }
1195
1196 pMsr->u32IndexMSR = MSR_K6_EFER;
1197 pMsr->u32Reserved = 0;
1198# if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1199 if (CPUMIsGuestInLongMode(pVCpu))
1200 {
1201 /* Must match the efer value in our 64 bits switcher. */
1202 pMsr->u64Value = ASMRdMsr(MSR_K6_EFER) | MSR_K6_EFER_LME | MSR_K6_EFER_SCE | MSR_K6_EFER_NXE;
1203 }
1204 else
1205# endif
1206 pMsr->u64Value = ASMRdMsr(MSR_K6_EFER);
1207 pMsr++; idxMsr++;
1208 }
1209
1210# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1211 if (VMX_IS_64BIT_HOST_MODE())
1212 {
1213 pMsr->u32IndexMSR = MSR_K8_LSTAR;
1214 pMsr->u32Reserved = 0;
1215 pMsr->u64Value = ASMRdMsr(MSR_K8_LSTAR); /* 64 bits mode syscall rip */
1216 pMsr++; idxMsr++;
1217 pMsr->u32IndexMSR = MSR_K8_SF_MASK;
1218 pMsr->u32Reserved = 0;
1219 pMsr->u64Value = ASMRdMsr(MSR_K8_SF_MASK); /* syscall flag mask */
1220 pMsr++; idxMsr++;
1221 pMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
1222 pMsr->u32Reserved = 0;
1223 pMsr->u64Value = ASMRdMsr(MSR_K8_KERNEL_GS_BASE); /* swapgs exchange value */
1224 pMsr++; idxMsr++;
1225 }
1226# endif
1227 rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_MSR_LOAD_COUNT, idxMsr);
1228 AssertRC(rc);
1229#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
1230
1231 pVCpu->hwaccm.s.fContextUseFlags &= ~HWACCM_CHANGED_HOST_CONTEXT;
1232 }
1233 return rc;
1234}
1235
1236/**
1237 * Loads the 4 PDPEs into the guest state when nested paging is used and the
1238 * guest operates in PAE mode.
1239 *
1240 * @returns VINF_SUCCESS or fatal error.
1241 * @param pVCpu The VMCPU to operate on.
1242 * @param pCtx Guest context
1243 */
1244static int hmR0VmxLoadPaePdpes(PVMCPU pVCpu, PCPUMCTX pCtx)
1245{
1246 if (CPUMIsGuestInPAEModeEx(pCtx))
1247 {
1248 X86PDPE aPdpes[4];
1249 int rc = PGMGstGetPaePdpes(pVCpu, &aPdpes[0]);
1250 AssertRCReturn(rc, rc);
1251
1252 rc = VMXWriteVMCS64(VMX_VMCS_GUEST_PDPTR0_FULL, aPdpes[0].u); AssertRCReturn(rc, rc);
1253 rc = VMXWriteVMCS64(VMX_VMCS_GUEST_PDPTR1_FULL, aPdpes[1].u); AssertRCReturn(rc, rc);
1254 rc = VMXWriteVMCS64(VMX_VMCS_GUEST_PDPTR2_FULL, aPdpes[2].u); AssertRCReturn(rc, rc);
1255 rc = VMXWriteVMCS64(VMX_VMCS_GUEST_PDPTR3_FULL, aPdpes[3].u); AssertRCReturn(rc, rc);
1256 }
1257 return VINF_SUCCESS;
1258}
1259
1260/**
1261 * Saves the 4 PDPEs into the guest state when nested paging is used and the
1262 * guest operates in PAE mode.
1263 *
1264 * @returns VINF_SUCCESS or fatal error.
1265 * @param pVCpu The VMCPU to operate on.
1266 * @param pCtx Guest context
1267 *
1268 * @remarks Tell PGM about CR3 changes before calling this helper.
1269 */
1270static int hmR0VmxSavePaePdpes(PVMCPU pVCpu, PCPUMCTX pCtx)
1271{
1272 if (CPUMIsGuestInPAEModeEx(pCtx))
1273 {
1274 int rc;
1275 X86PDPE aPdpes[4];
1276 rc = VMXReadVMCS64(VMX_VMCS_GUEST_PDPTR0_FULL, &aPdpes[0].u); AssertRCReturn(rc, rc);
1277 rc = VMXReadVMCS64(VMX_VMCS_GUEST_PDPTR1_FULL, &aPdpes[1].u); AssertRCReturn(rc, rc);
1278 rc = VMXReadVMCS64(VMX_VMCS_GUEST_PDPTR2_FULL, &aPdpes[2].u); AssertRCReturn(rc, rc);
1279 rc = VMXReadVMCS64(VMX_VMCS_GUEST_PDPTR3_FULL, &aPdpes[3].u); AssertRCReturn(rc, rc);
1280
1281 rc = PGMGstUpdatePaePdpes(pVCpu, &aPdpes[0]);
1282 AssertRCReturn(rc, rc);
1283 }
1284 return VINF_SUCCESS;
1285}
1286
1287
1288/**
1289 * Update the exception bitmap according to the current CPU state
1290 *
1291 * @param pVM The VM to operate on.
1292 * @param pVCpu The VMCPU to operate on.
1293 * @param pCtx Guest context
1294 */
1295static void hmR0VmxUpdateExceptionBitmap(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1296{
1297 uint32_t u32TrapMask;
1298 Assert(pCtx);
1299
1300 u32TrapMask = HWACCM_VMX_TRAP_MASK;
1301#ifndef DEBUG
1302 if (pVM->hwaccm.s.fNestedPaging)
1303 u32TrapMask &= ~RT_BIT(X86_XCPT_PF); /* no longer need to intercept #PF. */
1304#endif
1305
1306 /* Also catch floating point exceptions as we need to report them to the guest in a different way. */
1307 if ( CPUMIsGuestFPUStateActive(pVCpu) == true
1308 && !(pCtx->cr0 & X86_CR0_NE)
1309 && !pVCpu->hwaccm.s.fFPUOldStyleOverride)
1310 {
1311 u32TrapMask |= RT_BIT(X86_XCPT_MF);
1312 pVCpu->hwaccm.s.fFPUOldStyleOverride = true;
1313 }
1314
1315#ifdef VBOX_STRICT
1316 Assert(u32TrapMask & RT_BIT(X86_XCPT_GP));
1317#endif
1318
1319 /* Intercept all exceptions in real mode as none of them can be injected directly (#GP otherwise). */
1320 if ( CPUMIsGuestInRealModeEx(pCtx)
1321 && pVM->hwaccm.s.vmx.pRealModeTSS)
1322 u32TrapMask |= HWACCM_VMX_TRAP_MASK_REALMODE;
1323
1324 int rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXCEPTION_BITMAP, u32TrapMask);
1325 AssertRC(rc);
1326}
1327
1328/**
1329 * Loads a minimal guest state
1330 *
1331 * NOTE: Don't do anything here that can cause a jump back to ring 3!!!!!
1332 *
1333 * @param pVM The VM to operate on.
1334 * @param pVCpu The VMCPU to operate on.
1335 * @param pCtx Guest context
1336 */
1337VMMR0DECL(void) VMXR0LoadMinimalGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1338{
1339 int rc;
1340 X86EFLAGS eflags;
1341
1342 Assert(!(pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_ALL_GUEST));
1343
1344 /* EIP, ESP and EFLAGS */
1345 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_RIP, pCtx->rip);
1346 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_RSP, pCtx->rsp);
1347 AssertRC(rc);
1348
1349 /* Bits 22-31, 15, 5 & 3 must be zero. Bit 1 must be 1. */
1350 eflags = pCtx->eflags;
1351 eflags.u32 &= VMX_EFLAGS_RESERVED_0;
1352 eflags.u32 |= VMX_EFLAGS_RESERVED_1;
1353
1354 /* Real mode emulation using v86 mode. */
1355 if ( CPUMIsGuestInRealModeEx(pCtx)
1356 && pVM->hwaccm.s.vmx.pRealModeTSS)
1357 {
1358 pVCpu->hwaccm.s.vmx.RealMode.eflags = eflags;
1359
1360 eflags.Bits.u1VM = 1;
1361 eflags.Bits.u2IOPL = 0; /* must always be 0 or else certain instructions won't cause faults. */
1362 }
1363 rc = VMXWriteVMCS(VMX_VMCS_GUEST_RFLAGS, eflags.u32);
1364 AssertRC(rc);
1365}
1366
1367/**
1368 * Loads the guest state
1369 *
1370 * NOTE: Don't do anything here that can cause a jump back to ring 3!!!!!
1371 *
1372 * @returns VBox status code.
1373 * @param pVM The VM to operate on.
1374 * @param pVCpu The VMCPU to operate on.
1375 * @param pCtx Guest context
1376 */
1377VMMR0DECL(int) VMXR0LoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1378{
1379 int rc = VINF_SUCCESS;
1380 RTGCUINTPTR val;
1381
1382 /* VMX_VMCS_CTRL_ENTRY_CONTROLS
1383 * Set required bits to one and zero according to the MSR capabilities.
1384 */
1385 val = pVM->hwaccm.s.vmx.msr.vmx_entry.n.disallowed0;
1386 /* Load guest debug controls (dr7 & IA32_DEBUGCTL_MSR) (forced to 1 on the 'first' VT-x capable CPUs; this actually includes the newest Nehalem CPUs) */
1387 val |= VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_DEBUG;
1388 /* 64 bits guest mode? */
1389 if (CPUMIsGuestInLongModeEx(pCtx))
1390 val |= VMX_VMCS_CTRL_ENTRY_CONTROLS_IA64_MODE;
1391 /* else Must be zero when AMD64 is not available. */
1392
1393 /* Mask away the bits that the CPU doesn't support */
1394 val &= pVM->hwaccm.s.vmx.msr.vmx_entry.n.allowed1;
1395 rc = VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_CONTROLS, val);
1396 AssertRC(rc);
1397
1398 /* VMX_VMCS_CTRL_EXIT_CONTROLS
1399 * Set required bits to one and zero according to the MSR capabilities.
1400 */
1401 val = pVM->hwaccm.s.vmx.msr.vmx_exit.n.disallowed0;
1402
1403 /* Save debug controls (dr7 & IA32_DEBUGCTL_MSR) (forced to 1 on the 'first' VT-x capable CPUs; this actually includes the newest Nehalem CPUs) */
1404 val |= VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_DEBUG;
1405
1406#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1407 if (VMX_IS_64BIT_HOST_MODE())
1408 val |= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_AMD64;
1409 /* else: Must be zero when AMD64 is not available. */
1410#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
1411 if (CPUMIsGuestInLongModeEx(pCtx))
1412 val |= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_AMD64; /* our switcher goes to long mode */
1413 else
1414 Assert(!(val & VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_AMD64));
1415#endif
1416 val &= pVM->hwaccm.s.vmx.msr.vmx_exit.n.allowed1;
1417 /* Don't acknowledge external interrupts on VM-exit. */
1418 rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_CONTROLS, val);
1419 AssertRC(rc);
1420
1421 /* Guest CPU context: ES, CS, SS, DS, FS, GS. */
1422 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_SEGMENT_REGS)
1423 {
1424 if (pVM->hwaccm.s.vmx.pRealModeTSS)
1425 {
1426 PGMMODE enmGuestMode = PGMGetGuestMode(pVCpu);
1427 if (pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode != enmGuestMode)
1428 {
1429 /* Correct weird requirements for switching to protected mode. */
1430 if ( pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode == PGMMODE_REAL
1431 && enmGuestMode >= PGMMODE_PROTECTED)
1432 {
1433 /* Flush the recompiler code cache as it's not unlikely
1434 * the guest will rewrite code it will later execute in real
1435 * mode (OpenBSD 4.0 is one such example)
1436 */
1437 REMFlushTBs(pVM);
1438
1439 /* DPL of all hidden selector registers must match the current CPL (0). */
1440 pCtx->csHid.Attr.n.u2Dpl = 0;
1441 pCtx->csHid.Attr.n.u4Type = X86_SEL_TYPE_CODE | X86_SEL_TYPE_RW_ACC;
1442
1443 pCtx->dsHid.Attr.n.u2Dpl = 0;
1444 pCtx->esHid.Attr.n.u2Dpl = 0;
1445 pCtx->fsHid.Attr.n.u2Dpl = 0;
1446 pCtx->gsHid.Attr.n.u2Dpl = 0;
1447 pCtx->ssHid.Attr.n.u2Dpl = 0;
1448
1449 /* The limit must correspond to the 32 bits setting. */
1450 if (!pCtx->csHid.Attr.n.u1DefBig)
1451 pCtx->csHid.u32Limit &= 0xffff;
1452 if (!pCtx->dsHid.Attr.n.u1DefBig)
1453 pCtx->dsHid.u32Limit &= 0xffff;
1454 if (!pCtx->esHid.Attr.n.u1DefBig)
1455 pCtx->esHid.u32Limit &= 0xffff;
1456 if (!pCtx->fsHid.Attr.n.u1DefBig)
1457 pCtx->fsHid.u32Limit &= 0xffff;
1458 if (!pCtx->gsHid.Attr.n.u1DefBig)
1459 pCtx->gsHid.u32Limit &= 0xffff;
1460 if (!pCtx->ssHid.Attr.n.u1DefBig)
1461 pCtx->ssHid.u32Limit &= 0xffff;
1462 }
1463 else
1464 /* Switching from protected mode to real mode. */
1465 if ( pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode >= PGMMODE_PROTECTED
1466 && enmGuestMode == PGMMODE_REAL)
1467 {
1468 /* The limit must also be set to 0xffff. */
1469 pCtx->csHid.u32Limit = 0xffff;
1470 pCtx->dsHid.u32Limit = 0xffff;
1471 pCtx->esHid.u32Limit = 0xffff;
1472 pCtx->fsHid.u32Limit = 0xffff;
1473 pCtx->gsHid.u32Limit = 0xffff;
1474 pCtx->ssHid.u32Limit = 0xffff;
1475
1476 Assert(pCtx->csHid.u64Base <= 0xfffff);
1477 Assert(pCtx->dsHid.u64Base <= 0xfffff);
1478 Assert(pCtx->esHid.u64Base <= 0xfffff);
1479 Assert(pCtx->fsHid.u64Base <= 0xfffff);
1480 Assert(pCtx->gsHid.u64Base <= 0xfffff);
1481 }
1482 pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode = enmGuestMode;
1483 }
1484 else
1485 /* VT-x will fail with a guest invalid state otherwise... (CPU state after a reset) */
1486 if ( CPUMIsGuestInRealModeEx(pCtx)
1487 && pCtx->csHid.u64Base == 0xffff0000)
1488 {
1489 pCtx->csHid.u64Base = 0xf0000;
1490 pCtx->cs = 0xf000;
1491 }
1492 }
1493
1494 VMX_WRITE_SELREG(ES, es);
1495 AssertRC(rc);
1496
1497 VMX_WRITE_SELREG(CS, cs);
1498 AssertRC(rc);
1499
1500 VMX_WRITE_SELREG(SS, ss);
1501 AssertRC(rc);
1502
1503 VMX_WRITE_SELREG(DS, ds);
1504 AssertRC(rc);
1505
1506 VMX_WRITE_SELREG(FS, fs);
1507 AssertRC(rc);
1508
1509 VMX_WRITE_SELREG(GS, gs);
1510 AssertRC(rc);
1511 }
1512
1513 /* Guest CPU context: LDTR. */
1514 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_LDTR)
1515 {
1516 if (pCtx->ldtr == 0)
1517 {
1518 rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_LDTR, 0);
1519 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_LIMIT, 0);
1520 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_LDTR_BASE, 0);
1521 /* Note: vmlaunch will fail with 0 or just 0x02. No idea why. */
1522 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, 0x82 /* present, LDT */);
1523 }
1524 else
1525 {
1526 rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_LDTR, pCtx->ldtr);
1527 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtrHid.u32Limit);
1528 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_LDTR_BASE, pCtx->ldtrHid.u64Base);
1529 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, pCtx->ldtrHid.Attr.u);
1530 }
1531 AssertRC(rc);
1532 }
1533 /* Guest CPU context: TR. */
1534 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_TR)
1535 {
1536 /* Real mode emulation using v86 mode with CR4.VME (interrupt redirection using the int bitmap in the TSS) */
1537 if ( CPUMIsGuestInRealModeEx(pCtx)
1538 && pVM->hwaccm.s.vmx.pRealModeTSS)
1539 {
1540 RTGCPHYS GCPhys;
1541
1542 /* We convert it here every time as pci regions could be reconfigured. */
1543 rc = PDMVMMDevHeapR3ToGCPhys(pVM, pVM->hwaccm.s.vmx.pRealModeTSS, &GCPhys);
1544 AssertRC(rc);
1545
1546 rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_TR, 0);
1547 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_TR_LIMIT, HWACCM_VTX_TSS_SIZE);
1548 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_TR_BASE, GCPhys /* phys = virt in this mode */);
1549
1550 X86DESCATTR attr;
1551
1552 attr.u = 0;
1553 attr.n.u1Present = 1;
1554 attr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
1555 val = attr.u;
1556 }
1557 else
1558 {
1559 rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_TR, pCtx->tr);
1560 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_TR_LIMIT, pCtx->trHid.u32Limit);
1561 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_TR_BASE, pCtx->trHid.u64Base);
1562
1563 val = pCtx->trHid.Attr.u;
1564
1565 /* The TSS selector must be busy (REM bugs? see defect #XXXX). */
1566 if (!(val & X86_SEL_TYPE_SYS_TSS_BUSY_MASK))
1567 {
1568 if (val & 0xf)
1569 val |= X86_SEL_TYPE_SYS_TSS_BUSY_MASK;
1570 else
1571 /* Default if no TR selector has been set (otherwise vmlaunch will fail!) */
1572 val = (val & ~0xF) | X86_SEL_TYPE_SYS_386_TSS_BUSY;
1573 }
1574 AssertMsg((val & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY || (val & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("%#x\n", val));
1575 }
1576 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, val);
1577 AssertRC(rc);
1578 }
1579 /* Guest CPU context: GDTR. */
1580 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_GDTR)
1581 {
1582 rc = VMXWriteVMCS(VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt);
1583 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_GDTR_BASE, pCtx->gdtr.pGdt);
1584 AssertRC(rc);
1585 }
1586 /* Guest CPU context: IDTR. */
1587 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_IDTR)
1588 {
1589 rc = VMXWriteVMCS(VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt);
1590 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_IDTR_BASE, pCtx->idtr.pIdt);
1591 AssertRC(rc);
1592 }
1593
1594 /*
1595 * Sysenter MSRs
1596 */
1597 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_MSR)
1598 {
1599 rc = VMXWriteVMCS(VMX_VMCS32_GUEST_SYSENTER_CS, pCtx->SysEnter.cs);
1600 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_SYSENTER_EIP, pCtx->SysEnter.eip);
1601 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_SYSENTER_ESP, pCtx->SysEnter.esp);
1602 AssertRC(rc);
1603 }
1604
1605 /* Control registers */
1606 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_CR0)
1607 {
1608 val = pCtx->cr0;
1609 rc = VMXWriteVMCS(VMX_VMCS_CTRL_CR0_READ_SHADOW, val);
1610 Log2(("Guest CR0-shadow %08x\n", val));
1611 if (CPUMIsGuestFPUStateActive(pVCpu) == false)
1612 {
1613 /* Always use #NM exceptions to load the FPU/XMM state on demand. */
1614 val |= X86_CR0_TS | X86_CR0_ET | X86_CR0_NE | X86_CR0_MP;
1615 }
1616 else
1617 {
1618 /** @todo check if we support the old style mess correctly. */
1619 if (!(val & X86_CR0_NE))
1620 Log(("Forcing X86_CR0_NE!!!\n"));
1621
1622 val |= X86_CR0_NE; /* always turn on the native mechanism to report FPU errors (old style uses interrupts) */
1623 }
1624 /* Note: protected mode & paging are always enabled; we use them for emulating real and protected mode without paging too. */
1625 if (!pVM->hwaccm.s.vmx.fUnrestrictedGuest)
1626 val |= X86_CR0_PE | X86_CR0_PG;
1627
1628 if (pVM->hwaccm.s.fNestedPaging)
1629 {
1630 if (CPUMIsGuestInPagedProtectedModeEx(pCtx))
1631 {
1632 /* Disable cr3 read/write monitoring as we don't need it for EPT. */
1633 pVCpu->hwaccm.s.vmx.proc_ctls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
1634 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT);
1635 }
1636 else
1637 {
1638 /* Reenable cr3 read/write monitoring as our identity mapped page table is active. */
1639 pVCpu->hwaccm.s.vmx.proc_ctls |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
1640 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
1641 }
1642 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1643 AssertRC(rc);
1644 }
1645 else
1646 {
1647 /* Note: We must also set this as we rely on protecting various pages for which supervisor writes must be caught. */
1648 val |= X86_CR0_WP;
1649 }
1650
1651 /* Always enable caching. */
1652 val &= ~(X86_CR0_CD|X86_CR0_NW);
1653
1654 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_CR0, val);
1655 Log2(("Guest CR0 %08x\n", val));
1656 /* CR0 flags owned by the host; if the guests attempts to change them, then
1657 * the VM will exit.
1658 */
1659 val = X86_CR0_PE /* Must monitor this bit (assumptions are made for real mode emulation) */
1660 | X86_CR0_WP /* Must monitor this bit (it must always be enabled). */
1661 | X86_CR0_PG /* Must monitor this bit (assumptions are made for real mode & protected mode without paging emulation) */
1662 | X86_CR0_CD /* Bit not restored during VM-exit! */
1663 | X86_CR0_NW /* Bit not restored during VM-exit! */
1664 | X86_CR0_NE;
1665
1666 /* When the guest's FPU state is active, then we no longer care about
1667 * the FPU related bits.
1668 */
1669 if (CPUMIsGuestFPUStateActive(pVCpu) == false)
1670 val |= X86_CR0_TS | X86_CR0_ET | X86_CR0_MP;
1671
1672 pVCpu->hwaccm.s.vmx.cr0_mask = val;
1673
1674 rc |= VMXWriteVMCS(VMX_VMCS_CTRL_CR0_MASK, val);
1675 Log2(("Guest CR0-mask %08x\n", val));
1676 AssertRC(rc);
1677 }
1678 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_CR4)
1679 {
1680 /* CR4 */
1681 rc = VMXWriteVMCS(VMX_VMCS_CTRL_CR4_READ_SHADOW, pCtx->cr4);
1682 Log2(("Guest CR4-shadow %08x\n", pCtx->cr4));
1683 /* Set the required bits in cr4 too (currently X86_CR4_VMXE). */
1684 val = pCtx->cr4 | (uint32_t)pVM->hwaccm.s.vmx.msr.vmx_cr4_fixed0;
1685
1686 if (!pVM->hwaccm.s.fNestedPaging)
1687 {
1688 switch(pVCpu->hwaccm.s.enmShadowMode)
1689 {
1690 case PGMMODE_REAL: /* Real mode -> emulated using v86 mode */
1691 case PGMMODE_PROTECTED: /* Protected mode, no paging -> emulated using identity mapping. */
1692 case PGMMODE_32_BIT: /* 32-bit paging. */
1693 val &= ~X86_CR4_PAE;
1694 break;
1695
1696 case PGMMODE_PAE: /* PAE paging. */
1697 case PGMMODE_PAE_NX: /* PAE paging with NX enabled. */
1698 /** Must use PAE paging as we could use physical memory > 4 GB */
1699 val |= X86_CR4_PAE;
1700 break;
1701
1702 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
1703 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
1704#ifdef VBOX_ENABLE_64_BITS_GUESTS
1705 break;
1706#else
1707 AssertFailed();
1708 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1709#endif
1710 default: /* shut up gcc */
1711 AssertFailed();
1712 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1713 }
1714 }
1715 else
1716 if ( !CPUMIsGuestInPagedProtectedModeEx(pCtx)
1717 && !pVM->hwaccm.s.vmx.fUnrestrictedGuest)
1718 {
1719 /* We use 4 MB pages in our identity mapping page table for real and protected mode without paging. */
1720 val |= X86_CR4_PSE;
1721 /* Our identity mapping is a 32 bits page directory. */
1722 val &= ~X86_CR4_PAE;
1723 }
1724
1725 /* Turn off VME if we're in emulated real mode. */
1726 if ( CPUMIsGuestInRealModeEx(pCtx)
1727 && pVM->hwaccm.s.vmx.pRealModeTSS)
1728 val &= ~X86_CR4_VME;
1729
1730 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_CR4, val);
1731 Log2(("Guest CR4 %08x\n", val));
1732 /* CR4 flags owned by the host; if the guests attempts to change them, then
1733 * the VM will exit.
1734 */
1735 val = 0
1736 | X86_CR4_VME
1737 | X86_CR4_PAE
1738 | X86_CR4_PGE
1739 | X86_CR4_PSE
1740 | X86_CR4_VMXE;
1741 pVCpu->hwaccm.s.vmx.cr4_mask = val;
1742
1743 rc |= VMXWriteVMCS(VMX_VMCS_CTRL_CR4_MASK, val);
1744 Log2(("Guest CR4-mask %08x\n", val));
1745 AssertRC(rc);
1746 }
1747
1748 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_CR3)
1749 {
1750 if (pVM->hwaccm.s.fNestedPaging)
1751 {
1752 Assert(PGMGetHyperCR3(pVCpu));
1753 pVCpu->hwaccm.s.vmx.GCPhysEPTP = PGMGetHyperCR3(pVCpu);
1754
1755 Assert(!(pVCpu->hwaccm.s.vmx.GCPhysEPTP & 0xfff));
1756 /** @todo Check the IA32_VMX_EPT_VPID_CAP MSR for other supported memory types. */
1757 pVCpu->hwaccm.s.vmx.GCPhysEPTP |= VMX_EPT_MEMTYPE_WB
1758 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
1759
1760 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_EPTP_FULL, pVCpu->hwaccm.s.vmx.GCPhysEPTP);
1761 AssertRC(rc);
1762
1763 if ( !CPUMIsGuestInPagedProtectedModeEx(pCtx)
1764 && !pVM->hwaccm.s.vmx.fUnrestrictedGuest)
1765 {
1766 RTGCPHYS GCPhys;
1767
1768 /* We convert it here every time as pci regions could be reconfigured. */
1769 rc = PDMVMMDevHeapR3ToGCPhys(pVM, pVM->hwaccm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
1770 AssertMsgRC(rc, ("pNonPagingModeEPTPageTable = %RGv\n", pVM->hwaccm.s.vmx.pNonPagingModeEPTPageTable));
1771
1772 /* We use our identity mapping page table here as we need to map guest virtual to guest physical addresses; EPT will
1773 * take care of the translation to host physical addresses.
1774 */
1775 val = GCPhys;
1776 }
1777 else
1778 {
1779 /* Save the real guest CR3 in VMX_VMCS_GUEST_CR3 */
1780 val = pCtx->cr3;
1781 rc = hmR0VmxLoadPaePdpes(pVCpu, pCtx);
1782 AssertRCReturn(rc, rc);
1783 }
1784 }
1785 else
1786 {
1787 val = PGMGetHyperCR3(pVCpu);
1788 Assert(val || VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL));
1789 }
1790
1791 /* Save our shadow CR3 register. */
1792 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_CR3, val);
1793 AssertRC(rc);
1794 }
1795
1796 /* Debug registers. */
1797 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_DEBUG)
1798 {
1799 pCtx->dr[6] |= X86_DR6_INIT_VAL; /* set all reserved bits to 1. */
1800 pCtx->dr[6] &= ~RT_BIT(12); /* must be zero. */
1801
1802 pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
1803 pCtx->dr[7] &= ~(RT_BIT(11) | RT_BIT(12) | RT_BIT(14) | RT_BIT(15)); /* must be zero */
1804 pCtx->dr[7] |= 0x400; /* must be one */
1805
1806 /* Resync DR7 */
1807 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_DR7, pCtx->dr[7]);
1808 AssertRC(rc);
1809
1810#ifdef DEBUG
1811 /* Sync the hypervisor debug state now if any breakpoint is armed. */
1812 if ( CPUMGetHyperDR7(pVCpu) & (X86_DR7_ENABLED_MASK|X86_DR7_GD)
1813 && !CPUMIsHyperDebugStateActive(pVCpu)
1814 && !DBGFIsStepping(pVCpu))
1815 {
1816 /* Save the host and load the hypervisor debug state. */
1817 rc = CPUMR0LoadHyperDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
1818 AssertRC(rc);
1819
1820 /* DRx intercepts remain enabled. */
1821
1822 /* Override dr7 with the hypervisor value. */
1823 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_DR7, CPUMGetHyperDR7(pVCpu));
1824 AssertRC(rc);
1825 }
1826 else
1827#endif
1828 /* Sync the debug state now if any breakpoint is armed. */
1829 if ( (pCtx->dr[7] & (X86_DR7_ENABLED_MASK|X86_DR7_GD))
1830 && !CPUMIsGuestDebugStateActive(pVCpu)
1831 && !DBGFIsStepping(pVCpu))
1832 {
1833 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatDRxArmed);
1834
1835 /* Disable drx move intercepts. */
1836 pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
1837 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1838 AssertRC(rc);
1839
1840 /* Save the host and load the guest debug state. */
1841 rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
1842 AssertRC(rc);
1843 }
1844
1845 /* IA32_DEBUGCTL MSR. */
1846 rc = VMXWriteVMCS64(VMX_VMCS_GUEST_DEBUGCTL_FULL, 0);
1847 AssertRC(rc);
1848
1849 /** @todo do we really ever need this? */
1850 rc |= VMXWriteVMCS(VMX_VMCS_GUEST_DEBUG_EXCEPTIONS, 0);
1851 AssertRC(rc);
1852 }
1853
1854 /* 64 bits guest mode? */
1855 if (CPUMIsGuestInLongModeEx(pCtx))
1856 {
1857#if !defined(VBOX_ENABLE_64_BITS_GUESTS)
1858 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1859#elif HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1860 pVCpu->hwaccm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
1861#else
1862# ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1863 if (!pVM->hwaccm.s.fAllow64BitGuests)
1864 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1865# endif
1866 pVCpu->hwaccm.s.vmx.pfnStartVM = VMXR0StartVM64;
1867#endif
1868 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_MSR)
1869 {
1870 /* Update these as wrmsr might have changed them. */
1871 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_FS_BASE, pCtx->fsHid.u64Base);
1872 AssertRC(rc);
1873 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_GS_BASE, pCtx->gsHid.u64Base);
1874 AssertRC(rc);
1875 }
1876 }
1877 else
1878 {
1879 pVCpu->hwaccm.s.vmx.pfnStartVM = VMXR0StartVM32;
1880 }
1881
1882 hmR0VmxUpdateExceptionBitmap(pVM, pVCpu, pCtx);
1883
1884#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
1885 /* Store all guest MSRs in the VM-Entry load area, so they will be loaded during the world switch. */
1886 PVMXMSR pMsr = (PVMXMSR)pVCpu->hwaccm.s.vmx.pGuestMSR;
1887 unsigned idxMsr = 0;
1888
1889 uint32_t ulEdx;
1890 uint32_t ulTemp;
1891 CPUMGetGuestCpuId(pVCpu, 0x80000001, &ulTemp, &ulTemp, &ulTemp, &ulEdx);
1892 /* EFER MSR present? */
1893 if (ulEdx & (X86_CPUID_AMD_FEATURE_EDX_NX|X86_CPUID_AMD_FEATURE_EDX_LONG_MODE))
1894 {
1895 pMsr->u32IndexMSR = MSR_K6_EFER;
1896 pMsr->u32Reserved = 0;
1897 pMsr->u64Value = pCtx->msrEFER;
1898 /* VT-x will complain if only MSR_K6_EFER_LME is set. */
1899 if (!CPUMIsGuestInLongModeEx(pCtx))
1900 pMsr->u64Value &= ~(MSR_K6_EFER_LMA|MSR_K6_EFER_LME);
1901 pMsr++; idxMsr++;
1902
1903 if (ulEdx & X86_CPUID_AMD_FEATURE_EDX_LONG_MODE)
1904 {
1905 pMsr->u32IndexMSR = MSR_K8_LSTAR;
1906 pMsr->u32Reserved = 0;
1907 pMsr->u64Value = pCtx->msrLSTAR; /* 64 bits mode syscall rip */
1908 pMsr++; idxMsr++;
1909 pMsr->u32IndexMSR = MSR_K6_STAR;
1910 pMsr->u32Reserved = 0;
1911 pMsr->u64Value = pCtx->msrSTAR; /* legacy syscall eip, cs & ss */
1912 pMsr++; idxMsr++;
1913 pMsr->u32IndexMSR = MSR_K8_SF_MASK;
1914 pMsr->u32Reserved = 0;
1915 pMsr->u64Value = pCtx->msrSFMASK; /* syscall flag mask */
1916 pMsr++; idxMsr++;
1917 pMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
1918 pMsr->u32Reserved = 0;
1919 pMsr->u64Value = pCtx->msrKERNELGSBASE; /* swapgs exchange value */
1920 pMsr++; idxMsr++;
1921 }
1922 }
1923 pVCpu->hwaccm.s.vmx.cCachedMSRs = idxMsr;
1924
1925 rc = VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_MSR_LOAD_COUNT, idxMsr);
1926 AssertRC(rc);
1927
1928 rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_MSR_STORE_COUNT, idxMsr);
1929 AssertRC(rc);
1930#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
1931
1932 bool fOffsettedTsc;
1933 if (pVM->hwaccm.s.vmx.fUsePreemptTimer)
1934 {
1935 uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &pVCpu->hwaccm.s.vmx.u64TSCOffset);
1936
1937 /* Make sure the returned values have sane upper and lower boundaries. */
1938 uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
1939
1940 cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64 of a second */
1941 cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
1942
1943 cTicksToDeadline >>= pVM->hwaccm.s.vmx.cPreemptTimerShift;
1944 uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
1945 rc = VMXWriteVMCS(VMX_VMCS32_GUEST_PREEMPTION_TIMER_VALUE, cPreemptionTickCount);
1946 AssertRC(rc);
1947 }
1948 else
1949 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hwaccm.s.vmx.u64TSCOffset);
1950 if (fOffsettedTsc)
1951 {
1952 uint64_t u64CurTSC = ASMReadTSC();
1953 if (u64CurTSC + pVCpu->hwaccm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
1954 {
1955 /* Note: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT takes precedence over TSC_OFFSET */
1956 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_TSC_OFFSET_FULL, pVCpu->hwaccm.s.vmx.u64TSCOffset);
1957 AssertRC(rc);
1958
1959 pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
1960 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1961 AssertRC(rc);
1962 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTSCOffset);
1963 }
1964 else
1965 {
1966 /* Fall back to rdtsc emulation as we would otherwise pass decreasing tsc values to the guest. */
1967 LogFlow(("TSC %RX64 offset %RX64 time=%RX64 last=%RX64 (diff=%RX64, virt_tsc=%RX64)\n", u64CurTSC, pVCpu->hwaccm.s.vmx.u64TSCOffset, u64CurTSC + pVCpu->hwaccm.s.vmx.u64TSCOffset, TMCpuTickGetLastSeen(pVCpu), TMCpuTickGetLastSeen(pVCpu) - u64CurTSC - pVCpu->hwaccm.s.vmx.u64TSCOffset, TMCpuTickGet(pVCpu)));
1968 pVCpu->hwaccm.s.vmx.proc_ctls |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
1969 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1970 AssertRC(rc);
1971 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTSCInterceptOverFlow);
1972 }
1973 }
1974 else
1975 {
1976 pVCpu->hwaccm.s.vmx.proc_ctls |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
1977 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1978 AssertRC(rc);
1979 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTSCIntercept);
1980 }
1981
1982 /* Done with the major changes */
1983 pVCpu->hwaccm.s.fContextUseFlags &= ~HWACCM_CHANGED_ALL_GUEST;
1984
1985 /* Minimal guest state update (esp, eip, eflags mostly) */
1986 VMXR0LoadMinimalGuestState(pVM, pVCpu, pCtx);
1987 return rc;
1988}
1989
1990/**
1991 * Syncs back the guest state
1992 *
1993 * @returns VBox status code.
1994 * @param pVM The VM to operate on.
1995 * @param pVCpu The VMCPU to operate on.
1996 * @param pCtx Guest context
1997 */
1998DECLINLINE(int) VMXR0SaveGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1999{
2000 RTGCUINTREG val, valShadow;
2001 RTGCUINTPTR uInterruptState;
2002 int rc;
2003
2004 /* Let's first sync back eip, esp, and eflags. */
2005 rc = VMXReadCachedVMCS(VMX_VMCS64_GUEST_RIP, &val);
2006 AssertRC(rc);
2007 pCtx->rip = val;
2008 rc = VMXReadCachedVMCS(VMX_VMCS64_GUEST_RSP, &val);
2009 AssertRC(rc);
2010 pCtx->rsp = val;
2011 rc = VMXReadCachedVMCS(VMX_VMCS_GUEST_RFLAGS, &val);
2012 AssertRC(rc);
2013 pCtx->eflags.u32 = val;
2014
2015 /* Take care of instruction fusing (sti, mov ss) */
2016 rc |= VMXReadCachedVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &val);
2017 uInterruptState = val;
2018 if (uInterruptState != 0)
2019 {
2020 Assert(uInterruptState <= 2); /* only sti & mov ss */
2021 Log(("uInterruptState %x eip=%RGv\n", (uint32_t)uInterruptState, pCtx->rip));
2022 EMSetInhibitInterruptsPC(pVCpu, pCtx->rip);
2023 }
2024 else
2025 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2026
2027 /* Control registers. */
2028 VMXReadCachedVMCS(VMX_VMCS_CTRL_CR0_READ_SHADOW, &valShadow);
2029 VMXReadCachedVMCS(VMX_VMCS64_GUEST_CR0, &val);
2030 val = (valShadow & pVCpu->hwaccm.s.vmx.cr0_mask) | (val & ~pVCpu->hwaccm.s.vmx.cr0_mask);
2031 CPUMSetGuestCR0(pVCpu, val);
2032
2033 VMXReadCachedVMCS(VMX_VMCS_CTRL_CR4_READ_SHADOW, &valShadow);
2034 VMXReadCachedVMCS(VMX_VMCS64_GUEST_CR4, &val);
2035 val = (valShadow & pVCpu->hwaccm.s.vmx.cr4_mask) | (val & ~pVCpu->hwaccm.s.vmx.cr4_mask);
2036 CPUMSetGuestCR4(pVCpu, val);
2037
2038 /* Note: no reason to sync back the CRx registers. They can't be changed by the guest. */
2039 /* Note: only in the nested paging case can CR3 & CR4 be changed by the guest. */
2040 if ( pVM->hwaccm.s.fNestedPaging
2041 && CPUMIsGuestInPagedProtectedModeEx(pCtx))
2042 {
2043 PVMCSCACHE pCache = &pVCpu->hwaccm.s.vmx.VMCSCache;
2044
2045 /* Can be updated behind our back in the nested paging case. */
2046 CPUMSetGuestCR2(pVCpu, pCache->cr2);
2047
2048 VMXReadCachedVMCS(VMX_VMCS64_GUEST_CR3, &val);
2049
2050 if (val != pCtx->cr3)
2051 {
2052 CPUMSetGuestCR3(pVCpu, val);
2053 PGMUpdateCR3(pVCpu, val);
2054 }
2055 rc = hmR0VmxSavePaePdpes(pVCpu, pCtx);
2056 AssertRCReturn(rc, rc);
2057 }
2058
2059 /* Sync back DR7 here. */
2060 VMXReadCachedVMCS(VMX_VMCS64_GUEST_DR7, &val);
2061 pCtx->dr[7] = val;
2062
2063 /* Guest CPU context: ES, CS, SS, DS, FS, GS. */
2064 VMX_READ_SELREG(ES, es);
2065 VMX_READ_SELREG(SS, ss);
2066 VMX_READ_SELREG(CS, cs);
2067 VMX_READ_SELREG(DS, ds);
2068 VMX_READ_SELREG(FS, fs);
2069 VMX_READ_SELREG(GS, gs);
2070
2071 /*
2072 * System MSRs
2073 */
2074 VMXReadCachedVMCS(VMX_VMCS32_GUEST_SYSENTER_CS, &val);
2075 pCtx->SysEnter.cs = val;
2076 VMXReadCachedVMCS(VMX_VMCS64_GUEST_SYSENTER_EIP, &val);
2077 pCtx->SysEnter.eip = val;
2078 VMXReadCachedVMCS(VMX_VMCS64_GUEST_SYSENTER_ESP, &val);
2079 pCtx->SysEnter.esp = val;
2080
2081 /* Misc. registers; must sync everything otherwise we can get out of sync when jumping to ring 3. */
2082 VMX_READ_SELREG(LDTR, ldtr);
2083
2084 VMXReadCachedVMCS(VMX_VMCS32_GUEST_GDTR_LIMIT, &val);
2085 pCtx->gdtr.cbGdt = val;
2086 VMXReadCachedVMCS(VMX_VMCS64_GUEST_GDTR_BASE, &val);
2087 pCtx->gdtr.pGdt = val;
2088
2089 VMXReadCachedVMCS(VMX_VMCS32_GUEST_IDTR_LIMIT, &val);
2090 pCtx->idtr.cbIdt = val;
2091 VMXReadCachedVMCS(VMX_VMCS64_GUEST_IDTR_BASE, &val);
2092 pCtx->idtr.pIdt = val;
2093
2094 /* Real mode emulation using v86 mode. */
2095 if ( CPUMIsGuestInRealModeEx(pCtx)
2096 && pVM->hwaccm.s.vmx.pRealModeTSS)
2097 {
2098 /* Hide our emulation flags */
2099 pCtx->eflags.Bits.u1VM = 0;
2100
2101 /* Restore original IOPL setting as we always use 0. */
2102 pCtx->eflags.Bits.u2IOPL = pVCpu->hwaccm.s.vmx.RealMode.eflags.Bits.u2IOPL;
2103
2104 /* Force a TR resync every time in case we switch modes. */
2105 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_TR;
2106 }
2107 else
2108 {
2109 /* In real mode we have a fake TSS, so only sync it back when it's supposed to be valid. */
2110 VMX_READ_SELREG(TR, tr);
2111 }
2112
2113#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
2114 /* Save the possibly changed MSRs that we automatically restore and save during a world switch. */
2115 for (unsigned i = 0; i < pVCpu->hwaccm.s.vmx.cCachedMSRs; i++)
2116 {
2117 PVMXMSR pMsr = (PVMXMSR)pVCpu->hwaccm.s.vmx.pGuestMSR;
2118 pMsr += i;
2119
2120 switch (pMsr->u32IndexMSR)
2121 {
2122 case MSR_K8_LSTAR:
2123 pCtx->msrLSTAR = pMsr->u64Value;
2124 break;
2125 case MSR_K6_STAR:
2126 pCtx->msrSTAR = pMsr->u64Value;
2127 break;
2128 case MSR_K8_SF_MASK:
2129 pCtx->msrSFMASK = pMsr->u64Value;
2130 break;
2131 case MSR_K8_KERNEL_GS_BASE:
2132 pCtx->msrKERNELGSBASE = pMsr->u64Value;
2133 break;
2134 case MSR_K6_EFER:
2135 /* EFER can't be changed without causing a VM-exit. */
2136// Assert(pCtx->msrEFER == pMsr->u64Value);
2137 break;
2138 default:
2139 AssertFailed();
2140 return VERR_HM_UNEXPECTED_LD_ST_MSR;
2141 }
2142 }
2143#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
2144 return VINF_SUCCESS;
2145}
2146
2147/**
2148 * Dummy placeholder
2149 *
2150 * @param pVM The VM to operate on.
2151 * @param pVCpu The VMCPU to operate on.
2152 */
2153static void hmR0VmxSetupTLBDummy(PVM pVM, PVMCPU pVCpu)
2154{
2155 NOREF(pVM);
2156 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
2157 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2158 pVCpu->hwaccm.s.TlbShootdown.cPages = 0;
2159 return;
2160}
2161
2162/**
2163 * Setup the tagged TLB for EPT
2164 *
2165 * @returns VBox status code.
2166 * @param pVM The VM to operate on.
2167 * @param pVCpu The VMCPU to operate on.
2168 */
2169static void hmR0VmxSetupTLBEPT(PVM pVM, PVMCPU pVCpu)
2170{
2171 PHMGLOBLCPUINFO pCpu;
2172
2173 Assert(pVM->hwaccm.s.fNestedPaging);
2174 Assert(!pVM->hwaccm.s.vmx.fVPID);
2175
2176 /* Deal with tagged TLBs if VPID or EPT is supported. */
2177 pCpu = HWACCMR0GetCurrentCpu();
2178 /* Force a TLB flush for the first world switch if the current cpu differs from the one we ran on last. */
2179 /* Note that this can happen both for start and resume due to long jumps back to ring 3. */
2180 if ( pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu
2181 /* if the tlb flush count has changed, another VM has flushed the TLB of this cpu, so we can't use our current ASID anymore. */
2182 || pVCpu->hwaccm.s.cTLBFlushes != pCpu->cTLBFlushes)
2183 {
2184 /* Force a TLB flush on VM entry. */
2185 pVCpu->hwaccm.s.fForceTLBFlush = true;
2186 }
2187 /* Disabled because this has triggered every time I have suspended my
2188 * laptop with a VM running for the past three months or more. */
2189 // else
2190 // Assert(!pCpu->fFlushTLB);
2191
2192 /* Check for tlb shootdown flushes. */
2193 if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2194 pVCpu->hwaccm.s.fForceTLBFlush = true;
2195
2196 pVCpu->hwaccm.s.idLastCpu = pCpu->idCpu;
2197 pCpu->fFlushTLB = false;
2198
2199 if (pVCpu->hwaccm.s.fForceTLBFlush)
2200 {
2201 hmR0VmxFlushEPT(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushContext, 0);
2202 }
2203 else
2204 if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2205 {
2206 /* Deal with pending TLB shootdown actions which were queued when we were not executing code. */
2207 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTlbShootdown);
2208
2209 for (unsigned i=0;i<pVCpu->hwaccm.s.TlbShootdown.cPages;i++)
2210 {
2211 /* aTlbShootdownPages contains physical addresses in this case. */
2212 hmR0VmxFlushEPT(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushPage, pVCpu->hwaccm.s.TlbShootdown.aPages[i]);
2213 }
2214 }
2215 pVCpu->hwaccm.s.TlbShootdown.cPages= 0;
2216 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2217
2218#ifdef VBOX_WITH_STATISTICS
2219 if (pVCpu->hwaccm.s.fForceTLBFlush)
2220 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushTLBWorldSwitch);
2221 else
2222 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatNoFlushTLBWorldSwitch);
2223#endif
2224}
2225
2226#ifdef HWACCM_VTX_WITH_VPID
2227/**
2228 * Setup the tagged TLB for VPID
2229 *
2230 * @returns VBox status code.
2231 * @param pVM The VM to operate on.
2232 * @param pVCpu The VMCPU to operate on.
2233 */
2234static void hmR0VmxSetupTLBVPID(PVM pVM, PVMCPU pVCpu)
2235{
2236 PHMGLOBLCPUINFO pCpu;
2237
2238 Assert(pVM->hwaccm.s.vmx.fVPID);
2239 Assert(!pVM->hwaccm.s.fNestedPaging);
2240
2241 /* Deal with tagged TLBs if VPID or EPT is supported. */
2242 pCpu = HWACCMR0GetCurrentCpu();
2243 /* Force a TLB flush for the first world switch if the current cpu differs from the one we ran on last. */
2244 /* Note that this can happen both for start and resume due to long jumps back to ring 3. */
2245 if ( pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu
2246 /* if the tlb flush count has changed, another VM has flushed the TLB of this cpu, so we can't use our current ASID anymore. */
2247 || pVCpu->hwaccm.s.cTLBFlushes != pCpu->cTLBFlushes)
2248 {
2249 /* Force a TLB flush on VM entry. */
2250 pVCpu->hwaccm.s.fForceTLBFlush = true;
2251 }
2252 else
2253 Assert(!pCpu->fFlushTLB);
2254
2255 pVCpu->hwaccm.s.idLastCpu = pCpu->idCpu;
2256
2257 /* Check for tlb shootdown flushes. */
2258 if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2259 pVCpu->hwaccm.s.fForceTLBFlush = true;
2260
2261 /* Make sure we flush the TLB when required. Switch ASID to achieve the same thing, but without actually flushing the whole TLB (which is expensive). */
2262 if (pVCpu->hwaccm.s.fForceTLBFlush)
2263 {
2264 if ( ++pCpu->uCurrentASID >= pVM->hwaccm.s.uMaxASID
2265 || pCpu->fFlushTLB)
2266 {
2267 pCpu->fFlushTLB = false;
2268 pCpu->uCurrentASID = 1; /* start at 1; host uses 0 */
2269 pCpu->cTLBFlushes++;
2270 hmR0VmxFlushVPID(pVM, pVCpu, VMX_FLUSH_ALL_CONTEXTS, 0);
2271 }
2272 else
2273 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushASID);
2274
2275 pVCpu->hwaccm.s.fForceTLBFlush = false;
2276 pVCpu->hwaccm.s.cTLBFlushes = pCpu->cTLBFlushes;
2277 pVCpu->hwaccm.s.uCurrentASID = pCpu->uCurrentASID;
2278 }
2279 else
2280 {
2281 Assert(!pCpu->fFlushTLB);
2282 Assert(pVCpu->hwaccm.s.uCurrentASID && pCpu->uCurrentASID);
2283
2284 if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2285 {
2286 /* Deal with pending TLB shootdown actions which were queued when we were not executing code. */
2287 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTlbShootdown);
2288 for (unsigned i = 0; i < pVCpu->hwaccm.s.TlbShootdown.cPages; i++)
2289 hmR0VmxFlushVPID(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushPage, pVCpu->hwaccm.s.TlbShootdown.aPages[i]);
2290 }
2291 }
2292 pVCpu->hwaccm.s.TlbShootdown.cPages = 0;
2293 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2294
2295 AssertMsg(pVCpu->hwaccm.s.cTLBFlushes == pCpu->cTLBFlushes, ("Flush count mismatch for cpu %d (%x vs %x)\n", pCpu->idCpu, pVCpu->hwaccm.s.cTLBFlushes, pCpu->cTLBFlushes));
2296 AssertMsg(pCpu->uCurrentASID >= 1 && pCpu->uCurrentASID < pVM->hwaccm.s.uMaxASID, ("cpu%d uCurrentASID = %x\n", pCpu->idCpu, pCpu->uCurrentASID));
2297 AssertMsg(pVCpu->hwaccm.s.uCurrentASID >= 1 && pVCpu->hwaccm.s.uCurrentASID < pVM->hwaccm.s.uMaxASID, ("cpu%d VM uCurrentASID = %x\n", pCpu->idCpu, pVCpu->hwaccm.s.uCurrentASID));
2298
2299 int rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hwaccm.s.uCurrentASID);
2300 AssertRC(rc);
2301
2302 if (pVCpu->hwaccm.s.fForceTLBFlush)
2303 hmR0VmxFlushVPID(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushContext, 0);
2304
2305# ifdef VBOX_WITH_STATISTICS
2306 if (pVCpu->hwaccm.s.fForceTLBFlush)
2307 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushTLBWorldSwitch);
2308 else
2309 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatNoFlushTLBWorldSwitch);
2310# endif
2311}
2312#endif /* HWACCM_VTX_WITH_VPID */
2313
2314/**
2315 * Runs guest code in a VT-x VM.
2316 *
2317 * @returns VBox status code.
2318 * @param pVM The VM to operate on.
2319 * @param pVCpu The VMCPU to operate on.
2320 * @param pCtx Guest context
2321 */
2322VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2323{
2324 STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatEntry, x);
2325 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hwaccm.s.StatExit1);
2326 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hwaccm.s.StatExit2);
2327
2328 VBOXSTRICTRC rc = VINF_SUCCESS;
2329 int rc2;
2330 RTGCUINTREG val;
2331 RTGCUINTREG exitReason = (RTGCUINTREG)VMX_EXIT_INVALID;
2332 RTGCUINTREG instrError, cbInstr;
2333 RTGCUINTPTR exitQualification = 0;
2334 RTGCUINTPTR intInfo = 0; /* shut up buggy gcc 4 */
2335 RTGCUINTPTR errCode, instrInfo;
2336 bool fSetupTPRCaching = false;
2337 uint64_t u64OldLSTAR = 0;
2338 uint8_t u8LastTPR = 0;
2339 RTCCUINTREG uOldEFlags = ~(RTCCUINTREG)0;
2340 unsigned cResume = 0;
2341#ifdef VBOX_STRICT
2342 RTCPUID idCpuCheck;
2343 bool fWasInLongMode = false;
2344#endif
2345#ifdef VBOX_HIGH_RES_TIMERS_HACK_IN_RING0
2346 uint64_t u64LastTime = RTTimeMilliTS();
2347#endif
2348
2349 Assert(!(pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC) || (pVCpu->hwaccm.s.vmx.pbVAPIC && pVM->hwaccm.s.vmx.pAPIC));
2350
2351 /* Check if we need to use TPR shadowing. */
2352 if ( CPUMIsGuestInLongModeEx(pCtx)
2353 || ( ((pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC) || pVM->hwaccm.s.fTRPPatchingAllowed)
2354 && pVM->hwaccm.s.fHasIoApic)
2355 )
2356 {
2357 fSetupTPRCaching = true;
2358 }
2359
2360 Log2(("\nE"));
2361
2362#ifdef VBOX_STRICT
2363 {
2364 RTCCUINTREG val2;
2365
2366 rc2 = VMXReadVMCS(VMX_VMCS_CTRL_PIN_EXEC_CONTROLS, &val2);
2367 AssertRC(rc2);
2368 Log2(("VMX_VMCS_CTRL_PIN_EXEC_CONTROLS = %08x\n", val2));
2369
2370 /* allowed zero */
2371 if ((val2 & pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.disallowed0)
2372 Log(("Invalid VMX_VMCS_CTRL_PIN_EXEC_CONTROLS: zero\n"));
2373
2374 /* allowed one */
2375 if ((val2 & ~pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.allowed1) != 0)
2376 Log(("Invalid VMX_VMCS_CTRL_PIN_EXEC_CONTROLS: one\n"));
2377
2378 rc2 = VMXReadVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, &val2);
2379 AssertRC(rc2);
2380 Log2(("VMX_VMCS_CTRL_PROC_EXEC_CONTROLS = %08x\n", val2));
2381
2382 /* Must be set according to the MSR, but can be cleared in case of EPT. */
2383 if (pVM->hwaccm.s.fNestedPaging)
2384 val2 |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT
2385 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
2386 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
2387
2388 /* allowed zero */
2389 if ((val2 & pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.disallowed0)
2390 Log(("Invalid VMX_VMCS_CTRL_PROC_EXEC_CONTROLS: zero\n"));
2391
2392 /* allowed one */
2393 if ((val2 & ~pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1) != 0)
2394 Log(("Invalid VMX_VMCS_CTRL_PROC_EXEC_CONTROLS: one\n"));
2395
2396 rc2 = VMXReadVMCS(VMX_VMCS_CTRL_ENTRY_CONTROLS, &val2);
2397 AssertRC(rc2);
2398 Log2(("VMX_VMCS_CTRL_ENTRY_CONTROLS = %08x\n", val2));
2399
2400 /* allowed zero */
2401 if ((val2 & pVM->hwaccm.s.vmx.msr.vmx_entry.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_entry.n.disallowed0)
2402 Log(("Invalid VMX_VMCS_CTRL_ENTRY_CONTROLS: zero\n"));
2403
2404 /* allowed one */
2405 if ((val2 & ~pVM->hwaccm.s.vmx.msr.vmx_entry.n.allowed1) != 0)
2406 Log(("Invalid VMX_VMCS_CTRL_ENTRY_CONTROLS: one\n"));
2407
2408 rc2 = VMXReadVMCS(VMX_VMCS_CTRL_EXIT_CONTROLS, &val2);
2409 AssertRC(rc2);
2410 Log2(("VMX_VMCS_CTRL_EXIT_CONTROLS = %08x\n", val2));
2411
2412 /* allowed zero */
2413 if ((val2 & pVM->hwaccm.s.vmx.msr.vmx_exit.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_exit.n.disallowed0)
2414 Log(("Invalid VMX_VMCS_CTRL_EXIT_CONTROLS: zero\n"));
2415
2416 /* allowed one */
2417 if ((val2 & ~pVM->hwaccm.s.vmx.msr.vmx_exit.n.allowed1) != 0)
2418 Log(("Invalid VMX_VMCS_CTRL_EXIT_CONTROLS: one\n"));
2419 }
2420 fWasInLongMode = CPUMIsGuestInLongModeEx(pCtx);
2421#endif /* VBOX_STRICT */
2422
2423#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2424 pVCpu->hwaccm.s.vmx.VMCSCache.u64TimeEntry = RTTimeNanoTS();
2425#endif
2426
2427 /* We can jump to this point to resume execution after determining that a VM-exit is innocent.
2428 */
2429ResumeExecution:
2430 if (!STAM_REL_PROFILE_ADV_IS_RUNNING(&pVCpu->hwaccm.s.StatEntry))
2431 STAM_REL_PROFILE_ADV_STOP_START(&pVCpu->hwaccm.s.StatExit2, &pVCpu->hwaccm.s.StatEntry, x);
2432 AssertMsg(pVCpu->hwaccm.s.idEnteredCpu == RTMpCpuId(),
2433 ("Expected %d, I'm %d; cResume=%d exitReason=%RGv exitQualification=%RGv\n",
2434 (int)pVCpu->hwaccm.s.idEnteredCpu, (int)RTMpCpuId(), cResume, exitReason, exitQualification));
2435 Assert(!HWACCMR0SuspendPending());
2436 /* Not allowed to switch modes without reloading the host state (32->64 switcher)!! */
2437 Assert(fWasInLongMode == CPUMIsGuestInLongModeEx(pCtx));
2438
2439 /* Safety precaution; looping for too long here can have a very bad effect on the host */
2440 if (RT_UNLIKELY(++cResume > pVM->hwaccm.s.cMaxResumeLoops))
2441 {
2442 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitMaxResume);
2443 rc = VINF_EM_RAW_INTERRUPT;
2444 goto end;
2445 }
2446
2447 /* Check for irq inhibition due to instruction fusing (sti, mov ss). */
2448 if (VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
2449 {
2450 Log(("VM_FF_INHIBIT_INTERRUPTS at %RGv successor %RGv\n", (RTGCPTR)pCtx->rip, EMGetInhibitInterruptsPC(pVCpu)));
2451 if (pCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
2452 {
2453 /* Note: we intentionally don't clear VM_FF_INHIBIT_INTERRUPTS here.
2454 * Before we are able to execute this instruction in raw mode (iret to guest code) an external interrupt might
2455 * force a world switch again. Possibly allowing a guest interrupt to be dispatched in the process. This could
2456 * break the guest. Sounds very unlikely, but such timing sensitive problems are not as rare as you might think.
2457 */
2458 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2459 /* Irq inhibition is no longer active; clear the corresponding VMX state. */
2460 rc2 = VMXWriteVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, 0);
2461 AssertRC(rc2);
2462 }
2463 }
2464 else
2465 {
2466 /* Irq inhibition is no longer active; clear the corresponding VMX state. */
2467 rc2 = VMXWriteVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, 0);
2468 AssertRC(rc2);
2469 }
2470
2471#ifdef VBOX_HIGH_RES_TIMERS_HACK_IN_RING0
2472 if (RT_UNLIKELY((cResume & 0xf) == 0))
2473 {
2474 uint64_t u64CurTime = RTTimeMilliTS();
2475
2476 if (RT_UNLIKELY(u64CurTime > u64LastTime))
2477 {
2478 u64LastTime = u64CurTime;
2479 TMTimerPollVoid(pVM, pVCpu);
2480 }
2481 }
2482#endif
2483
2484 /* Check for pending actions that force us to go back to ring 3. */
2485 if ( VM_FF_ISPENDING(pVM, VM_FF_HWACCM_TO_R3_MASK | VM_FF_REQUEST | VM_FF_PGM_POOL_FLUSH_PENDING | VM_FF_PDM_DMA)
2486 || VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_HWACCM_TO_R3_MASK | VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL | VMCPU_FF_REQUEST))
2487 {
2488 /* Check if a sync operation is pending. */
2489 if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
2490 {
2491 rc = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4, VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
2492 if (rc != VINF_SUCCESS)
2493 {
2494 AssertRC(VBOXSTRICTRC_VAL(rc));
2495 Log(("Pending pool sync is forcing us back to ring 3; rc=%d\n", VBOXSTRICTRC_VAL(rc)));
2496 goto end;
2497 }
2498 }
2499
2500#ifdef DEBUG
2501 /* Intercept X86_XCPT_DB if stepping is enabled */
2502 if (!DBGFIsStepping(pVCpu))
2503#endif
2504 {
2505 if ( VM_FF_ISPENDING(pVM, VM_FF_HWACCM_TO_R3_MASK)
2506 || VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_HWACCM_TO_R3_MASK))
2507 {
2508 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatSwitchToR3);
2509 rc = RT_UNLIKELY(VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
2510 goto end;
2511 }
2512 }
2513
2514 /* Pending request packets might contain actions that need immediate attention, such as pending hardware interrupts. */
2515 if ( VM_FF_ISPENDING(pVM, VM_FF_REQUEST)
2516 || VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_REQUEST))
2517 {
2518 rc = VINF_EM_PENDING_REQUEST;
2519 goto end;
2520 }
2521
2522 /* Check if a pgm pool flush is in progress. */
2523 if (VM_FF_ISPENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
2524 {
2525 rc = VINF_PGM_POOL_FLUSH_PENDING;
2526 goto end;
2527 }
2528
2529 /* Check if DMA work is pending (2nd+ run). */
2530 if (VM_FF_ISPENDING(pVM, VM_FF_PDM_DMA) && cResume > 1)
2531 {
2532 rc = VINF_EM_RAW_TO_R3;
2533 goto end;
2534 }
2535 }
2536
2537#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
2538 /*
2539 * Exit to ring-3 preemption/work is pending.
2540 *
2541 * Interrupts are disabled before the call to make sure we don't miss any interrupt
2542 * that would flag preemption (IPI, timer tick, ++). (Would've been nice to do this
2543 * further down, but hmR0VmxCheckPendingInterrupt makes that impossible.)
2544 *
2545 * Note! Interrupts must be disabled done *before* we check for TLB flushes; TLB
2546 * shootdowns rely on this.
2547 */
2548 uOldEFlags = ASMIntDisableFlags();
2549 if (RTThreadPreemptIsPending(NIL_RTTHREAD))
2550 {
2551 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitPreemptPending);
2552 rc = VINF_EM_RAW_INTERRUPT;
2553 goto end;
2554 }
2555 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
2556#endif
2557
2558 /* When external interrupts are pending, we should exit the VM when IF is set. */
2559 /* Note! *After* VM_FF_INHIBIT_INTERRUPTS check!!! */
2560 rc = hmR0VmxCheckPendingInterrupt(pVM, pVCpu, pCtx);
2561 if (RT_FAILURE(rc))
2562 goto end;
2563
2564 /** @todo check timers?? */
2565
2566 /* TPR caching using CR8 is only available in 64 bits mode */
2567 /* Note the 32 bits exception for AMD (X86_CPUID_AMD_FEATURE_ECX_CR8L), but that appears missing in Intel CPUs */
2568 /* Note: we can't do this in LoadGuestState as PDMApicGetTPR can jump back to ring 3 (lock)!!!!! (no longer true) */
2569 /**
2570 * @todo query and update the TPR only when it could have been changed (mmio access & wrmsr (x2apic))
2571 */
2572 if (fSetupTPRCaching)
2573 {
2574 /* TPR caching in CR8 */
2575 bool fPending;
2576
2577 rc2 = PDMApicGetTPR(pVCpu, &u8LastTPR, &fPending);
2578 AssertRC(rc2);
2579 /* The TPR can be found at offset 0x80 in the APIC mmio page. */
2580 pVCpu->hwaccm.s.vmx.pbVAPIC[0x80] = u8LastTPR;
2581
2582 /* Two options here:
2583 * - external interrupt pending, but masked by the TPR value.
2584 * -> a CR8 update that lower the current TPR value should cause an exit
2585 * - no pending interrupts
2586 * -> We don't need to be explicitely notified. There are enough world switches for detecting pending interrupts.
2587 */
2588 rc = VMXWriteVMCS(VMX_VMCS_CTRL_TPR_THRESHOLD, (fPending) ? (u8LastTPR >> 4) : 0); /* cr8 bits 3-0 correspond to bits 7-4 of the task priority mmio register. */
2589 AssertRC(VBOXSTRICTRC_VAL(rc));
2590
2591 if (pVM->hwaccm.s.fTPRPatchingActive)
2592 {
2593 Assert(!CPUMIsGuestInLongModeEx(pCtx));
2594 /* Our patch code uses LSTAR for TPR caching. */
2595 pCtx->msrLSTAR = u8LastTPR;
2596
2597 if (fPending)
2598 {
2599 /* A TPR change could activate a pending interrupt, so catch lstar writes. */
2600 hmR0VmxSetMSRPermission(pVCpu, MSR_K8_LSTAR, true, false);
2601 }
2602 else
2603 {
2604 /* No interrupts are pending, so we don't need to be explicitely notified.
2605 * There are enough world switches for detecting pending interrupts.
2606 */
2607 hmR0VmxSetMSRPermission(pVCpu, MSR_K8_LSTAR, true, true);
2608 }
2609 }
2610 }
2611
2612#if defined(HWACCM_VTX_WITH_EPT) && defined(LOG_ENABLED)
2613 if ( pVM->hwaccm.s.fNestedPaging
2614# ifdef HWACCM_VTX_WITH_VPID
2615 || pVM->hwaccm.s.vmx.fVPID
2616# endif /* HWACCM_VTX_WITH_VPID */
2617 )
2618 {
2619 PHMGLOBLCPUINFO pCpu;
2620
2621 pCpu = HWACCMR0GetCurrentCpu();
2622 if ( pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu
2623 || pVCpu->hwaccm.s.cTLBFlushes != pCpu->cTLBFlushes)
2624 {
2625 if (pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu)
2626 LogFlow(("Force TLB flush due to rescheduling to a different cpu (%d vs %d)\n", pVCpu->hwaccm.s.idLastCpu, pCpu->idCpu));
2627 else
2628 LogFlow(("Force TLB flush due to changed TLB flush count (%x vs %x)\n", pVCpu->hwaccm.s.cTLBFlushes, pCpu->cTLBFlushes));
2629 }
2630 if (pCpu->fFlushTLB)
2631 LogFlow(("Force TLB flush: first time cpu %d is used -> flush\n", pCpu->idCpu));
2632 else
2633 if (pVCpu->hwaccm.s.fForceTLBFlush)
2634 LogFlow(("Manual TLB flush\n"));
2635 }
2636#endif
2637#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
2638 PGMRZDynMapFlushAutoSet(pVCpu);
2639#endif
2640
2641 /*
2642 * NOTE: DO NOT DO ANYTHING AFTER THIS POINT THAT MIGHT JUMP BACK TO RING 3!
2643 * (until the actual world switch)
2644 */
2645#ifdef VBOX_STRICT
2646 idCpuCheck = RTMpCpuId();
2647#endif
2648#ifdef LOG_ENABLED
2649 VMMR0LogFlushDisable(pVCpu);
2650#endif
2651 /* Save the host state first. */
2652 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_HOST_CONTEXT)
2653 {
2654 rc = VMXR0SaveHostState(pVM, pVCpu);
2655 if (RT_UNLIKELY(rc != VINF_SUCCESS))
2656 {
2657 VMMR0LogFlushEnable(pVCpu);
2658 goto end;
2659 }
2660 }
2661
2662 /* Load the guest state */
2663 if (!pVCpu->hwaccm.s.fContextUseFlags)
2664 {
2665 VMXR0LoadMinimalGuestState(pVM, pVCpu, pCtx);
2666 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatLoadMinimal);
2667 }
2668 else
2669 {
2670 rc = VMXR0LoadGuestState(pVM, pVCpu, pCtx);
2671 if (RT_UNLIKELY(rc != VINF_SUCCESS))
2672 {
2673 VMMR0LogFlushEnable(pVCpu);
2674 goto end;
2675 }
2676 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatLoadFull);
2677 }
2678
2679#ifndef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
2680 /* Disable interrupts to make sure a poke will interrupt execution.
2681 * This must be done *before* we check for TLB flushes; TLB shootdowns rely on this.
2682 */
2683 uOldEFlags = ASMIntDisableFlags();
2684 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
2685#endif
2686
2687 /* Non-register state Guest Context */
2688 /** @todo change me according to cpu state */
2689 rc2 = VMXWriteVMCS(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_CMS_GUEST_ACTIVITY_ACTIVE);
2690 AssertRC(rc2);
2691
2692 /* Set TLB flush state as checked until we return from the world switch. */
2693 ASMAtomicWriteBool(&pVCpu->hwaccm.s.fCheckedTLBFlush, true);
2694 /* Deal with tagged TLB setup and invalidation. */
2695 pVM->hwaccm.s.vmx.pfnSetupTaggedTLB(pVM, pVCpu);
2696
2697 /* Manual save and restore:
2698 * - General purpose registers except RIP, RSP
2699 *
2700 * Trashed:
2701 * - CR2 (we don't care)
2702 * - LDTR (reset to 0)
2703 * - DRx (presumably not changed at all)
2704 * - DR7 (reset to 0x400)
2705 * - EFLAGS (reset to RT_BIT(1); not relevant)
2706 *
2707 */
2708
2709 /* All done! Let's start VM execution. */
2710 STAM_PROFILE_ADV_STOP_START(&pVCpu->hwaccm.s.StatEntry, &pVCpu->hwaccm.s.StatInGC, x);
2711 Assert(idCpuCheck == RTMpCpuId());
2712
2713#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2714 pVCpu->hwaccm.s.vmx.VMCSCache.cResume = cResume;
2715 pVCpu->hwaccm.s.vmx.VMCSCache.u64TimeSwitch = RTTimeNanoTS();
2716#endif
2717
2718 /* Save the current TPR value in the LSTAR msr so our patches can access it. */
2719 if (pVM->hwaccm.s.fTPRPatchingActive)
2720 {
2721 Assert(pVM->hwaccm.s.fTPRPatchingActive);
2722 u64OldLSTAR = ASMRdMsr(MSR_K8_LSTAR);
2723 ASMWrMsr(MSR_K8_LSTAR, u8LastTPR);
2724 }
2725
2726 TMNotifyStartOfExecution(pVCpu);
2727#ifdef VBOX_WITH_KERNEL_USING_XMM
2728 rc = hwaccmR0VMXStartVMWrapXMM(pVCpu->hwaccm.s.fResumeVM, pCtx, &pVCpu->hwaccm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hwaccm.s.vmx.pfnStartVM);
2729#else
2730 rc = pVCpu->hwaccm.s.vmx.pfnStartVM(pVCpu->hwaccm.s.fResumeVM, pCtx, &pVCpu->hwaccm.s.vmx.VMCSCache, pVM, pVCpu);
2731#endif
2732 ASMAtomicWriteBool(&pVCpu->hwaccm.s.fCheckedTLBFlush, false);
2733 ASMAtomicIncU32(&pVCpu->hwaccm.s.cWorldSwitchExits);
2734 /* Possibly the last TSC value seen by the guest (too high) (only when we're in tsc offset mode). */
2735 if (!(pVCpu->hwaccm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT))
2736 TMCpuTickSetLastSeen(pVCpu, ASMReadTSC() + pVCpu->hwaccm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
2737
2738 TMNotifyEndOfExecution(pVCpu);
2739 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED);
2740 Assert(!(ASMGetFlags() & X86_EFL_IF));
2741
2742 /* Restore the host LSTAR msr if the guest could have changed it. */
2743 if (pVM->hwaccm.s.fTPRPatchingActive)
2744 {
2745 Assert(pVM->hwaccm.s.fTPRPatchingActive);
2746 pVCpu->hwaccm.s.vmx.pbVAPIC[0x80] = pCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
2747 ASMWrMsr(MSR_K8_LSTAR, u64OldLSTAR);
2748 }
2749
2750 STAM_PROFILE_ADV_STOP_START(&pVCpu->hwaccm.s.StatInGC, &pVCpu->hwaccm.s.StatExit1, x);
2751 ASMSetFlags(uOldEFlags);
2752#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
2753 uOldEFlags = ~(RTCCUINTREG)0;
2754#endif
2755
2756 AssertMsg(!pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries, ("pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries=%d\n", pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries));
2757
2758 /* In case we execute a goto ResumeExecution later on. */
2759 pVCpu->hwaccm.s.fResumeVM = true;
2760 pVCpu->hwaccm.s.fForceTLBFlush = false;
2761
2762 /*
2763 * !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
2764 * IMPORTANT: WE CAN'T DO ANY LOGGING OR OPERATIONS THAT CAN DO A LONGJMP BACK TO RING 3 *BEFORE* WE'VE SYNCED BACK (MOST OF) THE GUEST STATE
2765 * !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
2766 */
2767
2768 if (RT_UNLIKELY(rc != VINF_SUCCESS))
2769 {
2770 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx);
2771 VMMR0LogFlushEnable(pVCpu);
2772 goto end;
2773 }
2774
2775 /* Success. Query the guest state and figure out what has happened. */
2776
2777 /* Investigate why there was a VM-exit. */
2778 rc2 = VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_REASON, &exitReason);
2779 STAM_COUNTER_INC(&pVCpu->hwaccm.s.paStatExitReasonR0[exitReason & MASK_EXITREASON_STAT]);
2780
2781 exitReason &= 0xffff; /* bit 0-15 contain the exit code. */
2782 rc2 |= VMXReadCachedVMCS(VMX_VMCS32_RO_VM_INSTR_ERROR, &instrError);
2783 rc2 |= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &cbInstr);
2784 rc2 |= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &intInfo);
2785 /* might not be valid; depends on VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID. */
2786 rc2 |= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERRCODE, &errCode);
2787 rc2 |= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INSTR_INFO, &instrInfo);
2788 rc2 |= VMXReadCachedVMCS(VMX_VMCS_RO_EXIT_QUALIFICATION, &exitQualification);
2789 AssertRC(rc2);
2790
2791 /* Sync back the guest state */
2792 rc2 = VMXR0SaveGuestState(pVM, pVCpu, pCtx);
2793 AssertRC(rc2);
2794
2795 /* Note! NOW IT'S SAFE FOR LOGGING! */
2796 VMMR0LogFlushEnable(pVCpu);
2797 Log2(("Raw exit reason %08x\n", exitReason));
2798
2799 /* Check if an injected event was interrupted prematurely. */
2800 rc2 = VMXReadCachedVMCS(VMX_VMCS32_RO_IDT_INFO, &val);
2801 AssertRC(rc2);
2802 pVCpu->hwaccm.s.Event.intInfo = VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(val);
2803 if ( VMX_EXIT_INTERRUPTION_INFO_VALID(pVCpu->hwaccm.s.Event.intInfo)
2804 /* Ignore 'int xx' as they'll be restarted anyway. */
2805 && VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hwaccm.s.Event.intInfo) != VMX_EXIT_INTERRUPTION_INFO_TYPE_SW
2806 /* Ignore software exceptions (such as int3) as they'll reoccur when we restart the instruction anyway. */
2807 && VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hwaccm.s.Event.intInfo) != VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT)
2808 {
2809 Assert(!pVCpu->hwaccm.s.Event.fPending);
2810 pVCpu->hwaccm.s.Event.fPending = true;
2811 /* Error code present? */
2812 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVCpu->hwaccm.s.Event.intInfo))
2813 {
2814 rc2 = VMXReadCachedVMCS(VMX_VMCS32_RO_IDT_ERRCODE, &val);
2815 AssertRC(rc2);
2816 pVCpu->hwaccm.s.Event.errCode = val;
2817 Log(("Pending inject %RX64 at %RGv exit=%08x intInfo=%08x exitQualification=%RGv pending error=%RX64\n", pVCpu->hwaccm.s.Event.intInfo, (RTGCPTR)pCtx->rip, exitReason, intInfo, exitQualification, val));
2818 }
2819 else
2820 {
2821 Log(("Pending inject %RX64 at %RGv exit=%08x intInfo=%08x exitQualification=%RGv\n", pVCpu->hwaccm.s.Event.intInfo, (RTGCPTR)pCtx->rip, exitReason, intInfo, exitQualification));
2822 pVCpu->hwaccm.s.Event.errCode = 0;
2823 }
2824 }
2825#ifdef VBOX_STRICT
2826 else
2827 if ( VMX_EXIT_INTERRUPTION_INFO_VALID(pVCpu->hwaccm.s.Event.intInfo)
2828 /* Ignore software exceptions (such as int3) as they're reoccur when we restart the instruction anyway. */
2829 && VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hwaccm.s.Event.intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT)
2830 {
2831 Log(("Ignore pending inject %RX64 at %RGv exit=%08x intInfo=%08x exitQualification=%RGv\n", pVCpu->hwaccm.s.Event.intInfo, (RTGCPTR)pCtx->rip, exitReason, intInfo, exitQualification));
2832 }
2833
2834 if (exitReason == VMX_EXIT_ERR_INVALID_GUEST_STATE)
2835 HWACCMDumpRegs(pVM, pVCpu, pCtx);
2836#endif
2837
2838 Log2(("E%d: New EIP=%x:%RGv\n", (uint32_t)exitReason, pCtx->cs, (RTGCPTR)pCtx->rip));
2839 Log2(("Exit reason %d, exitQualification %RGv\n", (uint32_t)exitReason, exitQualification));
2840 Log2(("instrInfo=%d instrError=%d instr length=%d\n", (uint32_t)instrInfo, (uint32_t)instrError, (uint32_t)cbInstr));
2841 Log2(("Interruption error code %d\n", (uint32_t)errCode));
2842 Log2(("IntInfo = %08x\n", (uint32_t)intInfo));
2843
2844 /* Sync back the TPR if it was changed. */
2845 if ( fSetupTPRCaching
2846 && u8LastTPR != pVCpu->hwaccm.s.vmx.pbVAPIC[0x80])
2847 {
2848 rc2 = PDMApicSetTPR(pVCpu, pVCpu->hwaccm.s.vmx.pbVAPIC[0x80]);
2849 AssertRC(rc2);
2850 }
2851
2852 STAM_PROFILE_ADV_STOP_START(&pVCpu->hwaccm.s.StatExit1, &pVCpu->hwaccm.s.StatExit2, x);
2853
2854 /* Some cases don't need a complete resync of the guest CPU state; handle them here. */
2855 Assert(rc == VINF_SUCCESS); /* might consider VERR_IPE_UNINITIALIZED_STATUS here later... */
2856 switch (exitReason)
2857 {
2858 case VMX_EXIT_EXCEPTION: /* 0 Exception or non-maskable interrupt (NMI). */
2859 case VMX_EXIT_EXTERNAL_IRQ: /* 1 External interrupt. */
2860 {
2861 uint32_t vector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(intInfo);
2862
2863 if (!VMX_EXIT_INTERRUPTION_INFO_VALID(intInfo))
2864 {
2865 Assert(exitReason == VMX_EXIT_EXTERNAL_IRQ);
2866#if 0 //def VBOX_WITH_VMMR0_DISABLE_PREEMPTION
2867 if ( RTThreadPreemptIsPendingTrusty()
2868 && !RTThreadPreemptIsPending(NIL_RTTHREAD))
2869 goto ResumeExecution;
2870#endif
2871 /* External interrupt; leave to allow it to be dispatched again. */
2872 rc = VINF_EM_RAW_INTERRUPT;
2873 break;
2874 }
2875 STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2876 switch (VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo))
2877 {
2878 case VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI: /* Non-maskable interrupt. */
2879 /* External interrupt; leave to allow it to be dispatched again. */
2880 rc = VINF_EM_RAW_INTERRUPT;
2881 break;
2882
2883 case VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT: /* External hardware interrupt. */
2884 AssertFailed(); /* can't come here; fails the first check. */
2885 break;
2886
2887 case VMX_EXIT_INTERRUPTION_INFO_TYPE_DBEXCPT: /* Unknown why we get this type for #DB */
2888 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT: /* Software exception. (#BP or #OF) */
2889 Assert(vector == 1 || vector == 3 || vector == 4);
2890 /* no break */
2891 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT: /* Hardware exception. */
2892 Log2(("Hardware/software interrupt %d\n", vector));
2893 switch (vector)
2894 {
2895 case X86_XCPT_NM:
2896 {
2897 Log(("#NM fault at %RGv error code %x\n", (RTGCPTR)pCtx->rip, errCode));
2898
2899 /** @todo don't intercept #NM exceptions anymore when we've activated the guest FPU state. */
2900 /* If we sync the FPU/XMM state on-demand, then we can continue execution as if nothing has happened. */
2901 rc = CPUMR0LoadGuestFPU(pVM, pVCpu, pCtx);
2902 if (rc == VINF_SUCCESS)
2903 {
2904 Assert(CPUMIsGuestFPUStateActive(pVCpu));
2905
2906 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitShadowNM);
2907
2908 /* Continue execution. */
2909 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_CR0;
2910
2911 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2912 goto ResumeExecution;
2913 }
2914
2915 Log(("Forward #NM fault to the guest\n"));
2916 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestNM);
2917 rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, 0);
2918 AssertRC(rc2);
2919 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2920 goto ResumeExecution;
2921 }
2922
2923 case X86_XCPT_PF: /* Page fault */
2924 {
2925#ifdef DEBUG
2926 if (pVM->hwaccm.s.fNestedPaging)
2927 { /* A genuine pagefault.
2928 * Forward the trap to the guest by injecting the exception and resuming execution.
2929 */
2930 Log(("Guest page fault at %RGv cr2=%RGv error code %RGv rsp=%RGv\n", (RTGCPTR)pCtx->rip, exitQualification, errCode, (RTGCPTR)pCtx->rsp));
2931
2932 Assert(CPUMIsGuestInPagedProtectedModeEx(pCtx));
2933
2934 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestPF);
2935
2936 /* Now we must update CR2. */
2937 pCtx->cr2 = exitQualification;
2938 rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
2939 AssertRC(rc2);
2940
2941 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2942 goto ResumeExecution;
2943 }
2944#endif
2945 Assert(!pVM->hwaccm.s.fNestedPaging);
2946
2947#ifdef VBOX_HWACCM_WITH_GUEST_PATCHING
2948 /* Shortcut for APIC TPR reads and writes; 32 bits guests only */
2949 if ( pVM->hwaccm.s.fTRPPatchingAllowed
2950 && pVM->hwaccm.s.pGuestPatchMem
2951 && (exitQualification & 0xfff) == 0x080
2952 && !(errCode & X86_TRAP_PF_P) /* not present */
2953 && CPUMGetGuestCPL(pVCpu, CPUMCTX2CORE(pCtx)) == 0
2954 && !CPUMIsGuestInLongModeEx(pCtx)
2955 && pVM->hwaccm.s.cPatches < RT_ELEMENTS(pVM->hwaccm.s.aPatches))
2956 {
2957 RTGCPHYS GCPhysApicBase, GCPhys;
2958 PDMApicGetBase(pVM, &GCPhysApicBase); /* @todo cache this */
2959 GCPhysApicBase &= PAGE_BASE_GC_MASK;
2960
2961 rc = PGMGstGetPage(pVCpu, (RTGCPTR)exitQualification, NULL, &GCPhys);
2962 if ( rc == VINF_SUCCESS
2963 && GCPhys == GCPhysApicBase)
2964 {
2965 /* Only attempt to patch the instruction once. */
2966 PHWACCMTPRPATCH pPatch = (PHWACCMTPRPATCH)RTAvloU32Get(&pVM->hwaccm.s.PatchTree, (AVLOU32KEY)pCtx->eip);
2967 if (!pPatch)
2968 {
2969 rc = VINF_EM_HWACCM_PATCH_TPR_INSTR;
2970 break;
2971 }
2972 }
2973 }
2974#endif
2975
2976 Log2(("Page fault at %RGv error code %x\n", exitQualification, errCode));
2977 /* Exit qualification contains the linear address of the page fault. */
2978 TRPMAssertTrap(pVCpu, X86_XCPT_PF, TRPM_TRAP);
2979 TRPMSetErrorCode(pVCpu, errCode);
2980 TRPMSetFaultAddress(pVCpu, exitQualification);
2981
2982 /* Shortcut for APIC TPR reads and writes. */
2983 if ( (exitQualification & 0xfff) == 0x080
2984 && !(errCode & X86_TRAP_PF_P) /* not present */
2985 && fSetupTPRCaching
2986 && (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
2987 {
2988 RTGCPHYS GCPhysApicBase, GCPhys;
2989 PDMApicGetBase(pVM, &GCPhysApicBase); /* @todo cache this */
2990 GCPhysApicBase &= PAGE_BASE_GC_MASK;
2991
2992 rc = PGMGstGetPage(pVCpu, (RTGCPTR)exitQualification, NULL, &GCPhys);
2993 if ( rc == VINF_SUCCESS
2994 && GCPhys == GCPhysApicBase)
2995 {
2996 Log(("Enable VT-x virtual APIC access filtering\n"));
2997 rc2 = IOMMMIOMapMMIOHCPage(pVM, GCPhysApicBase, pVM->hwaccm.s.vmx.pAPICPhys, X86_PTE_RW | X86_PTE_P);
2998 AssertRC(rc2);
2999 }
3000 }
3001
3002 /* Forward it to our trap handler first, in case our shadow pages are out of sync. */
3003 rc = PGMTrap0eHandler(pVCpu, errCode, CPUMCTX2CORE(pCtx), (RTGCPTR)exitQualification);
3004 Log2(("PGMTrap0eHandler %RGv returned %Rrc\n", (RTGCPTR)pCtx->rip, VBOXSTRICTRC_VAL(rc)));
3005
3006 if (rc == VINF_SUCCESS)
3007 { /* We've successfully synced our shadow pages, so let's just continue execution. */
3008 Log2(("Shadow page fault at %RGv cr2=%RGv error code %x\n", (RTGCPTR)pCtx->rip, exitQualification ,errCode));
3009 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitShadowPF);
3010
3011 TRPMResetTrap(pVCpu);
3012 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3013 goto ResumeExecution;
3014 }
3015 else
3016 if (rc == VINF_EM_RAW_GUEST_TRAP)
3017 { /* A genuine pagefault.
3018 * Forward the trap to the guest by injecting the exception and resuming execution.
3019 */
3020 Log2(("Forward page fault to the guest\n"));
3021
3022 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestPF);
3023 /* The error code might have been changed. */
3024 errCode = TRPMGetErrorCode(pVCpu);
3025
3026 TRPMResetTrap(pVCpu);
3027
3028 /* Now we must update CR2. */
3029 pCtx->cr2 = exitQualification;
3030 rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
3031 AssertRC(rc2);
3032
3033 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3034 goto ResumeExecution;
3035 }
3036#ifdef VBOX_STRICT
3037 if (rc != VINF_EM_RAW_EMULATE_INSTR && rc != VINF_EM_RAW_EMULATE_IO_BLOCK)
3038 Log2(("PGMTrap0eHandler failed with %d\n", VBOXSTRICTRC_VAL(rc)));
3039#endif
3040 /* Need to go back to the recompiler to emulate the instruction. */
3041 TRPMResetTrap(pVCpu);
3042 break;
3043 }
3044
3045 case X86_XCPT_MF: /* Floating point exception. */
3046 {
3047 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestMF);
3048 if (!(pCtx->cr0 & X86_CR0_NE))
3049 {
3050 /* old style FPU error reporting needs some extra work. */
3051 /** @todo don't fall back to the recompiler, but do it manually. */
3052 rc = VINF_EM_RAW_EMULATE_INSTR;
3053 break;
3054 }
3055 Log(("Trap %x at %04X:%RGv\n", vector, pCtx->cs, (RTGCPTR)pCtx->rip));
3056 rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
3057 AssertRC(rc2);
3058
3059 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3060 goto ResumeExecution;
3061 }
3062
3063 case X86_XCPT_DB: /* Debug exception. */
3064 {
3065 uint64_t uDR6;
3066
3067 /* DR6, DR7.GD and IA32_DEBUGCTL.LBR are not updated yet.
3068 *
3069 * Exit qualification bits:
3070 * 3:0 B0-B3 which breakpoint condition was met
3071 * 12:4 Reserved (0)
3072 * 13 BD - debug register access detected
3073 * 14 BS - single step execution or branch taken
3074 * 63:15 Reserved (0)
3075 */
3076 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestDB);
3077
3078 /* Note that we don't support guest and host-initiated debugging at the same time. */
3079
3080 uDR6 = X86_DR6_INIT_VAL;
3081 uDR6 |= (exitQualification & (X86_DR6_B0|X86_DR6_B1|X86_DR6_B2|X86_DR6_B3|X86_DR6_BD|X86_DR6_BS));
3082 rc = DBGFRZTrap01Handler(pVM, pVCpu, CPUMCTX2CORE(pCtx), uDR6);
3083 if (rc == VINF_EM_RAW_GUEST_TRAP)
3084 {
3085 /* Update DR6 here. */
3086 pCtx->dr[6] = uDR6;
3087
3088 /* Resync DR6 if the debug state is active. */
3089 if (CPUMIsGuestDebugStateActive(pVCpu))
3090 ASMSetDR6(pCtx->dr[6]);
3091
3092 /* X86_DR7_GD will be cleared if drx accesses should be trapped inside the guest. */
3093 pCtx->dr[7] &= ~X86_DR7_GD;
3094
3095 /* Paranoia. */
3096 pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
3097 pCtx->dr[7] &= ~(RT_BIT(11) | RT_BIT(12) | RT_BIT(14) | RT_BIT(15)); /* must be zero */
3098 pCtx->dr[7] |= 0x400; /* must be one */
3099
3100 /* Resync DR7 */
3101 rc2 = VMXWriteVMCS64(VMX_VMCS64_GUEST_DR7, pCtx->dr[7]);
3102 AssertRC(rc2);
3103
3104 Log(("Trap %x (debug) at %RGv exit qualification %RX64 dr6=%x dr7=%x\n", vector, (RTGCPTR)pCtx->rip, exitQualification, (uint32_t)pCtx->dr[6], (uint32_t)pCtx->dr[7]));
3105 rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
3106 AssertRC(rc2);
3107
3108 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3109 goto ResumeExecution;
3110 }
3111 /* Return to ring 3 to deal with the debug exit code. */
3112 Log(("Debugger hardware BP at %04x:%RGv (rc=%Rrc)\n", pCtx->cs, pCtx->rip, VBOXSTRICTRC_VAL(rc)));
3113 break;
3114 }
3115
3116 case X86_XCPT_BP: /* Breakpoint. */
3117 {
3118 rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pCtx));
3119 if (rc == VINF_EM_RAW_GUEST_TRAP)
3120 {
3121 Log(("Guest #BP at %04x:%RGv\n", pCtx->cs, pCtx->rip));
3122 rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
3123 AssertRC(rc2);
3124 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3125 goto ResumeExecution;
3126 }
3127 if (rc == VINF_SUCCESS)
3128 {
3129 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3130 goto ResumeExecution;
3131 }
3132 Log(("Debugger BP at %04x:%RGv (rc=%Rrc)\n", pCtx->cs, pCtx->rip, VBOXSTRICTRC_VAL(rc)));
3133 break;
3134 }
3135
3136 case X86_XCPT_GP: /* General protection failure exception.*/
3137 {
3138 uint32_t cbOp;
3139 uint32_t cbSize;
3140 PDISCPUSTATE pDis = &pVCpu->hwaccm.s.DisState;
3141
3142 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestGP);
3143#ifdef VBOX_STRICT
3144 if ( !CPUMIsGuestInRealModeEx(pCtx)
3145 || !pVM->hwaccm.s.vmx.pRealModeTSS)
3146 {
3147 Log(("Trap %x at %04X:%RGv errorCode=%RGv\n", vector, pCtx->cs, (RTGCPTR)pCtx->rip, errCode));
3148 rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
3149 AssertRC(rc2);
3150 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3151 goto ResumeExecution;
3152 }
3153#endif
3154 Assert(CPUMIsGuestInRealModeEx(pCtx));
3155
3156 LogFlow(("Real mode X86_XCPT_GP instruction emulation at %x:%RGv\n", pCtx->cs, (RTGCPTR)pCtx->rip));
3157
3158 rc2 = EMInterpretDisasOne(pVM, pVCpu, CPUMCTX2CORE(pCtx), pDis, &cbOp);
3159 if (RT_SUCCESS(rc2))
3160 {
3161 bool fUpdateRIP = true;
3162
3163 rc = VINF_SUCCESS;
3164 Assert(cbOp == pDis->opsize);
3165 switch (pDis->pCurInstr->opcode)
3166 {
3167 case OP_CLI:
3168 pCtx->eflags.Bits.u1IF = 0;
3169 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCli);
3170 break;
3171
3172 case OP_STI:
3173 pCtx->eflags.Bits.u1IF = 1;
3174 EMSetInhibitInterruptsPC(pVCpu, pCtx->rip + pDis->opsize);
3175 Assert(VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
3176 rc2 = VMXWriteVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
3177 AssertRC(rc2);
3178 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitSti);
3179 break;
3180
3181 case OP_HLT:
3182 fUpdateRIP = false;
3183 rc = VINF_EM_HALT;
3184 pCtx->rip += pDis->opsize;
3185 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitHlt);
3186 break;
3187
3188 case OP_POPF:
3189 {
3190 RTGCPTR GCPtrStack;
3191 uint32_t cbParm;
3192 uint32_t uMask;
3193 X86EFLAGS eflags;
3194
3195 if (pDis->prefix & PREFIX_OPSIZE)
3196 {
3197 cbParm = 4;
3198 uMask = 0xffffffff;
3199 }
3200 else
3201 {
3202 cbParm = 2;
3203 uMask = 0xffff;
3204 }
3205
3206 rc2 = SELMToFlatEx(pVM, DIS_SELREG_SS, CPUMCTX2CORE(pCtx), pCtx->esp & uMask, 0, &GCPtrStack);
3207 if (RT_FAILURE(rc2))
3208 {
3209 rc = VERR_EM_INTERPRETER;
3210 break;
3211 }
3212 eflags.u = 0;
3213 rc2 = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &eflags.u, cbParm);
3214 if (RT_FAILURE(rc2))
3215 {
3216 rc = VERR_EM_INTERPRETER;
3217 break;
3218 }
3219 LogFlow(("POPF %x -> %RGv mask=%x\n", eflags.u, pCtx->rsp, uMask));
3220 pCtx->eflags.u = (pCtx->eflags.u & ~(X86_EFL_POPF_BITS & uMask)) | (eflags.u & X86_EFL_POPF_BITS & uMask);
3221 /* RF cleared when popped in real mode; see pushf description in AMD manual. */
3222 pCtx->eflags.Bits.u1RF = 0;
3223 pCtx->esp += cbParm;
3224 pCtx->esp &= uMask;
3225
3226 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitPopf);
3227 break;
3228 }
3229
3230 case OP_PUSHF:
3231 {
3232 RTGCPTR GCPtrStack;
3233 uint32_t cbParm;
3234 uint32_t uMask;
3235 X86EFLAGS eflags;
3236
3237 if (pDis->prefix & PREFIX_OPSIZE)
3238 {
3239 cbParm = 4;
3240 uMask = 0xffffffff;
3241 }
3242 else
3243 {
3244 cbParm = 2;
3245 uMask = 0xffff;
3246 }
3247
3248 rc2 = SELMToFlatEx(pVM, DIS_SELREG_SS, CPUMCTX2CORE(pCtx), (pCtx->esp - cbParm) & uMask, 0, &GCPtrStack);
3249 if (RT_FAILURE(rc2))
3250 {
3251 rc = VERR_EM_INTERPRETER;
3252 break;
3253 }
3254 eflags = pCtx->eflags;
3255 /* RF & VM cleared when pushed in real mode; see pushf description in AMD manual. */
3256 eflags.Bits.u1RF = 0;
3257 eflags.Bits.u1VM = 0;
3258
3259 rc2 = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &eflags.u, cbParm);
3260 if (RT_FAILURE(rc2))
3261 {
3262 rc = VERR_EM_INTERPRETER;
3263 break;
3264 }
3265 LogFlow(("PUSHF %x -> %RGv\n", eflags.u, GCPtrStack));
3266 pCtx->esp -= cbParm;
3267 pCtx->esp &= uMask;
3268 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitPushf);
3269 break;
3270 }
3271
3272 case OP_IRET:
3273 {
3274 RTGCPTR GCPtrStack;
3275 uint32_t uMask = 0xffff;
3276 uint16_t aIretFrame[3];
3277
3278 if (pDis->prefix & (PREFIX_OPSIZE | PREFIX_ADDRSIZE))
3279 {
3280 rc = VERR_EM_INTERPRETER;
3281 break;
3282 }
3283
3284 rc2 = SELMToFlatEx(pVM, DIS_SELREG_SS, CPUMCTX2CORE(pCtx), pCtx->esp & uMask, 0, &GCPtrStack);
3285 if (RT_FAILURE(rc2))
3286 {
3287 rc = VERR_EM_INTERPRETER;
3288 break;
3289 }
3290 rc2 = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
3291 if (RT_FAILURE(rc2))
3292 {
3293 rc = VERR_EM_INTERPRETER;
3294 break;
3295 }
3296 pCtx->ip = aIretFrame[0];
3297 pCtx->cs = aIretFrame[1];
3298 pCtx->csHid.u64Base = pCtx->cs << 4;
3299 pCtx->eflags.u = (pCtx->eflags.u & ~(X86_EFL_POPF_BITS & uMask)) | (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
3300 pCtx->sp += sizeof(aIretFrame);
3301
3302 LogFlow(("iret to %04x:%x\n", pCtx->cs, pCtx->ip));
3303 fUpdateRIP = false;
3304 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIret);
3305 break;
3306 }
3307
3308 case OP_INT:
3309 {
3310 uint32_t intInfo2;
3311
3312 LogFlow(("Realmode: INT %x\n", pDis->param1.parval & 0xff));
3313 intInfo2 = pDis->param1.parval & 0xff;
3314 intInfo2 |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
3315 intInfo2 |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
3316
3317 rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo2, cbOp, 0);
3318 AssertRC(VBOXSTRICTRC_VAL(rc));
3319 fUpdateRIP = false;
3320 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInt);
3321 break;
3322 }
3323
3324 case OP_INTO:
3325 {
3326 if (pCtx->eflags.Bits.u1OF)
3327 {
3328 uint32_t intInfo2;
3329
3330 LogFlow(("Realmode: INTO\n"));
3331 intInfo2 = X86_XCPT_OF;
3332 intInfo2 |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
3333 intInfo2 |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
3334
3335 rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo2, cbOp, 0);
3336 AssertRC(VBOXSTRICTRC_VAL(rc));
3337 fUpdateRIP = false;
3338 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInt);
3339 }
3340 break;
3341 }
3342
3343 case OP_INT3:
3344 {
3345 uint32_t intInfo2;
3346
3347 LogFlow(("Realmode: INT 3\n"));
3348 intInfo2 = 3;
3349 intInfo2 |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
3350 intInfo2 |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
3351
3352 rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo2, cbOp, 0);
3353 AssertRC(VBOXSTRICTRC_VAL(rc));
3354 fUpdateRIP = false;
3355 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInt);
3356 break;
3357 }
3358
3359 default:
3360 rc = EMInterpretInstructionCPU(pVM, pVCpu, pDis, CPUMCTX2CORE(pCtx), 0, EMCODETYPE_SUPERVISOR, &cbSize);
3361 break;
3362 }
3363
3364 if (rc == VINF_SUCCESS)
3365 {
3366 if (fUpdateRIP)
3367 pCtx->rip += cbOp; /* Move on to the next instruction. */
3368
3369 /* lidt, lgdt can end up here. In the future crx changes as well. Just reload the whole context to be done with it. */
3370 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_ALL;
3371
3372 /* Only resume if successful. */
3373 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3374 goto ResumeExecution;
3375 }
3376 }
3377 else
3378 rc = VERR_EM_INTERPRETER;
3379
3380 AssertMsg(rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_EM_HALT, ("Unexpected rc=%Rrc\n", VBOXSTRICTRC_VAL(rc)));
3381 break;
3382 }
3383
3384#ifdef VBOX_STRICT
3385 case X86_XCPT_XF: /* SIMD exception. */
3386 case X86_XCPT_DE: /* Divide error. */
3387 case X86_XCPT_UD: /* Unknown opcode exception. */
3388 case X86_XCPT_SS: /* Stack segment exception. */
3389 case X86_XCPT_NP: /* Segment not present exception. */
3390 {
3391 switch(vector)
3392 {
3393 case X86_XCPT_DE:
3394 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestDE);
3395 break;
3396 case X86_XCPT_UD:
3397 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestUD);
3398 break;
3399 case X86_XCPT_SS:
3400 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestSS);
3401 break;
3402 case X86_XCPT_NP:
3403 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestNP);
3404 break;
3405 }
3406
3407 Log(("Trap %x at %04X:%RGv\n", vector, pCtx->cs, (RTGCPTR)pCtx->rip));
3408 rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
3409 AssertRC(rc2);
3410
3411 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3412 goto ResumeExecution;
3413 }
3414#endif
3415 default:
3416 if ( CPUMIsGuestInRealModeEx(pCtx)
3417 && pVM->hwaccm.s.vmx.pRealModeTSS)
3418 {
3419 Log(("Real Mode Trap %x at %04x:%04X error code %x\n", vector, pCtx->cs, pCtx->eip, errCode));
3420 rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
3421 AssertRC(VBOXSTRICTRC_VAL(rc)); /* Strict RC check below. */
3422
3423 /* Go back to ring 3 in case of a triple fault. */
3424 if ( vector == X86_XCPT_DF
3425 && rc == VINF_EM_RESET)
3426 break;
3427
3428 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3429 goto ResumeExecution;
3430 }
3431 AssertMsgFailed(("Unexpected vm-exit caused by exception %x\n", vector));
3432 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
3433 break;
3434 } /* switch (vector) */
3435
3436 break;
3437
3438 default:
3439 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_CODE;
3440 AssertMsgFailed(("Unexpected interruption code %x\n", intInfo));
3441 break;
3442 }
3443
3444 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3445 break;
3446 }
3447
3448 case VMX_EXIT_EPT_VIOLATION: /* 48 EPT violation. An attempt to access memory with a guest-physical address was disallowed by the configuration of the EPT paging structures. */
3449 {
3450 RTGCPHYS GCPhys;
3451
3452 Assert(pVM->hwaccm.s.fNestedPaging);
3453
3454 rc2 = VMXReadVMCS64(VMX_VMCS_EXIT_PHYS_ADDR_FULL, &GCPhys);
3455 AssertRC(rc2);
3456 Assert(((exitQualification >> 7) & 3) != 2);
3457
3458 /* Determine the kind of violation. */
3459 errCode = 0;
3460 if (exitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
3461 errCode |= X86_TRAP_PF_ID;
3462
3463 if (exitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
3464 errCode |= X86_TRAP_PF_RW;
3465
3466 /* If the page is present, then it's a page level protection fault. */
3467 if (exitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
3468 {
3469 errCode |= X86_TRAP_PF_P;
3470 }
3471 else
3472 {
3473 /* Shortcut for APIC TPR reads and writes. */
3474 if ( (GCPhys & 0xfff) == 0x080
3475 && GCPhys > 0x1000000 /* to skip VGA frame buffer accesses */
3476 && fSetupTPRCaching
3477 && (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
3478 {
3479 RTGCPHYS GCPhysApicBase;
3480 PDMApicGetBase(pVM, &GCPhysApicBase); /* @todo cache this */
3481 GCPhysApicBase &= PAGE_BASE_GC_MASK;
3482 if (GCPhys == GCPhysApicBase + 0x80)
3483 {
3484 Log(("Enable VT-x virtual APIC access filtering\n"));
3485 rc2 = IOMMMIOMapMMIOHCPage(pVM, GCPhysApicBase, pVM->hwaccm.s.vmx.pAPICPhys, X86_PTE_RW | X86_PTE_P);
3486 AssertRC(rc2);
3487 }
3488 }
3489 }
3490 Log(("EPT Page fault %x at %RGp error code %x\n", (uint32_t)exitQualification, GCPhys, errCode));
3491
3492 /* GCPhys contains the guest physical address of the page fault. */
3493 TRPMAssertTrap(pVCpu, X86_XCPT_PF, TRPM_TRAP);
3494 TRPMSetErrorCode(pVCpu, errCode);
3495 TRPMSetFaultAddress(pVCpu, GCPhys);
3496
3497 /* Handle the pagefault trap for the nested shadow table. */
3498 rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, errCode, CPUMCTX2CORE(pCtx), GCPhys);
3499 Log2(("PGMR0Trap0eHandlerNestedPaging %RGv returned %Rrc\n", (RTGCPTR)pCtx->rip, VBOXSTRICTRC_VAL(rc)));
3500 if (rc == VINF_SUCCESS)
3501 { /* We've successfully synced our shadow pages, so let's just continue execution. */
3502 Log2(("Shadow page fault at %RGv cr2=%RGp error code %x\n", (RTGCPTR)pCtx->rip, exitQualification , errCode));
3503 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitReasonNPF);
3504
3505 TRPMResetTrap(pVCpu);
3506 goto ResumeExecution;
3507 }
3508
3509#ifdef VBOX_STRICT
3510 if (rc != VINF_EM_RAW_EMULATE_INSTR)
3511 LogFlow(("PGMTrap0eHandlerNestedPaging failed with %d\n", VBOXSTRICTRC_VAL(rc)));
3512#endif
3513 /* Need to go back to the recompiler to emulate the instruction. */
3514 TRPMResetTrap(pVCpu);
3515 break;
3516 }
3517
3518 case VMX_EXIT_EPT_MISCONFIG:
3519 {
3520 RTGCPHYS GCPhys;
3521
3522 Assert(pVM->hwaccm.s.fNestedPaging);
3523
3524 rc2 = VMXReadVMCS64(VMX_VMCS_EXIT_PHYS_ADDR_FULL, &GCPhys);
3525 AssertRC(rc2);
3526 Log(("VMX_EXIT_EPT_MISCONFIG for %RGp\n", GCPhys));
3527
3528 /* Shortcut for APIC TPR reads and writes. */
3529 if ( (GCPhys & 0xfff) == 0x080
3530 && GCPhys > 0x1000000 /* to skip VGA frame buffer accesses */
3531 && fSetupTPRCaching
3532 && (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
3533 {
3534 RTGCPHYS GCPhysApicBase;
3535 PDMApicGetBase(pVM, &GCPhysApicBase); /* @todo cache this */
3536 GCPhysApicBase &= PAGE_BASE_GC_MASK;
3537 if (GCPhys == GCPhysApicBase + 0x80)
3538 {
3539 Log(("Enable VT-x virtual APIC access filtering\n"));
3540 rc2 = IOMMMIOMapMMIOHCPage(pVM, GCPhysApicBase, pVM->hwaccm.s.vmx.pAPICPhys, X86_PTE_RW | X86_PTE_P);
3541 AssertRC(rc2);
3542 }
3543 }
3544
3545 rc = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pCtx), GCPhys, UINT32_MAX);
3546 if (rc == VINF_SUCCESS)
3547 {
3548 Log2(("PGMR0Trap0eHandlerNPMisconfig(,,,%RGp) at %RGv -> resume\n", GCPhys, (RTGCPTR)pCtx->rip));
3549 goto ResumeExecution;
3550 }
3551
3552 Log2(("PGMR0Trap0eHandlerNPMisconfig(,,,%RGp) at %RGv -> %Rrc\n", GCPhys, (RTGCPTR)pCtx->rip, VBOXSTRICTRC_VAL(rc)));
3553 break;
3554 }
3555
3556 case VMX_EXIT_IRQ_WINDOW: /* 7 Interrupt window. */
3557 /* Clear VM-exit on IF=1 change. */
3558 LogFlow(("VMX_EXIT_IRQ_WINDOW %RGv pending=%d IF=%d\n", (RTGCPTR)pCtx->rip, VMCPU_FF_ISPENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC|VMCPU_FF_INTERRUPT_PIC)), pCtx->eflags.Bits.u1IF));
3559 pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_IRQ_WINDOW_EXIT;
3560 rc2 = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
3561 AssertRC(rc2);
3562 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIrqWindow);
3563 goto ResumeExecution; /* we check for pending guest interrupts there */
3564
3565 case VMX_EXIT_WBINVD: /* 54 Guest software attempted to execute WBINVD. (conditional) */
3566 case VMX_EXIT_INVD: /* 13 Guest software attempted to execute INVD. (unconditional) */
3567 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInvd);
3568 /* Skip instruction and continue directly. */
3569 pCtx->rip += cbInstr;
3570 /* Continue execution.*/
3571 goto ResumeExecution;
3572
3573 case VMX_EXIT_CPUID: /* 10 Guest software attempted to execute CPUID. */
3574 {
3575 Log2(("VMX: Cpuid %x\n", pCtx->eax));
3576 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCpuid);
3577 rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pCtx));
3578 if (rc == VINF_SUCCESS)
3579 {
3580 /* Update EIP and continue execution. */
3581 Assert(cbInstr == 2);
3582 pCtx->rip += cbInstr;
3583 goto ResumeExecution;
3584 }
3585 AssertMsgFailed(("EMU: cpuid failed with %Rrc\n", VBOXSTRICTRC_VAL(rc)));
3586 rc = VINF_EM_RAW_EMULATE_INSTR;
3587 break;
3588 }
3589
3590 case VMX_EXIT_RDPMC: /* 15 Guest software attempted to execute RDPMC. */
3591 {
3592 Log2(("VMX: Rdpmc %x\n", pCtx->ecx));
3593 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitRdpmc);
3594 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pCtx));
3595 if (rc == VINF_SUCCESS)
3596 {
3597 /* Update EIP and continue execution. */
3598 Assert(cbInstr == 2);
3599 pCtx->rip += cbInstr;
3600 goto ResumeExecution;
3601 }
3602 rc = VINF_EM_RAW_EMULATE_INSTR;
3603 break;
3604 }
3605
3606 case VMX_EXIT_RDTSC: /* 16 Guest software attempted to execute RDTSC. */
3607 {
3608 Log2(("VMX: Rdtsc\n"));
3609 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitRdtsc);
3610 rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pCtx));
3611 if (rc == VINF_SUCCESS)
3612 {
3613 /* Update EIP and continue execution. */
3614 Assert(cbInstr == 2);
3615 pCtx->rip += cbInstr;
3616 goto ResumeExecution;
3617 }
3618 rc = VINF_EM_RAW_EMULATE_INSTR;
3619 break;
3620 }
3621
3622 case VMX_EXIT_INVPG: /* 14 Guest software attempted to execute INVPG. */
3623 {
3624 Log2(("VMX: invlpg\n"));
3625 Assert(!pVM->hwaccm.s.fNestedPaging);
3626
3627 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInvpg);
3628 rc = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pCtx), exitQualification);
3629 if (rc == VINF_SUCCESS)
3630 {
3631 /* Update EIP and continue execution. */
3632 pCtx->rip += cbInstr;
3633 goto ResumeExecution;
3634 }
3635 AssertMsg(rc == VERR_EM_INTERPRETER, ("EMU: invlpg %RGv failed with %Rrc\n", exitQualification, VBOXSTRICTRC_VAL(rc)));
3636 break;
3637 }
3638
3639 case VMX_EXIT_MONITOR: /* 39 Guest software attempted to execute MONITOR. */
3640 {
3641 Log2(("VMX: monitor\n"));
3642
3643 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitMonitor);
3644 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pCtx));
3645 if (rc == VINF_SUCCESS)
3646 {
3647 /* Update EIP and continue execution. */
3648 pCtx->rip += cbInstr;
3649 goto ResumeExecution;
3650 }
3651 AssertMsg(rc == VERR_EM_INTERPRETER, ("EMU: monitor failed with %Rrc\n", VBOXSTRICTRC_VAL(rc)));
3652 break;
3653 }
3654
3655 case VMX_EXIT_WRMSR: /* 32 WRMSR. Guest software attempted to execute WRMSR. */
3656 /* When an interrupt is pending, we'll let MSR_K8_LSTAR writes fault in our TPR patch code. */
3657 if ( pVM->hwaccm.s.fTPRPatchingActive
3658 && pCtx->ecx == MSR_K8_LSTAR)
3659 {
3660 Assert(!CPUMIsGuestInLongModeEx(pCtx));
3661 if ((pCtx->eax & 0xff) != u8LastTPR)
3662 {
3663 Log(("VMX: Faulting MSR_K8_LSTAR write with new TPR value %x\n", pCtx->eax & 0xff));
3664
3665 /* Our patch code uses LSTAR for TPR caching. */
3666 rc2 = PDMApicSetTPR(pVCpu, pCtx->eax & 0xff);
3667 AssertRC(rc2);
3668 }
3669
3670 /* Skip the instruction and continue. */
3671 pCtx->rip += cbInstr; /* wrmsr = [0F 30] */
3672
3673 /* Only resume if successful. */
3674 goto ResumeExecution;
3675 }
3676 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_MSR;
3677 /* no break */
3678 case VMX_EXIT_RDMSR: /* 31 RDMSR. Guest software attempted to execute RDMSR. */
3679 {
3680 uint32_t cbSize;
3681
3682 STAM_COUNTER_INC((exitReason == VMX_EXIT_RDMSR) ? &pVCpu->hwaccm.s.StatExitRdmsr : &pVCpu->hwaccm.s.StatExitWrmsr);
3683
3684 /* Note: the intel manual claims there's a REX version of RDMSR that's slightly different, so we play safe by completely disassembling the instruction. */
3685 Log2(("VMX: %s\n", (exitReason == VMX_EXIT_RDMSR) ? "rdmsr" : "wrmsr"));
3686 rc = EMInterpretInstruction(pVM, pVCpu, CPUMCTX2CORE(pCtx), 0, &cbSize);
3687 if (rc == VINF_SUCCESS)
3688 {
3689 /* EIP has been updated already. */
3690
3691 /* Only resume if successful. */
3692 goto ResumeExecution;
3693 }
3694 AssertMsg(rc == VERR_EM_INTERPRETER, ("EMU: %s failed with %Rrc\n", (exitReason == VMX_EXIT_RDMSR) ? "rdmsr" : "wrmsr", VBOXSTRICTRC_VAL(rc)));
3695 break;
3696 }
3697
3698 case VMX_EXIT_CRX_MOVE: /* 28 Control-register accesses. */
3699 {
3700 STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatExit2Sub2, y2);
3701
3702 switch (VMX_EXIT_QUALIFICATION_CRX_ACCESS(exitQualification))
3703 {
3704 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE:
3705 Log2(("VMX: %RGv mov cr%d, x\n", (RTGCPTR)pCtx->rip, VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification)));
3706 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification)]);
3707 rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
3708 VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification),
3709 VMX_EXIT_QUALIFICATION_CRX_GENREG(exitQualification));
3710
3711 switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification))
3712 {
3713 case 0:
3714 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_CR0 | HWACCM_CHANGED_GUEST_CR3;
3715 break;
3716 case 2:
3717 break;
3718 case 3:
3719 Assert(!pVM->hwaccm.s.fNestedPaging || !CPUMIsGuestInPagedProtectedModeEx(pCtx));
3720 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_CR3;
3721 break;
3722 case 4:
3723 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_CR4;
3724 break;
3725 case 8:
3726 /* CR8 contains the APIC TPR */
3727 Assert(!(pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
3728 break;
3729
3730 default:
3731 AssertFailed();
3732 break;
3733 }
3734 break;
3735
3736 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ:
3737 Log2(("VMX: mov x, crx\n"));
3738 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification)]);
3739
3740 Assert(!pVM->hwaccm.s.fNestedPaging || !CPUMIsGuestInPagedProtectedModeEx(pCtx) || VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification) != USE_REG_CR3);
3741
3742 /* CR8 reads only cause an exit when the TPR shadow feature isn't present. */
3743 Assert(VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification) != 8 || !(pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
3744
3745 rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
3746 VMX_EXIT_QUALIFICATION_CRX_GENREG(exitQualification),
3747 VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification));
3748 break;
3749
3750 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS:
3751 Log2(("VMX: clts\n"));
3752 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCLTS);
3753 rc = EMInterpretCLTS(pVM, pVCpu);
3754 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_CR0;
3755 break;
3756
3757 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW:
3758 Log2(("VMX: lmsw %x\n", VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(exitQualification)));
3759 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitLMSW);
3760 rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(exitQualification));
3761 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_CR0;
3762 break;
3763 }
3764
3765 /* Update EIP if no error occurred. */
3766 if (RT_SUCCESS(rc))
3767 pCtx->rip += cbInstr;
3768
3769 if (rc == VINF_SUCCESS)
3770 {
3771 /* Only resume if successful. */
3772 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub2, y2);
3773 goto ResumeExecution;
3774 }
3775 Assert(rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_PGM_SYNC_CR3);
3776 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub2, y2);
3777 break;
3778 }
3779
3780 case VMX_EXIT_DRX_MOVE: /* 29 Debug-register accesses. */
3781 {
3782 if ( !DBGFIsStepping(pVCpu)
3783 && !CPUMIsHyperDebugStateActive(pVCpu))
3784 {
3785 /* Disable drx move intercepts. */
3786 pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
3787 rc2 = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
3788 AssertRC(rc2);
3789
3790 /* Save the host and load the guest debug state. */
3791 rc2 = CPUMR0LoadGuestDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
3792 AssertRC(rc2);
3793
3794#ifdef LOG_ENABLED
3795 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
3796 Log(("VMX_EXIT_DRX_MOVE: write DR%d genreg %d\n", VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification), VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification)));
3797 else
3798 Log(("VMX_EXIT_DRX_MOVE: read DR%d\n", VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification)));
3799#endif
3800
3801#ifdef VBOX_WITH_STATISTICS
3802 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatDRxContextSwitch);
3803 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
3804 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxWrite);
3805 else
3806 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxRead);
3807#endif
3808
3809 goto ResumeExecution;
3810 }
3811
3812 /** @todo clear VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT after the first time and restore drx registers afterwards */
3813 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
3814 {
3815 Log2(("VMX: mov drx%d, genreg%d\n", VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification), VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification)));
3816 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxWrite);
3817 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
3818 VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification),
3819 VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification));
3820 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_DEBUG;
3821 Log2(("DR7=%08x\n", pCtx->dr[7]));
3822 }
3823 else
3824 {
3825 Log2(("VMX: mov x, drx\n"));
3826 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxRead);
3827 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
3828 VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification),
3829 VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification));
3830 }
3831 /* Update EIP if no error occurred. */
3832 if (RT_SUCCESS(rc))
3833 pCtx->rip += cbInstr;
3834
3835 if (rc == VINF_SUCCESS)
3836 {
3837 /* Only resume if successful. */
3838 goto ResumeExecution;
3839 }
3840 Assert(rc == VERR_EM_INTERPRETER);
3841 break;
3842 }
3843
3844 /* Note: We'll get a #GP if the IO instruction isn't allowed (IOPL or TSS bitmap); no need to double check. */
3845 case VMX_EXIT_PORT_IO: /* 30 I/O instruction. */
3846 {
3847 STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3848 uint32_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(exitQualification);
3849 uint32_t uPort;
3850 bool fIOWrite = (VMX_EXIT_QUALIFICATION_IO_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
3851
3852 /** @todo necessary to make the distinction? */
3853 if (VMX_EXIT_QUALIFICATION_IO_ENCODING(exitQualification) == VMX_EXIT_QUALIFICATION_IO_ENCODING_DX)
3854 {
3855 uPort = pCtx->edx & 0xffff;
3856 }
3857 else
3858 uPort = VMX_EXIT_QUALIFICATION_IO_PORT(exitQualification); /* Immediate encoding. */
3859
3860 /* paranoia */
3861 if (RT_UNLIKELY(uIOWidth == 2 || uIOWidth >= 4))
3862 {
3863 rc = fIOWrite ? VINF_IOM_HC_IOPORT_WRITE : VINF_IOM_HC_IOPORT_READ;
3864 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3865 break;
3866 }
3867
3868 uint32_t cbSize = g_aIOSize[uIOWidth];
3869
3870 if (VMX_EXIT_QUALIFICATION_IO_STRING(exitQualification))
3871 {
3872 /* ins/outs */
3873 PDISCPUSTATE pDis = &pVCpu->hwaccm.s.DisState;
3874
3875 /* Disassemble manually to deal with segment prefixes. */
3876 /** @todo VMX_VMCS_EXIT_GUEST_LINEAR_ADDR contains the flat pointer operand of the instruction. */
3877 /** @todo VMX_VMCS32_RO_EXIT_INSTR_INFO also contains segment prefix info. */
3878 rc2 = EMInterpretDisasOne(pVM, pVCpu, CPUMCTX2CORE(pCtx), pDis, NULL);
3879 if (RT_SUCCESS(rc))
3880 {
3881 if (fIOWrite)
3882 {
3883 Log2(("IOMInterpretOUTSEx %RGv %x size=%d\n", (RTGCPTR)pCtx->rip, uPort, cbSize));
3884 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIOStringWrite);
3885 rc = IOMInterpretOUTSEx(pVM, CPUMCTX2CORE(pCtx), uPort, pDis->prefix, pDis->addrmode, cbSize);
3886 }
3887 else
3888 {
3889 Log2(("IOMInterpretINSEx %RGv %x size=%d\n", (RTGCPTR)pCtx->rip, uPort, cbSize));
3890 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIOStringRead);
3891 rc = IOMInterpretINSEx(pVM, CPUMCTX2CORE(pCtx), uPort, pDis->prefix, pDis->addrmode, cbSize);
3892 }
3893 }
3894 else
3895 rc = VINF_EM_RAW_EMULATE_INSTR;
3896 }
3897 else
3898 {
3899 /* normal in/out */
3900 uint32_t uAndVal = g_aIOOpAnd[uIOWidth];
3901
3902 Assert(!VMX_EXIT_QUALIFICATION_IO_REP(exitQualification));
3903
3904 if (fIOWrite)
3905 {
3906 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIOWrite);
3907 rc = IOMIOPortWrite(pVM, uPort, pCtx->eax & uAndVal, cbSize);
3908 if (rc == VINF_IOM_HC_IOPORT_WRITE)
3909 HWACCMR0SavePendingIOPortWrite(pVCpu, pCtx->rip, pCtx->rip + cbInstr, uPort, uAndVal, cbSize);
3910 }
3911 else
3912 {
3913 uint32_t u32Val = 0;
3914
3915 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIORead);
3916 rc = IOMIOPortRead(pVM, uPort, &u32Val, cbSize);
3917 if (IOM_SUCCESS(rc))
3918 {
3919 /* Write back to the EAX register. */
3920 pCtx->eax = (pCtx->eax & ~uAndVal) | (u32Val & uAndVal);
3921 }
3922 else
3923 if (rc == VINF_IOM_HC_IOPORT_READ)
3924 HWACCMR0SavePendingIOPortRead(pVCpu, pCtx->rip, pCtx->rip + cbInstr, uPort, uAndVal, cbSize);
3925 }
3926 }
3927 /*
3928 * Handled the I/O return codes.
3929 * (The unhandled cases end up with rc == VINF_EM_RAW_EMULATE_INSTR.)
3930 */
3931 if (IOM_SUCCESS(rc))
3932 {
3933 /* Update EIP and continue execution. */
3934 pCtx->rip += cbInstr;
3935 if (RT_LIKELY(rc == VINF_SUCCESS))
3936 {
3937 /* If any IO breakpoints are armed, then we should check if a debug trap needs to be generated. */
3938 if (pCtx->dr[7] & X86_DR7_ENABLED_MASK)
3939 {
3940 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatDRxIOCheck);
3941 for (unsigned i=0;i<4;i++)
3942 {
3943 unsigned uBPLen = g_aIOSize[X86_DR7_GET_LEN(pCtx->dr[7], i)];
3944
3945 if ( (uPort >= pCtx->dr[i] && uPort < pCtx->dr[i] + uBPLen)
3946 && (pCtx->dr[7] & (X86_DR7_L(i) | X86_DR7_G(i)))
3947 && (pCtx->dr[7] & X86_DR7_RW(i, X86_DR7_RW_IO)) == X86_DR7_RW(i, X86_DR7_RW_IO))
3948 {
3949 uint64_t uDR6;
3950
3951 Assert(CPUMIsGuestDebugStateActive(pVCpu));
3952
3953 uDR6 = ASMGetDR6();
3954
3955 /* Clear all breakpoint status flags and set the one we just hit. */
3956 uDR6 &= ~(X86_DR6_B0|X86_DR6_B1|X86_DR6_B2|X86_DR6_B3);
3957 uDR6 |= (uint64_t)RT_BIT(i);
3958
3959 /* Note: AMD64 Architecture Programmer's Manual 13.1:
3960 * Bits 15:13 of the DR6 register is never cleared by the processor and must be cleared by software after
3961 * the contents have been read.
3962 */
3963 ASMSetDR6(uDR6);
3964
3965 /* X86_DR7_GD will be cleared if drx accesses should be trapped inside the guest. */
3966 pCtx->dr[7] &= ~X86_DR7_GD;
3967
3968 /* Paranoia. */
3969 pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
3970 pCtx->dr[7] &= ~(RT_BIT(11) | RT_BIT(12) | RT_BIT(14) | RT_BIT(15)); /* must be zero */
3971 pCtx->dr[7] |= 0x400; /* must be one */
3972
3973 /* Resync DR7 */
3974 rc2 = VMXWriteVMCS64(VMX_VMCS64_GUEST_DR7, pCtx->dr[7]);
3975 AssertRC(rc2);
3976
3977 /* Construct inject info. */
3978 intInfo = X86_XCPT_DB;
3979 intInfo |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
3980 intInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
3981
3982 Log(("Inject IO debug trap at %RGv\n", (RTGCPTR)pCtx->rip));
3983 rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), 0, 0);
3984 AssertRC(rc2);
3985
3986 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3987 goto ResumeExecution;
3988 }
3989 }
3990 }
3991 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3992 goto ResumeExecution;
3993 }
3994 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3995 break;
3996 }
3997
3998#ifdef VBOX_STRICT
3999 if (rc == VINF_IOM_HC_IOPORT_READ)
4000 Assert(!fIOWrite);
4001 else if (rc == VINF_IOM_HC_IOPORT_WRITE)
4002 Assert(fIOWrite);
4003 else
4004 AssertMsg(RT_FAILURE(rc) || rc == VINF_EM_RAW_EMULATE_INSTR || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rc)));
4005#endif
4006 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
4007 break;
4008 }
4009
4010 case VMX_EXIT_TPR: /* 43 TPR below threshold. Guest software executed MOV to CR8. */
4011 LogFlow(("VMX_EXIT_TPR\n"));
4012 /* RIP is already set to the next instruction and the TPR has been synced back. Just resume. */
4013 goto ResumeExecution;
4014
4015 case VMX_EXIT_APIC_ACCESS: /* 44 APIC access. Guest software attempted to access memory at a physical address on the APIC-access page. */
4016 {
4017 LogFlow(("VMX_EXIT_APIC_ACCESS\n"));
4018 unsigned uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(exitQualification);
4019
4020 switch(uAccessType)
4021 {
4022 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
4023 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
4024 {
4025 RTGCPHYS GCPhys;
4026 PDMApicGetBase(pVM, &GCPhys);
4027 GCPhys &= PAGE_BASE_GC_MASK;
4028 GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(exitQualification);
4029
4030 LogFlow(("Apic access at %RGp\n", GCPhys));
4031 rc = IOMMMIOPhysHandler(pVM, (uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW, CPUMCTX2CORE(pCtx), GCPhys);
4032 if (rc == VINF_SUCCESS)
4033 goto ResumeExecution; /* rip already updated */
4034 break;
4035 }
4036
4037 default:
4038 rc = VINF_EM_RAW_EMULATE_INSTR;
4039 break;
4040 }
4041 break;
4042 }
4043
4044 case VMX_EXIT_PREEMPTION_TIMER: /* 52 VMX-preemption timer expired. The preemption timer counted down to zero. */
4045 if (!TMTimerPollBool(pVM, pVCpu))
4046 goto ResumeExecution;
4047 rc = VINF_EM_RAW_TIMER_PENDING;
4048 break;
4049
4050 default:
4051 /* The rest is handled after syncing the entire CPU state. */
4052 break;
4053 }
4054
4055 /* Note: the guest state isn't entirely synced back at this stage. */
4056
4057 /* Investigate why there was a VM-exit. (part 2) */
4058 switch (exitReason)
4059 {
4060 case VMX_EXIT_EXCEPTION: /* 0 Exception or non-maskable interrupt (NMI). */
4061 case VMX_EXIT_EXTERNAL_IRQ: /* 1 External interrupt. */
4062 case VMX_EXIT_EPT_VIOLATION:
4063 case VMX_EXIT_EPT_MISCONFIG: /* 49 EPT misconfig is used by the PGM/MMIO optimizations. */
4064 case VMX_EXIT_PREEMPTION_TIMER: /* 52 VMX-preemption timer expired. The preemption timer counted down to zero. */
4065 /* Already handled above. */
4066 break;
4067
4068 case VMX_EXIT_TRIPLE_FAULT: /* 2 Triple fault. */
4069 rc = VINF_EM_RESET; /* Triple fault equals a reset. */
4070 break;
4071
4072 case VMX_EXIT_INIT_SIGNAL: /* 3 INIT signal. */
4073 case VMX_EXIT_SIPI: /* 4 Start-up IPI (SIPI). */
4074 rc = VINF_EM_RAW_INTERRUPT;
4075 AssertFailed(); /* Can't happen. Yet. */
4076 break;
4077
4078 case VMX_EXIT_IO_SMI_IRQ: /* 5 I/O system-management interrupt (SMI). */
4079 case VMX_EXIT_SMI_IRQ: /* 6 Other SMI. */
4080 rc = VINF_EM_RAW_INTERRUPT;
4081 AssertFailed(); /* Can't happen afaik. */
4082 break;
4083
4084 case VMX_EXIT_TASK_SWITCH: /* 9 Task switch: too complicated to emulate, so fall back to the recompiler */
4085 Log(("VMX_EXIT_TASK_SWITCH: exit=%RX64\n", exitQualification));
4086 if ( (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(exitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
4087 && pVCpu->hwaccm.s.Event.fPending)
4088 {
4089 /* Caused by an injected interrupt. */
4090 pVCpu->hwaccm.s.Event.fPending = false;
4091
4092 Log(("VMX_EXIT_TASK_SWITCH: reassert trap %d\n", VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVCpu->hwaccm.s.Event.intInfo)));
4093 Assert(!VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVCpu->hwaccm.s.Event.intInfo));
4094 rc2 = TRPMAssertTrap(pVCpu, VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVCpu->hwaccm.s.Event.intInfo), TRPM_HARDWARE_INT);
4095 AssertRC(rc2);
4096 }
4097 /* else Exceptions and software interrupts can just be restarted. */
4098 rc = VERR_EM_INTERPRETER;
4099 break;
4100
4101 case VMX_EXIT_HLT: /* 12 Guest software attempted to execute HLT. */
4102 /** Check if external interrupts are pending; if so, don't switch back. */
4103 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitHlt);
4104 pCtx->rip++; /* skip hlt */
4105 if (EMShouldContinueAfterHalt(pVCpu, pCtx))
4106 goto ResumeExecution;
4107
4108 rc = VINF_EM_HALT;
4109 break;
4110
4111 case VMX_EXIT_MWAIT: /* 36 Guest software executed MWAIT. */
4112 Log2(("VMX: mwait\n"));
4113 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitMwait);
4114 rc = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pCtx));
4115 if ( rc == VINF_EM_HALT
4116 || rc == VINF_SUCCESS)
4117 {
4118 /* Update EIP and continue execution. */
4119 pCtx->rip += cbInstr;
4120
4121 /** Check if external interrupts are pending; if so, don't switch back. */
4122 if ( rc == VINF_SUCCESS
4123 || ( rc == VINF_EM_HALT
4124 && EMShouldContinueAfterHalt(pVCpu, pCtx))
4125 )
4126 goto ResumeExecution;
4127 }
4128 AssertMsg(rc == VERR_EM_INTERPRETER || rc == VINF_EM_HALT, ("EMU: mwait failed with %Rrc\n", VBOXSTRICTRC_VAL(rc)));
4129 break;
4130
4131 case VMX_EXIT_RSM: /* 17 Guest software attempted to execute RSM in SMM. */
4132 AssertFailed(); /* can't happen. */
4133 rc = VERR_EM_INTERPRETER;
4134 break;
4135
4136 case VMX_EXIT_VMCALL: /* 18 Guest software executed VMCALL. */
4137 case VMX_EXIT_VMCLEAR: /* 19 Guest software executed VMCLEAR. */
4138 case VMX_EXIT_VMLAUNCH: /* 20 Guest software executed VMLAUNCH. */
4139 case VMX_EXIT_VMPTRLD: /* 21 Guest software executed VMPTRLD. */
4140 case VMX_EXIT_VMPTRST: /* 22 Guest software executed VMPTRST. */
4141 case VMX_EXIT_VMREAD: /* 23 Guest software executed VMREAD. */
4142 case VMX_EXIT_VMRESUME: /* 24 Guest software executed VMRESUME. */
4143 case VMX_EXIT_VMWRITE: /* 25 Guest software executed VMWRITE. */
4144 case VMX_EXIT_VMXOFF: /* 26 Guest software executed VMXOFF. */
4145 case VMX_EXIT_VMXON: /* 27 Guest software executed VMXON. */
4146 /** @todo inject #UD immediately */
4147 rc = VERR_EM_INTERPRETER;
4148 break;
4149
4150 case VMX_EXIT_CPUID: /* 10 Guest software attempted to execute CPUID. */
4151 case VMX_EXIT_RDTSC: /* 16 Guest software attempted to execute RDTSC. */
4152 case VMX_EXIT_INVPG: /* 14 Guest software attempted to execute INVPG. */
4153 case VMX_EXIT_CRX_MOVE: /* 28 Control-register accesses. */
4154 case VMX_EXIT_DRX_MOVE: /* 29 Debug-register accesses. */
4155 case VMX_EXIT_PORT_IO: /* 30 I/O instruction. */
4156 case VMX_EXIT_RDPMC: /* 15 Guest software attempted to execute RDPMC. */
4157 /* already handled above */
4158 AssertMsg( rc == VINF_PGM_CHANGE_MODE
4159 || rc == VINF_EM_RAW_INTERRUPT
4160 || rc == VERR_EM_INTERPRETER
4161 || rc == VINF_EM_RAW_EMULATE_INSTR
4162 || rc == VINF_PGM_SYNC_CR3
4163 || rc == VINF_IOM_HC_IOPORT_READ
4164 || rc == VINF_IOM_HC_IOPORT_WRITE
4165 || rc == VINF_EM_RAW_GUEST_TRAP
4166 || rc == VINF_TRPM_XCPT_DISPATCHED
4167 || rc == VINF_EM_RESCHEDULE_REM,
4168 ("rc = %d\n", VBOXSTRICTRC_VAL(rc)));
4169 break;
4170
4171 case VMX_EXIT_TPR: /* 43 TPR below threshold. Guest software executed MOV to CR8. */
4172 case VMX_EXIT_APIC_ACCESS: /* 44 APIC access. Guest software attempted to access memory at a physical address on the APIC-access page. */
4173 case VMX_EXIT_RDMSR: /* 31 RDMSR. Guest software attempted to execute RDMSR. */
4174 case VMX_EXIT_WRMSR: /* 32 WRMSR. Guest software attempted to execute WRMSR. */
4175 case VMX_EXIT_PAUSE: /* 40 Guest software attempted to execute PAUSE. */
4176 case VMX_EXIT_MONITOR: /* 39 Guest software attempted to execute MONITOR. */
4177 /* Note: If we decide to emulate them here, then we must sync the MSRs that could have been changed (sysenter, fs/gs base)!!! */
4178 rc = VERR_EM_INTERPRETER;
4179 break;
4180
4181 case VMX_EXIT_IRQ_WINDOW: /* 7 Interrupt window. */
4182 Assert(rc == VINF_EM_RAW_INTERRUPT);
4183 break;
4184
4185 case VMX_EXIT_ERR_INVALID_GUEST_STATE: /* 33 VM-entry failure due to invalid guest state. */
4186 {
4187#ifdef VBOX_STRICT
4188 RTCCUINTREG val2 = 0;
4189
4190 Log(("VMX_EXIT_ERR_INVALID_GUEST_STATE\n"));
4191
4192 VMXReadVMCS(VMX_VMCS64_GUEST_RIP, &val2);
4193 Log(("Old eip %RGv new %RGv\n", (RTGCPTR)pCtx->rip, (RTGCPTR)val2));
4194
4195 VMXReadVMCS(VMX_VMCS64_GUEST_CR0, &val2);
4196 Log(("VMX_VMCS_GUEST_CR0 %RX64\n", (uint64_t)val2));
4197
4198 VMXReadVMCS(VMX_VMCS64_GUEST_CR3, &val2);
4199 Log(("VMX_VMCS_GUEST_CR3 %RX64\n", (uint64_t)val2));
4200
4201 VMXReadVMCS(VMX_VMCS64_GUEST_CR4, &val2);
4202 Log(("VMX_VMCS_GUEST_CR4 %RX64\n", (uint64_t)val2));
4203
4204 VMXReadVMCS(VMX_VMCS_GUEST_RFLAGS, &val2);
4205 Log(("VMX_VMCS_GUEST_RFLAGS %08x\n", val2));
4206
4207 VMX_LOG_SELREG(CS, "CS", val2);
4208 VMX_LOG_SELREG(DS, "DS", val2);
4209 VMX_LOG_SELREG(ES, "ES", val2);
4210 VMX_LOG_SELREG(FS, "FS", val2);
4211 VMX_LOG_SELREG(GS, "GS", val2);
4212 VMX_LOG_SELREG(SS, "SS", val2);
4213 VMX_LOG_SELREG(TR, "TR", val2);
4214 VMX_LOG_SELREG(LDTR, "LDTR", val2);
4215
4216 VMXReadVMCS(VMX_VMCS64_GUEST_GDTR_BASE, &val2);
4217 Log(("VMX_VMCS_GUEST_GDTR_BASE %RX64\n", (uint64_t)val2));
4218 VMXReadVMCS(VMX_VMCS64_GUEST_IDTR_BASE, &val2);
4219 Log(("VMX_VMCS_GUEST_IDTR_BASE %RX64\n", (uint64_t)val2));
4220#endif /* VBOX_STRICT */
4221 rc = VERR_VMX_INVALID_GUEST_STATE;
4222 break;
4223 }
4224
4225 case VMX_EXIT_ERR_MSR_LOAD: /* 34 VM-entry failure due to MSR loading. */
4226 case VMX_EXIT_ERR_MACHINE_CHECK: /* 41 VM-entry failure due to machine-check. */
4227 default:
4228 rc = VERR_VMX_UNEXPECTED_EXIT_CODE;
4229 AssertMsgFailed(("Unexpected exit code %d\n", exitReason)); /* Can't happen. */
4230 break;
4231
4232 }
4233end:
4234
4235 /* We now going back to ring-3, so clear the action flag. */
4236 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
4237
4238 /* Signal changes for the recompiler. */
4239 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR | CPUM_CHANGED_LDTR | CPUM_CHANGED_GDTR | CPUM_CHANGED_IDTR | CPUM_CHANGED_TR | CPUM_CHANGED_HIDDEN_SEL_REGS);
4240
4241 /* If we executed vmlaunch/vmresume and an external irq was pending, then we don't have to do a full sync the next time. */
4242 if ( exitReason == VMX_EXIT_EXTERNAL_IRQ
4243 && !VMX_EXIT_INTERRUPTION_INFO_VALID(intInfo))
4244 {
4245 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatPendingHostIrq);
4246 /* On the next entry we'll only sync the host context. */
4247 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_HOST_CONTEXT;
4248 }
4249 else
4250 {
4251 /* On the next entry we'll sync everything. */
4252 /** @todo we can do better than this */
4253 /* Not in the VINF_PGM_CHANGE_MODE though! */
4254 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_ALL;
4255 }
4256
4257 /* translate into a less severe return code */
4258 if (rc == VERR_EM_INTERPRETER)
4259 rc = VINF_EM_RAW_EMULATE_INSTR;
4260 else
4261 /* Try to extract more information about what might have gone wrong here. */
4262 if (rc == VERR_VMX_INVALID_VMCS_PTR)
4263 {
4264 VMXGetActivateVMCS(&pVCpu->hwaccm.s.vmx.lasterror.u64VMCSPhys);
4265 pVCpu->hwaccm.s.vmx.lasterror.ulVMCSRevision = *(uint32_t *)pVCpu->hwaccm.s.vmx.pvVMCS;
4266 pVCpu->hwaccm.s.vmx.lasterror.idEnteredCpu = pVCpu->hwaccm.s.idEnteredCpu;
4267 pVCpu->hwaccm.s.vmx.lasterror.idCurrentCpu = RTMpCpuId();
4268 }
4269
4270 /* Just set the correct state here instead of trying to catch every goto above. */
4271 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED, VMCPUSTATE_STARTED_EXEC);
4272
4273#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
4274 /* Restore interrupts if we exitted after disabling them. */
4275 if (uOldEFlags != ~(RTCCUINTREG)0)
4276 ASMSetFlags(uOldEFlags);
4277#endif
4278
4279 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2, x);
4280 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit1, x);
4281 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatEntry, x);
4282 Log2(("X"));
4283 return VBOXSTRICTRC_TODO(rc);
4284}
4285
4286
4287/**
4288 * Enters the VT-x session
4289 *
4290 * @returns VBox status code.
4291 * @param pVM The VM to operate on.
4292 * @param pVCpu The VMCPU to operate on.
4293 * @param pCpu CPU info struct
4294 */
4295VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBLCPUINFO pCpu)
4296{
4297 Assert(pVM->hwaccm.s.vmx.fSupported);
4298 NOREF(pCpu);
4299
4300 unsigned cr4 = ASMGetCR4();
4301 if (!(cr4 & X86_CR4_VMXE))
4302 {
4303 AssertMsgFailed(("X86_CR4_VMXE should be set!\n"));
4304 return VERR_VMX_X86_CR4_VMXE_CLEARED;
4305 }
4306
4307 /* Activate the VM Control Structure. */
4308 int rc = VMXActivateVMCS(pVCpu->hwaccm.s.vmx.HCPhysVMCS);
4309 if (RT_FAILURE(rc))
4310 return rc;
4311
4312 pVCpu->hwaccm.s.fResumeVM = false;
4313 return VINF_SUCCESS;
4314}
4315
4316
4317/**
4318 * Leaves the VT-x session
4319 *
4320 * @returns VBox status code.
4321 * @param pVM The VM to operate on.
4322 * @param pVCpu The VMCPU to operate on.
4323 * @param pCtx CPU context
4324 */
4325VMMR0DECL(int) VMXR0Leave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4326{
4327 Assert(pVM->hwaccm.s.vmx.fSupported);
4328
4329#ifdef DEBUG
4330 if (CPUMIsHyperDebugStateActive(pVCpu))
4331 {
4332 CPUMR0LoadHostDebugState(pVM, pVCpu);
4333 Assert(pVCpu->hwaccm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT);
4334 }
4335 else
4336#endif
4337 /* Save the guest debug state if necessary. */
4338 if (CPUMIsGuestDebugStateActive(pVCpu))
4339 {
4340 CPUMR0SaveGuestDebugState(pVM, pVCpu, pCtx, true /* save DR6 */);
4341
4342 /* Enable drx move intercepts again. */
4343 pVCpu->hwaccm.s.vmx.proc_ctls |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
4344 int rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
4345 AssertRC(rc);
4346
4347 /* Resync the debug registers the next time. */
4348 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_DEBUG;
4349 }
4350 else
4351 Assert(pVCpu->hwaccm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT);
4352
4353 /* Clear VM Control Structure. Marking it inactive, clearing implementation specific data and writing back VMCS data to memory. */
4354 int rc = VMXClearVMCS(pVCpu->hwaccm.s.vmx.HCPhysVMCS);
4355 AssertRC(rc);
4356
4357 return VINF_SUCCESS;
4358}
4359
4360/**
4361 * Flush the TLB (EPT)
4362 *
4363 * @returns VBox status code.
4364 * @param pVM The VM to operate on.
4365 * @param pVCpu The VM CPU to operate on.
4366 * @param enmFlush Type of flush
4367 * @param GCPhys Physical address of the page to flush
4368 */
4369static void hmR0VmxFlushEPT(PVM pVM, PVMCPU pVCpu, VMX_FLUSH enmFlush, RTGCPHYS GCPhys)
4370{
4371 uint64_t descriptor[2];
4372
4373 LogFlow(("hmR0VmxFlushEPT %d %RGv\n", enmFlush, GCPhys));
4374 Assert(pVM->hwaccm.s.fNestedPaging);
4375 descriptor[0] = pVCpu->hwaccm.s.vmx.GCPhysEPTP;
4376 descriptor[1] = GCPhys;
4377 int rc = VMXR0InvEPT(enmFlush, &descriptor[0]);
4378 AssertRC(rc);
4379}
4380
4381#ifdef HWACCM_VTX_WITH_VPID
4382/**
4383 * Flush the TLB (EPT)
4384 *
4385 * @returns VBox status code.
4386 * @param pVM The VM to operate on.
4387 * @param pVCpu The VM CPU to operate on.
4388 * @param enmFlush Type of flush
4389 * @param GCPtr Virtual address of the page to flush
4390 */
4391static void hmR0VmxFlushVPID(PVM pVM, PVMCPU pVCpu, VMX_FLUSH enmFlush, RTGCPTR GCPtr)
4392{
4393#if HC_ARCH_BITS == 32
4394 /* If we get a flush in 64 bits guest mode, then force a full TLB flush. Invvpid probably takes only 32 bits addresses. (@todo) */
4395 if ( CPUMIsGuestInLongMode(pVCpu)
4396 && !VMX_IS_64BIT_HOST_MODE())
4397 {
4398 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
4399 }
4400 else
4401#endif
4402 {
4403 uint64_t descriptor[2];
4404
4405 Assert(pVM->hwaccm.s.vmx.fVPID);
4406 descriptor[0] = pVCpu->hwaccm.s.uCurrentASID;
4407 descriptor[1] = GCPtr;
4408 int rc = VMXR0InvVPID(enmFlush, &descriptor[0]); NOREF(rc);
4409 AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvVPID %x %x %RGv failed with %d\n", enmFlush, pVCpu->hwaccm.s.uCurrentASID, GCPtr, rc));
4410 }
4411}
4412#endif /* HWACCM_VTX_WITH_VPID */
4413
4414/**
4415 * Invalidates a guest page
4416 *
4417 * @returns VBox status code.
4418 * @param pVM The VM to operate on.
4419 * @param pVCpu The VM CPU to operate on.
4420 * @param GCVirt Page to invalidate
4421 */
4422VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
4423{
4424 bool fFlushPending = VMCPU_FF_ISSET(pVCpu, VMCPU_FF_TLB_FLUSH);
4425
4426 Log2(("VMXR0InvalidatePage %RGv\n", GCVirt));
4427
4428 /* Only relevant if we want to use VPID.
4429 * In the nested paging case we still see such calls, but
4430 * can safely ignore them. (e.g. after cr3 updates)
4431 */
4432#ifdef HWACCM_VTX_WITH_VPID
4433 /* Skip it if a TLB flush is already pending. */
4434 if ( !fFlushPending
4435 && pVM->hwaccm.s.vmx.fVPID)
4436 hmR0VmxFlushVPID(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushPage, GCVirt);
4437#endif /* HWACCM_VTX_WITH_VPID */
4438
4439 return VINF_SUCCESS;
4440}
4441
4442/**
4443 * Invalidates a guest page by physical address
4444 *
4445 * NOTE: Assumes the current instruction references this physical page though a virtual address!!
4446 *
4447 * @returns VBox status code.
4448 * @param pVM The VM to operate on.
4449 * @param pVCpu The VM CPU to operate on.
4450 * @param GCPhys Page to invalidate
4451 */
4452VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
4453{
4454 bool fFlushPending = VMCPU_FF_ISSET(pVCpu, VMCPU_FF_TLB_FLUSH);
4455
4456 Assert(pVM->hwaccm.s.fNestedPaging);
4457
4458 LogFlow(("VMXR0InvalidatePhysPage %RGp\n", GCPhys));
4459
4460 /* Skip it if a TLB flush is already pending. */
4461 if (!fFlushPending)
4462 hmR0VmxFlushEPT(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushPage, GCPhys);
4463
4464 return VINF_SUCCESS;
4465}
4466
4467/**
4468 * Report world switch error and dump some useful debug info
4469 *
4470 * @param pVM The VM to operate on.
4471 * @param pVCpu The VMCPU to operate on.
4472 * @param rc Return code
4473 * @param pCtx Current CPU context (not updated)
4474 */
4475static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, VBOXSTRICTRC rc, PCPUMCTX pCtx)
4476{
4477 NOREF(pVM);
4478
4479 switch (VBOXSTRICTRC_VAL(rc))
4480 {
4481 case VERR_VMX_INVALID_VMXON_PTR:
4482 AssertFailed();
4483 break;
4484
4485 case VERR_VMX_UNABLE_TO_START_VM:
4486 case VERR_VMX_UNABLE_TO_RESUME_VM:
4487 {
4488 int rc2;
4489 RTCCUINTREG exitReason, instrError;
4490
4491 rc2 = VMXReadVMCS(VMX_VMCS32_RO_EXIT_REASON, &exitReason);
4492 rc2 |= VMXReadVMCS(VMX_VMCS32_RO_VM_INSTR_ERROR, &instrError);
4493 AssertRC(rc2);
4494 if (rc2 == VINF_SUCCESS)
4495 {
4496 Log(("Unable to start/resume VM for reason: %x. Instruction error %x\n", (uint32_t)exitReason, (uint32_t)instrError));
4497 Log(("Current stack %08x\n", &rc2));
4498
4499 pVCpu->hwaccm.s.vmx.lasterror.ulInstrError = instrError;
4500 pVCpu->hwaccm.s.vmx.lasterror.ulExitReason = exitReason;
4501
4502#ifdef VBOX_STRICT
4503 RTGDTR gdtr;
4504 PCX86DESCHC pDesc;
4505 RTCCUINTREG val;
4506
4507 ASMGetGDTR(&gdtr);
4508
4509 VMXReadVMCS(VMX_VMCS64_GUEST_RIP, &val);
4510 Log(("Old eip %RGv new %RGv\n", (RTGCPTR)pCtx->rip, (RTGCPTR)val));
4511 VMXReadVMCS(VMX_VMCS_CTRL_PIN_EXEC_CONTROLS, &val);
4512 Log(("VMX_VMCS_CTRL_PIN_EXEC_CONTROLS %08x\n", val));
4513 VMXReadVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, &val);
4514 Log(("VMX_VMCS_CTRL_PROC_EXEC_CONTROLS %08x\n", val));
4515 VMXReadVMCS(VMX_VMCS_CTRL_ENTRY_CONTROLS, &val);
4516 Log(("VMX_VMCS_CTRL_ENTRY_CONTROLS %08x\n", val));
4517 VMXReadVMCS(VMX_VMCS_CTRL_EXIT_CONTROLS, &val);
4518 Log(("VMX_VMCS_CTRL_EXIT_CONTROLS %08x\n", val));
4519
4520 VMXReadVMCS(VMX_VMCS_HOST_CR0, &val);
4521 Log(("VMX_VMCS_HOST_CR0 %08x\n", val));
4522
4523 VMXReadVMCS(VMX_VMCS_HOST_CR3, &val);
4524 Log(("VMX_VMCS_HOST_CR3 %08x\n", val));
4525
4526 VMXReadVMCS(VMX_VMCS_HOST_CR4, &val);
4527 Log(("VMX_VMCS_HOST_CR4 %08x\n", val));
4528
4529 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_CS, &val);
4530 Log(("VMX_VMCS_HOST_FIELD_CS %08x\n", val));
4531
4532 VMXReadVMCS(VMX_VMCS_GUEST_RFLAGS, &val);
4533 Log(("VMX_VMCS_GUEST_RFLAGS %08x\n", val));
4534
4535 if (val < gdtr.cbGdt)
4536 {
4537 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
4538 HWACCMR0DumpDescriptor(pDesc, val, "CS: ");
4539 }
4540
4541 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_DS, &val);
4542 Log(("VMX_VMCS_HOST_FIELD_DS %08x\n", val));
4543 if (val < gdtr.cbGdt)
4544 {
4545 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
4546 HWACCMR0DumpDescriptor(pDesc, val, "DS: ");
4547 }
4548
4549 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_ES, &val);
4550 Log(("VMX_VMCS_HOST_FIELD_ES %08x\n", val));
4551 if (val < gdtr.cbGdt)
4552 {
4553 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
4554 HWACCMR0DumpDescriptor(pDesc, val, "ES: ");
4555 }
4556
4557 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_FS, &val);
4558 Log(("VMX_VMCS16_HOST_FIELD_FS %08x\n", val));
4559 if (val < gdtr.cbGdt)
4560 {
4561 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
4562 HWACCMR0DumpDescriptor(pDesc, val, "FS: ");
4563 }
4564
4565 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_GS, &val);
4566 Log(("VMX_VMCS16_HOST_FIELD_GS %08x\n", val));
4567 if (val < gdtr.cbGdt)
4568 {
4569 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
4570 HWACCMR0DumpDescriptor(pDesc, val, "GS: ");
4571 }
4572
4573 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_SS, &val);
4574 Log(("VMX_VMCS16_HOST_FIELD_SS %08x\n", val));
4575 if (val < gdtr.cbGdt)
4576 {
4577 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
4578 HWACCMR0DumpDescriptor(pDesc, val, "SS: ");
4579 }
4580
4581 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_TR, &val);
4582 Log(("VMX_VMCS16_HOST_FIELD_TR %08x\n", val));
4583 if (val < gdtr.cbGdt)
4584 {
4585 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
4586 HWACCMR0DumpDescriptor(pDesc, val, "TR: ");
4587 }
4588
4589 VMXReadVMCS(VMX_VMCS_HOST_TR_BASE, &val);
4590 Log(("VMX_VMCS_HOST_TR_BASE %RHv\n", val));
4591
4592 VMXReadVMCS(VMX_VMCS_HOST_GDTR_BASE, &val);
4593 Log(("VMX_VMCS_HOST_GDTR_BASE %RHv\n", val));
4594 VMXReadVMCS(VMX_VMCS_HOST_IDTR_BASE, &val);
4595 Log(("VMX_VMCS_HOST_IDTR_BASE %RHv\n", val));
4596
4597 VMXReadVMCS(VMX_VMCS32_HOST_SYSENTER_CS, &val);
4598 Log(("VMX_VMCS_HOST_SYSENTER_CS %08x\n", val));
4599
4600 VMXReadVMCS(VMX_VMCS_HOST_SYSENTER_EIP, &val);
4601 Log(("VMX_VMCS_HOST_SYSENTER_EIP %RHv\n", val));
4602
4603 VMXReadVMCS(VMX_VMCS_HOST_SYSENTER_ESP, &val);
4604 Log(("VMX_VMCS_HOST_SYSENTER_ESP %RHv\n", val));
4605
4606 VMXReadVMCS(VMX_VMCS_HOST_RSP, &val);
4607 Log(("VMX_VMCS_HOST_RSP %RHv\n", val));
4608 VMXReadVMCS(VMX_VMCS_HOST_RIP, &val);
4609 Log(("VMX_VMCS_HOST_RIP %RHv\n", val));
4610
4611# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4612 if (VMX_IS_64BIT_HOST_MODE())
4613 {
4614 Log(("MSR_K6_EFER = %RX64\n", ASMRdMsr(MSR_K6_EFER)));
4615 Log(("MSR_K6_STAR = %RX64\n", ASMRdMsr(MSR_K6_STAR)));
4616 Log(("MSR_K8_LSTAR = %RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
4617 Log(("MSR_K8_CSTAR = %RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
4618 Log(("MSR_K8_SF_MASK = %RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
4619 }
4620# endif
4621#endif /* VBOX_STRICT */
4622 }
4623 break;
4624 }
4625
4626 default:
4627 /* impossible */
4628 AssertMsgFailed(("%Rrc (%#x)\n", VBOXSTRICTRC_VAL(rc), VBOXSTRICTRC_VAL(rc)));
4629 break;
4630 }
4631}
4632
4633#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4634
4635/**
4636 * Prepares for and executes VMLAUNCH (64 bits guest mode)
4637 *
4638 * @returns VBox status code
4639 * @param fResume vmlauch/vmresume
4640 * @param pCtx Guest context
4641 * @param pCache VMCS cache
4642 * @param pVM The VM to operate on.
4643 * @param pVCpu The VMCPU to operate on.
4644 */
4645DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
4646{
4647 uint32_t aParam[6];
4648 PHMGLOBLCPUINFO pCpu;
4649 RTHCPHYS HCPhysCpuPage;
4650 int rc;
4651
4652 pCpu = HWACCMR0GetCurrentCpu();
4653 HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4654
4655#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4656 pCache->uPos = 1;
4657 pCache->interPD = PGMGetInterPaeCR3(pVM);
4658 pCache->pSwitcher = (uint64_t)pVM->hwaccm.s.pfnHost32ToGuest64R0;
4659#endif
4660
4661#ifdef DEBUG
4662 pCache->TestIn.HCPhysCpuPage= 0;
4663 pCache->TestIn.HCPhysVMCS = 0;
4664 pCache->TestIn.pCache = 0;
4665 pCache->TestOut.HCPhysVMCS = 0;
4666 pCache->TestOut.pCache = 0;
4667 pCache->TestOut.pCtx = 0;
4668 pCache->TestOut.eflags = 0;
4669#endif
4670
4671 aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
4672 aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
4673 aParam[2] = (uint32_t)(pVCpu->hwaccm.s.vmx.HCPhysVMCS); /* Param 2: VMCS physical address - Lo. */
4674 aParam[3] = (uint32_t)(pVCpu->hwaccm.s.vmx.HCPhysVMCS >> 32); /* Param 2: VMCS physical address - Hi. */
4675 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hwaccm.s.vmx.VMCSCache);
4676 aParam[5] = 0;
4677
4678#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4679 pCtx->dr[4] = pVM->hwaccm.s.vmx.pScratchPhys + 16 + 8;
4680 *(uint32_t *)(pVM->hwaccm.s.vmx.pScratch + 16 + 8) = 1;
4681#endif
4682 rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, pVM->hwaccm.s.pfnVMXGCStartVM64, 6, &aParam[0]);
4683
4684#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4685 Assert(*(uint32_t *)(pVM->hwaccm.s.vmx.pScratch + 16 + 8) == 5);
4686 Assert(pCtx->dr[4] == 10);
4687 *(uint32_t *)(pVM->hwaccm.s.vmx.pScratch + 16 + 8) = 0xff;
4688#endif
4689
4690#ifdef DEBUG
4691 AssertMsg(pCache->TestIn.HCPhysCpuPage== HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
4692 AssertMsg(pCache->TestIn.HCPhysVMCS == pVCpu->hwaccm.s.vmx.HCPhysVMCS, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVMCS, pVCpu->hwaccm.s.vmx.HCPhysVMCS));
4693 AssertMsg(pCache->TestIn.HCPhysVMCS == pCache->TestOut.HCPhysVMCS, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVMCS, pCache->TestOut.HCPhysVMCS));
4694 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache, pCache->TestOut.pCache));
4695 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hwaccm.s.vmx.VMCSCache), ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hwaccm.s.vmx.VMCSCache)));
4696 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx, pCache->TestOut.pCtx));
4697 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
4698#endif
4699 return rc;
4700}
4701
4702# ifdef VBOX_STRICT
4703
4704static bool hmR0VmxIsValidReadField(uint32_t idxField)
4705{
4706 switch(idxField)
4707 {
4708 case VMX_VMCS64_GUEST_RIP:
4709 case VMX_VMCS64_GUEST_RSP:
4710 case VMX_VMCS_GUEST_RFLAGS:
4711 case VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE:
4712 case VMX_VMCS_CTRL_CR0_READ_SHADOW:
4713 case VMX_VMCS64_GUEST_CR0:
4714 case VMX_VMCS_CTRL_CR4_READ_SHADOW:
4715 case VMX_VMCS64_GUEST_CR4:
4716 case VMX_VMCS64_GUEST_DR7:
4717 case VMX_VMCS32_GUEST_SYSENTER_CS:
4718 case VMX_VMCS64_GUEST_SYSENTER_EIP:
4719 case VMX_VMCS64_GUEST_SYSENTER_ESP:
4720 case VMX_VMCS32_GUEST_GDTR_LIMIT:
4721 case VMX_VMCS64_GUEST_GDTR_BASE:
4722 case VMX_VMCS32_GUEST_IDTR_LIMIT:
4723 case VMX_VMCS64_GUEST_IDTR_BASE:
4724 case VMX_VMCS16_GUEST_FIELD_CS:
4725 case VMX_VMCS32_GUEST_CS_LIMIT:
4726 case VMX_VMCS64_GUEST_CS_BASE:
4727 case VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS:
4728 case VMX_VMCS16_GUEST_FIELD_DS:
4729 case VMX_VMCS32_GUEST_DS_LIMIT:
4730 case VMX_VMCS64_GUEST_DS_BASE:
4731 case VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS:
4732 case VMX_VMCS16_GUEST_FIELD_ES:
4733 case VMX_VMCS32_GUEST_ES_LIMIT:
4734 case VMX_VMCS64_GUEST_ES_BASE:
4735 case VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS:
4736 case VMX_VMCS16_GUEST_FIELD_FS:
4737 case VMX_VMCS32_GUEST_FS_LIMIT:
4738 case VMX_VMCS64_GUEST_FS_BASE:
4739 case VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS:
4740 case VMX_VMCS16_GUEST_FIELD_GS:
4741 case VMX_VMCS32_GUEST_GS_LIMIT:
4742 case VMX_VMCS64_GUEST_GS_BASE:
4743 case VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS:
4744 case VMX_VMCS16_GUEST_FIELD_SS:
4745 case VMX_VMCS32_GUEST_SS_LIMIT:
4746 case VMX_VMCS64_GUEST_SS_BASE:
4747 case VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS:
4748 case VMX_VMCS16_GUEST_FIELD_LDTR:
4749 case VMX_VMCS32_GUEST_LDTR_LIMIT:
4750 case VMX_VMCS64_GUEST_LDTR_BASE:
4751 case VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS:
4752 case VMX_VMCS16_GUEST_FIELD_TR:
4753 case VMX_VMCS32_GUEST_TR_LIMIT:
4754 case VMX_VMCS64_GUEST_TR_BASE:
4755 case VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS:
4756 case VMX_VMCS32_RO_EXIT_REASON:
4757 case VMX_VMCS32_RO_VM_INSTR_ERROR:
4758 case VMX_VMCS32_RO_EXIT_INSTR_LENGTH:
4759 case VMX_VMCS32_RO_EXIT_INTERRUPTION_ERRCODE:
4760 case VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO:
4761 case VMX_VMCS32_RO_EXIT_INSTR_INFO:
4762 case VMX_VMCS_RO_EXIT_QUALIFICATION:
4763 case VMX_VMCS32_RO_IDT_INFO:
4764 case VMX_VMCS32_RO_IDT_ERRCODE:
4765 case VMX_VMCS64_GUEST_CR3:
4766 case VMX_VMCS_EXIT_PHYS_ADDR_FULL:
4767 return true;
4768 }
4769 return false;
4770}
4771
4772static bool hmR0VmxIsValidWriteField(uint32_t idxField)
4773{
4774 switch(idxField)
4775 {
4776 case VMX_VMCS64_GUEST_LDTR_BASE:
4777 case VMX_VMCS64_GUEST_TR_BASE:
4778 case VMX_VMCS64_GUEST_GDTR_BASE:
4779 case VMX_VMCS64_GUEST_IDTR_BASE:
4780 case VMX_VMCS64_GUEST_SYSENTER_EIP:
4781 case VMX_VMCS64_GUEST_SYSENTER_ESP:
4782 case VMX_VMCS64_GUEST_CR0:
4783 case VMX_VMCS64_GUEST_CR4:
4784 case VMX_VMCS64_GUEST_CR3:
4785 case VMX_VMCS64_GUEST_DR7:
4786 case VMX_VMCS64_GUEST_RIP:
4787 case VMX_VMCS64_GUEST_RSP:
4788 case VMX_VMCS64_GUEST_CS_BASE:
4789 case VMX_VMCS64_GUEST_DS_BASE:
4790 case VMX_VMCS64_GUEST_ES_BASE:
4791 case VMX_VMCS64_GUEST_FS_BASE:
4792 case VMX_VMCS64_GUEST_GS_BASE:
4793 case VMX_VMCS64_GUEST_SS_BASE:
4794 return true;
4795 }
4796 return false;
4797}
4798
4799# endif /* VBOX_STRICT */
4800
4801/**
4802 * Executes the specified handler in 64 mode
4803 *
4804 * @returns VBox status code.
4805 * @param pVM The VM to operate on.
4806 * @param pVCpu The VMCPU to operate on.
4807 * @param pCtx Guest context
4808 * @param pfnHandler RC handler
4809 * @param cbParam Number of parameters
4810 * @param paParam Array of 32 bits parameters
4811 */
4812VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, RTRCPTR pfnHandler, uint32_t cbParam, uint32_t *paParam)
4813{
4814 int rc, rc2;
4815 PHMGLOBLCPUINFO pCpu;
4816 RTHCPHYS HCPhysCpuPage;
4817 RTHCUINTREG uOldEFlags;
4818
4819 AssertReturn(pVM->hwaccm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
4820 Assert(pfnHandler);
4821 Assert(pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hwaccm.s.vmx.VMCSCache.Write.aField));
4822 Assert(pVCpu->hwaccm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hwaccm.s.vmx.VMCSCache.Read.aField));
4823
4824#ifdef VBOX_STRICT
4825 for (unsigned i=0;i<pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries;i++)
4826 Assert(hmR0VmxIsValidWriteField(pVCpu->hwaccm.s.vmx.VMCSCache.Write.aField[i]));
4827
4828 for (unsigned i=0;i<pVCpu->hwaccm.s.vmx.VMCSCache.Read.cValidEntries;i++)
4829 Assert(hmR0VmxIsValidReadField(pVCpu->hwaccm.s.vmx.VMCSCache.Read.aField[i]));
4830#endif
4831
4832 /* Disable interrupts. */
4833 uOldEFlags = ASMIntDisableFlags();
4834
4835 pCpu = HWACCMR0GetCurrentCpu();
4836 HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4837
4838 /* Clear VM Control Structure. Marking it inactive, clearing implementation specific data and writing back VMCS data to memory. */
4839 VMXClearVMCS(pVCpu->hwaccm.s.vmx.HCPhysVMCS);
4840
4841 /* Leave VMX Root Mode. */
4842 VMXDisable();
4843
4844 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4845
4846 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
4847 CPUMSetHyperEIP(pVCpu, pfnHandler);
4848 for (int i=(int)cbParam-1;i>=0;i--)
4849 CPUMPushHyper(pVCpu, paParam[i]);
4850
4851 STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatWorldSwitch3264, z);
4852 /* Call switcher. */
4853 rc = pVM->hwaccm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
4854 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatWorldSwitch3264, z);
4855
4856 /* Make sure the VMX instructions don't cause #UD faults. */
4857 ASMSetCR4(ASMGetCR4() | X86_CR4_VMXE);
4858
4859 /* Enter VMX Root Mode */
4860 rc2 = VMXEnable(HCPhysCpuPage);
4861 if (RT_FAILURE(rc2))
4862 {
4863 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4864 ASMSetFlags(uOldEFlags);
4865 return VERR_VMX_VMXON_FAILED;
4866 }
4867
4868 rc2 = VMXActivateVMCS(pVCpu->hwaccm.s.vmx.HCPhysVMCS);
4869 AssertRC(rc2);
4870 Assert(!(ASMGetFlags() & X86_EFL_IF));
4871 ASMSetFlags(uOldEFlags);
4872 return rc;
4873}
4874
4875#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
4876
4877
4878#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
4879/**
4880 * Executes VMWRITE
4881 *
4882 * @returns VBox status code
4883 * @param pVCpu The VMCPU to operate on.
4884 * @param idxField VMCS index
4885 * @param u64Val 16, 32 or 64 bits value
4886 */
4887VMMR0DECL(int) VMXWriteVMCS64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4888{
4889 int rc;
4890
4891 switch (idxField)
4892 {
4893 case VMX_VMCS_CTRL_TSC_OFFSET_FULL:
4894 case VMX_VMCS_CTRL_IO_BITMAP_A_FULL:
4895 case VMX_VMCS_CTRL_IO_BITMAP_B_FULL:
4896 case VMX_VMCS_CTRL_MSR_BITMAP_FULL:
4897 case VMX_VMCS_CTRL_VMEXIT_MSR_STORE_FULL:
4898 case VMX_VMCS_CTRL_VMEXIT_MSR_LOAD_FULL:
4899 case VMX_VMCS_CTRL_VMENTRY_MSR_LOAD_FULL:
4900 case VMX_VMCS_CTRL_VAPIC_PAGEADDR_FULL:
4901 case VMX_VMCS_CTRL_APIC_ACCESSADDR_FULL:
4902 case VMX_VMCS_GUEST_LINK_PTR_FULL:
4903 case VMX_VMCS_GUEST_PDPTR0_FULL:
4904 case VMX_VMCS_GUEST_PDPTR1_FULL:
4905 case VMX_VMCS_GUEST_PDPTR2_FULL:
4906 case VMX_VMCS_GUEST_PDPTR3_FULL:
4907 case VMX_VMCS_GUEST_DEBUGCTL_FULL:
4908 case VMX_VMCS_GUEST_EFER_FULL:
4909 case VMX_VMCS_CTRL_EPTP_FULL:
4910 /* These fields consist of two parts, which are both writable in 32 bits mode. */
4911 rc = VMXWriteVMCS32(idxField, u64Val);
4912 rc |= VMXWriteVMCS32(idxField + 1, (uint32_t)(u64Val >> 32ULL));
4913 AssertRC(rc);
4914 return rc;
4915
4916 case VMX_VMCS64_GUEST_LDTR_BASE:
4917 case VMX_VMCS64_GUEST_TR_BASE:
4918 case VMX_VMCS64_GUEST_GDTR_BASE:
4919 case VMX_VMCS64_GUEST_IDTR_BASE:
4920 case VMX_VMCS64_GUEST_SYSENTER_EIP:
4921 case VMX_VMCS64_GUEST_SYSENTER_ESP:
4922 case VMX_VMCS64_GUEST_CR0:
4923 case VMX_VMCS64_GUEST_CR4:
4924 case VMX_VMCS64_GUEST_CR3:
4925 case VMX_VMCS64_GUEST_DR7:
4926 case VMX_VMCS64_GUEST_RIP:
4927 case VMX_VMCS64_GUEST_RSP:
4928 case VMX_VMCS64_GUEST_CS_BASE:
4929 case VMX_VMCS64_GUEST_DS_BASE:
4930 case VMX_VMCS64_GUEST_ES_BASE:
4931 case VMX_VMCS64_GUEST_FS_BASE:
4932 case VMX_VMCS64_GUEST_GS_BASE:
4933 case VMX_VMCS64_GUEST_SS_BASE:
4934 /* Queue a 64 bits value as we can't set it in 32 bits host mode. */
4935 if (u64Val >> 32ULL)
4936 rc = VMXWriteCachedVMCSEx(pVCpu, idxField, u64Val);
4937 else
4938 rc = VMXWriteVMCS32(idxField, (uint32_t)u64Val);
4939
4940 return rc;
4941
4942 default:
4943 AssertMsgFailed(("Unexpected field %x\n", idxField));
4944 return VERR_INVALID_PARAMETER;
4945 }
4946}
4947
4948/**
4949 * Cache VMCS writes for performance reasons (Darwin) and for running 64 bits guests on 32 bits hosts.
4950 *
4951 * @param pVCpu The VMCPU to operate on.
4952 * @param idxField VMCS field
4953 * @param u64Val Value
4954 */
4955VMMR0DECL(int) VMXWriteCachedVMCSEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4956{
4957 PVMCSCACHE pCache = &pVCpu->hwaccm.s.vmx.VMCSCache;
4958
4959 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1, ("entries=%x\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
4960
4961 /* Make sure there are no duplicates. */
4962 for (unsigned i=0;i<pCache->Write.cValidEntries;i++)
4963 {
4964 if (pCache->Write.aField[i] == idxField)
4965 {
4966 pCache->Write.aFieldVal[i] = u64Val;
4967 return VINF_SUCCESS;
4968 }
4969 }
4970
4971 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
4972 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
4973 pCache->Write.cValidEntries++;
4974 return VINF_SUCCESS;
4975}
4976
4977#endif /* HC_ARCH_BITS == 32 && !VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0 */
4978
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette