HWVMXR0.cpp@ 41138

Last change on this file since 41138 was 41098, checked in by vboxsync, 13 years ago
HWVMXR0.cpp: If we should do something, mark it as @todo's and put the comment were the missing code should go.
Property svn:eol-style set to `native` Property svn:keywords set to `Id`
File size: 200.8 KB

Line
1	/* $Id: HWVMXR0.cpp 41098 2012-04-30 12:01:52Z vboxsync $ */
2	/** @file
3	* HM VMX (VT-x) - Host Context Ring 0.
4	*/
5
6	/*
7	* Copyright (C) 2006-2011 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*******************************************************************************
20	* Header Files *
21	*******************************************************************************/
22	#define LOG_GROUP LOG_GROUP_HWACCM
23	#include <iprt/asm-amd64-x86.h>
24	#include <VBox/vmm/hwaccm.h>
25	#include <VBox/vmm/pgm.h>
26	#include <VBox/vmm/dbgf.h>
27	#include <VBox/vmm/dbgftrace.h>
28	#include <VBox/vmm/selm.h>
29	#include <VBox/vmm/iom.h>
30	#ifdef VBOX_WITH_REM
31	# include <VBox/vmm/rem.h>
32	#endif
33	#include <VBox/vmm/tm.h>
34	#include "HWACCMInternal.h"
35	#include <VBox/vmm/vm.h>
36	#include <VBox/vmm/pdmapi.h>
37	#include <VBox/err.h>
38	#include <VBox/log.h>
39	#include <iprt/assert.h>
40	#include <iprt/param.h>
41	#include <iprt/string.h>
42	#include <iprt/time.h>
43	#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
44	# include <iprt/thread.h>
45	#endif
46	#include <iprt/x86.h>
47	#include "HWVMXR0.h"
48
49	#include "dtrace/VBoxVMM.h"
50
51
52	/*******************************************************************************
53	* Defined Constants And Macros *
54	*******************************************************************************/
55	#if defined(RT_ARCH_AMD64)
56	# define VMX_IS_64BIT_HOST_MODE() (true)
57	#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
58	# define VMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
59	#else
60	# define VMX_IS_64BIT_HOST_MODE() (false)
61	#endif
62
63	/*******************************************************************************
64	* Global Variables *
65	*******************************************************************************/
66	/* IO operation lookup arrays. */
67	static uint32_t const g_aIOSize[4] = {1, 2, 0, 4};
68	static uint32_t const g_aIOOpAnd[4] = {0xff, 0xffff, 0, 0xffffffff};
69
70	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
71	/** See HWACCMR0A.asm. */
72	extern "C" uint32_t g_fVMXIs64bitHost;
73	#endif
74
75	/*******************************************************************************
76	* Local Functions *
77	*******************************************************************************/
78	static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, VBOXSTRICTRC rc, PCPUMCTX pCtx);
79	static DECLCALLBACK(void) hmR0VmxSetupTLBEPT(PVM pVM, PVMCPU pVCpu);
80	static DECLCALLBACK(void) hmR0VmxSetupTLBVPID(PVM pVM, PVMCPU pVCpu);
81	static DECLCALLBACK(void) hmR0VmxSetupTLBDummy(PVM pVM, PVMCPU pVCpu);
82	static void hmR0VmxFlushEPT(PVM pVM, PVMCPU pVCpu, VMX_FLUSH enmFlush, RTGCPHYS GCPhys);
83	static void hmR0VmxFlushVPID(PVM pVM, PVMCPU pVCpu, VMX_FLUSH enmFlush, RTGCPTR GCPtr);
84	static void hmR0VmxUpdateExceptionBitmap(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
85	static void hmR0VmxSetMSRPermission(PVMCPU pVCpu, unsigned ulMSR, bool fRead, bool fWrite);
86
87
88	static void hmR0VmxCheckError(PVM pVM, PVMCPU pVCpu, int rc)
89	{
90	if (rc == VERR_VMX_GENERIC)
91	{
92	RTCCUINTREG instrError;
93
94	VMXReadVMCS(VMX_VMCS32_RO_VM_INSTR_ERROR, &instrError);
95	pVCpu->hwaccm.s.vmx.lasterror.ulInstrError = instrError;
96	}
97	pVM->hwaccm.s.lLastError = rc;
98	}
99
100	/**
101	* Sets up and activates VT-x on the current CPU
102	*
103	* @returns VBox status code.
104	* @param pCpu CPU info struct
105	* @param pVM The VM to operate on. (can be NULL after a resume!!)
106	* @param pvCpuPage Pointer to the global cpu page.
107	* @param HCPhysCpuPage Physical address of the global cpu page.
108	*/
109	VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBLCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
110	{
111	AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
112	AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
113	NOREF(pCpu);
114
115	if (pVM)
116	{
117	/* Set revision dword at the beginning of the VMXON structure. */
118	(uint32_t )pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hwaccm.s.vmx.msr.vmx_basic_info);
119	}
120
121	/** @todo we should unmap the two pages from the virtual address space in order to prevent accidental corruption.
122	* (which can have very bad consequences!!!)
123	*/
124
125	if (ASMGetCR4() & X86_CR4_VMXE)
126	return VERR_VMX_IN_VMX_ROOT_MODE;
127
128	/* Make sure the VMX instructions don't cause #UD faults. */
129	ASMSetCR4(ASMGetCR4() \| X86_CR4_VMXE);
130
131	/* Enter VMX Root Mode. */
132	int rc = VMXEnable(HCPhysCpuPage);
133	if (RT_FAILURE(rc))
134	{
135	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
136	return VERR_VMX_VMXON_FAILED;
137	}
138	return VINF_SUCCESS;
139	}
140
141	/**
142	* Deactivates VT-x on the current CPU
143	*
144	* @returns VBox status code.
145	* @param pCpu CPU info struct
146	* @param pvCpuPage Pointer to the global cpu page.
147	* @param HCPhysCpuPage Physical address of the global cpu page.
148	*/
149	VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBLCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
150	{
151	AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
152	AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
153	NOREF(pCpu);
154
155	/* If we're somehow not in VMX root mode, then we shouldn't dare leaving it. */
156	if (!(ASMGetCR4() & X86_CR4_VMXE))
157	return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
158
159	/* Leave VMX Root Mode. */
160	VMXDisable();
161
162	/* And clear the X86_CR4_VMXE bit. */
163	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
164	return VINF_SUCCESS;
165	}
166
167	/**
168	* Does Ring-0 per VM VT-x init.
169	*
170	* @returns VBox status code.
171	* @param pVM The VM to operate on.
172	*/
173	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
174	{
175	int rc;
176
177	#ifdef LOG_ENABLED
178	SUPR0Printf("VMXR0InitVM %x\n", pVM);
179	#endif
180
181	pVM->hwaccm.s.vmx.pMemObjAPIC = NIL_RTR0MEMOBJ;
182
183	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
184	{
185	/* Allocate one page for the APIC physical page (serves for filtering accesses). */
186	rc = RTR0MemObjAllocCont(&pVM->hwaccm.s.vmx.pMemObjAPIC, PAGE_SIZE, true /* executable R0 mapping */);
187	AssertRC(rc);
188	if (RT_FAILURE(rc))
189	return rc;
190
191	pVM->hwaccm.s.vmx.pAPIC = (uint8_t *)RTR0MemObjAddress(pVM->hwaccm.s.vmx.pMemObjAPIC);
192	pVM->hwaccm.s.vmx.pAPICPhys = RTR0MemObjGetPagePhysAddr(pVM->hwaccm.s.vmx.pMemObjAPIC, 0);
193	ASMMemZero32(pVM->hwaccm.s.vmx.pAPIC, PAGE_SIZE);
194	}
195	else
196	{
197	pVM->hwaccm.s.vmx.pMemObjAPIC = 0;
198	pVM->hwaccm.s.vmx.pAPIC = 0;
199	pVM->hwaccm.s.vmx.pAPICPhys = 0;
200	}
201
202	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
203	{
204	rc = RTR0MemObjAllocCont(&pVM->hwaccm.s.vmx.pMemObjScratch, PAGE_SIZE, true /* executable R0 mapping */);
205	AssertRC(rc);
206	if (RT_FAILURE(rc))
207	return rc;
208
209	pVM->hwaccm.s.vmx.pScratch = (uint8_t *)RTR0MemObjAddress(pVM->hwaccm.s.vmx.pMemObjScratch);
210	pVM->hwaccm.s.vmx.pScratchPhys = RTR0MemObjGetPagePhysAddr(pVM->hwaccm.s.vmx.pMemObjScratch, 0);
211
212	ASMMemZero32(pVM->hwaccm.s.vmx.pScratch, PAGE_SIZE);
213	strcpy((char *)pVM->hwaccm.s.vmx.pScratch, "SCRATCH Magic");
214	(uint64_t )(pVM->hwaccm.s.vmx.pScratch + 16) = UINT64_C(0xDEADBEEFDEADBEEF);
215	}
216	#endif
217
218	/* Allocate VMCBs for all guest CPUs. */
219	for (VMCPUID i = 0; i < pVM->cCpus; i++)
220	{
221	PVMCPU pVCpu = &pVM->aCpus[i];
222
223	pVCpu->hwaccm.s.vmx.hMemObjVMCS = NIL_RTR0MEMOBJ;
224
225	/* Allocate one page for the VM control structure (VMCS). */
226	rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.hMemObjVMCS, PAGE_SIZE, true /* executable R0 mapping */);
227	AssertRC(rc);
228	if (RT_FAILURE(rc))
229	return rc;
230
231	pVCpu->hwaccm.s.vmx.pvVMCS = RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.hMemObjVMCS);
232	pVCpu->hwaccm.s.vmx.HCPhysVMCS = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.hMemObjVMCS, 0);
233	ASMMemZeroPage(pVCpu->hwaccm.s.vmx.pvVMCS);
234
235	pVCpu->hwaccm.s.vmx.cr0_mask = 0;
236	pVCpu->hwaccm.s.vmx.cr4_mask = 0;
237
238	/* Allocate one page for the virtual APIC page for TPR caching. */
239	rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.hMemObjVAPIC, PAGE_SIZE, true /* executable R0 mapping */);
240	AssertRC(rc);
241	if (RT_FAILURE(rc))
242	return rc;
243
244	pVCpu->hwaccm.s.vmx.pbVAPIC = (uint8_t *)RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.hMemObjVAPIC);
245	pVCpu->hwaccm.s.vmx.HCPhysVAPIC = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.hMemObjVAPIC, 0);
246	ASMMemZeroPage(pVCpu->hwaccm.s.vmx.pbVAPIC);
247
248	/* Allocate the MSR bitmap if this feature is supported. */
249	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
250	{
251	rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap, PAGE_SIZE, true /* executable R0 mapping */);
252	AssertRC(rc);
253	if (RT_FAILURE(rc))
254	return rc;
255
256	pVCpu->hwaccm.s.vmx.pMSRBitmap = (uint8_t *)RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap);
257	pVCpu->hwaccm.s.vmx.pMSRBitmapPhys = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap, 0);
258	memset(pVCpu->hwaccm.s.vmx.pMSRBitmap, 0xff, PAGE_SIZE);
259	}
260
261	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
262	/* Allocate one page for the guest MSR load area (for preloading guest MSRs during the world switch). */
263	rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.pMemObjGuestMSR, PAGE_SIZE, true /* executable R0 mapping */);
264	AssertRC(rc);
265	if (RT_FAILURE(rc))
266	return rc;
267
268	pVCpu->hwaccm.s.vmx.pGuestMSR = (uint8_t *)RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.pMemObjGuestMSR);
269	pVCpu->hwaccm.s.vmx.pGuestMSRPhys = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.pMemObjGuestMSR, 0);
270	memset(pVCpu->hwaccm.s.vmx.pGuestMSR, 0, PAGE_SIZE);
271
272	/* Allocate one page for the host MSR load area (for restoring host MSRs after the world switch back). */
273	rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.pMemObjHostMSR, PAGE_SIZE, true /* executable R0 mapping */);
274	AssertRC(rc);
275	if (RT_FAILURE(rc))
276	return rc;
277
278	pVCpu->hwaccm.s.vmx.pHostMSR = (uint8_t *)RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.pMemObjHostMSR);
279	pVCpu->hwaccm.s.vmx.pHostMSRPhys = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.pMemObjHostMSR, 0);
280	memset(pVCpu->hwaccm.s.vmx.pHostMSR, 0, PAGE_SIZE);
281	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
282
283	/* Current guest paging mode. */
284	pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode = PGMMODE_REAL;
285
286	#ifdef LOG_ENABLED
287	SUPR0Printf("VMXR0InitVM %x VMCS=%x (%x)\n", pVM, pVCpu->hwaccm.s.vmx.pvVMCS, (uint32_t)pVCpu->hwaccm.s.vmx.HCPhysVMCS);
288	#endif
289	}
290
291	return VINF_SUCCESS;
292	}
293
294	/**
295	* Does Ring-0 per VM VT-x termination.
296	*
297	* @returns VBox status code.
298	* @param pVM The VM to operate on.
299	*/
300	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
301	{
302	for (VMCPUID i = 0; i < pVM->cCpus; i++)
303	{
304	PVMCPU pVCpu = &pVM->aCpus[i];
305
306	if (pVCpu->hwaccm.s.vmx.hMemObjVMCS != NIL_RTR0MEMOBJ)
307	{
308	RTR0MemObjFree(pVCpu->hwaccm.s.vmx.hMemObjVMCS, false);
309	pVCpu->hwaccm.s.vmx.hMemObjVMCS = NIL_RTR0MEMOBJ;
310	pVCpu->hwaccm.s.vmx.pvVMCS = 0;
311	pVCpu->hwaccm.s.vmx.HCPhysVMCS = 0;
312	}
313	if (pVCpu->hwaccm.s.vmx.hMemObjVAPIC != NIL_RTR0MEMOBJ)
314	{
315	RTR0MemObjFree(pVCpu->hwaccm.s.vmx.hMemObjVAPIC, false);
316	pVCpu->hwaccm.s.vmx.hMemObjVAPIC = NIL_RTR0MEMOBJ;
317	pVCpu->hwaccm.s.vmx.pbVAPIC = 0;
318	pVCpu->hwaccm.s.vmx.HCPhysVAPIC = 0;
319	}
320	if (pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap != NIL_RTR0MEMOBJ)
321	{
322	RTR0MemObjFree(pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap, false);
323	pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap = NIL_RTR0MEMOBJ;
324	pVCpu->hwaccm.s.vmx.pMSRBitmap = 0;
325	pVCpu->hwaccm.s.vmx.pMSRBitmapPhys = 0;
326	}
327	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
328	if (pVCpu->hwaccm.s.vmx.pMemObjHostMSR != NIL_RTR0MEMOBJ)
329	{
330	RTR0MemObjFree(pVCpu->hwaccm.s.vmx.pMemObjHostMSR, false);
331	pVCpu->hwaccm.s.vmx.pMemObjHostMSR = NIL_RTR0MEMOBJ;
332	pVCpu->hwaccm.s.vmx.pHostMSR = 0;
333	pVCpu->hwaccm.s.vmx.pHostMSRPhys = 0;
334	}
335	if (pVCpu->hwaccm.s.vmx.pMemObjGuestMSR != NIL_RTR0MEMOBJ)
336	{
337	RTR0MemObjFree(pVCpu->hwaccm.s.vmx.pMemObjGuestMSR, false);
338	pVCpu->hwaccm.s.vmx.pMemObjGuestMSR = NIL_RTR0MEMOBJ;
339	pVCpu->hwaccm.s.vmx.pGuestMSR = 0;
340	pVCpu->hwaccm.s.vmx.pGuestMSRPhys = 0;
341	}
342	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
343	}
344	if (pVM->hwaccm.s.vmx.pMemObjAPIC != NIL_RTR0MEMOBJ)
345	{
346	RTR0MemObjFree(pVM->hwaccm.s.vmx.pMemObjAPIC, false);
347	pVM->hwaccm.s.vmx.pMemObjAPIC = NIL_RTR0MEMOBJ;
348	pVM->hwaccm.s.vmx.pAPIC = 0;
349	pVM->hwaccm.s.vmx.pAPICPhys = 0;
350	}
351	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
352	if (pVM->hwaccm.s.vmx.pMemObjScratch != NIL_RTR0MEMOBJ)
353	{
354	ASMMemZero32(pVM->hwaccm.s.vmx.pScratch, PAGE_SIZE);
355	RTR0MemObjFree(pVM->hwaccm.s.vmx.pMemObjScratch, false);
356	pVM->hwaccm.s.vmx.pMemObjScratch = NIL_RTR0MEMOBJ;
357	pVM->hwaccm.s.vmx.pScratch = 0;
358	pVM->hwaccm.s.vmx.pScratchPhys = 0;
359	}
360	#endif
361	return VINF_SUCCESS;
362	}
363
364	/**
365	* Sets up VT-x for the specified VM
366	*
367	* @returns VBox status code.
368	* @param pVM The VM to operate on.
369	*/
370	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
371	{
372	int rc = VINF_SUCCESS;
373	uint32_t val;
374
375	AssertReturn(pVM, VERR_INVALID_PARAMETER);
376
377	for (VMCPUID i = 0; i < pVM->cCpus; i++)
378	{
379	PVMCPU pVCpu = &pVM->aCpus[i];
380
381	AssertPtr(pVCpu->hwaccm.s.vmx.pvVMCS);
382
383	/* Set revision dword at the beginning of the VMCS structure. */
384	(uint32_t )pVCpu->hwaccm.s.vmx.pvVMCS = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hwaccm.s.vmx.msr.vmx_basic_info);
385
386	/* Clear VM Control Structure. */
387	Log(("HCPhysVMCS = %RHp\n", pVCpu->hwaccm.s.vmx.HCPhysVMCS));
388	rc = VMXClearVMCS(pVCpu->hwaccm.s.vmx.HCPhysVMCS);
389	if (RT_FAILURE(rc))
390	goto vmx_end;
391
392	/* Activate the VM Control Structure. */
393	rc = VMXActivateVMCS(pVCpu->hwaccm.s.vmx.HCPhysVMCS);
394	if (RT_FAILURE(rc))
395	goto vmx_end;
396
397	/* VMX_VMCS_CTRL_PIN_EXEC_CONTROLS
398	* Set required bits to one and zero according to the MSR capabilities.
399	*/
400	val = pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.disallowed0;
401	/* External and non-maskable interrupts cause VM-exits. */
402	val \|= VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_EXT_INT_EXIT \| VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_NMI_EXIT;
403	/* enable the preemption timer. */
404	if (pVM->hwaccm.s.vmx.fUsePreemptTimer)
405	val \|= VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_PREEMPT_TIMER;
406	val &= pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.allowed1;
407
408	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PIN_EXEC_CONTROLS, val);
409	AssertRC(rc);
410
411	/* VMX_VMCS_CTRL_PROC_EXEC_CONTROLS
412	* Set required bits to one and zero according to the MSR capabilities.
413	*/
414	val = pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.disallowed0;
415	/* Program which event cause VM-exits and which features we want to use. */
416	val = val \| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_HLT_EXIT
417	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_TSC_OFFSET
418	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT
419	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_UNCOND_IO_EXIT
420	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDPMC_EXIT
421	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_EXIT
422	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MWAIT_EXIT; /* don't execute mwait or else we'll idle inside the guest (host thinks the cpu load is high) */
423
424	/* Without nested paging we should intercept invlpg and cr3 mov instructions. */
425	if (!pVM->hwaccm.s.fNestedPaging)
426	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT
427	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
428	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
429
430	/* Note: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MWAIT_EXIT might cause a vmlaunch failure with an invalid control fields error. (combined with some other exit reasons) */
431	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
432	{
433	/* CR8 reads from the APIC shadow page; writes cause an exit is they lower the TPR below the threshold */
434	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW;
435	Assert(pVM->hwaccm.s.vmx.pAPIC);
436	}
437	else
438	/* Exit on CR8 reads & writes in case the TPR shadow feature isn't present. */
439	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_STORE_EXIT \| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_LOAD_EXIT;
440
441	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
442	{
443	Assert(pVCpu->hwaccm.s.vmx.pMSRBitmapPhys);
444	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS;
445	}
446
447	/* We will use the secondary control if it's present. */
448	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
449
450	/* Mask away the bits that the CPU doesn't support */
451	/** @todo make sure they don't conflict with the above requirements. */
452	val &= pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1;
453	pVCpu->hwaccm.s.vmx.proc_ctls = val;
454
455	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, val);
456	AssertRC(rc);
457
458	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
459	{
460	/* VMX_VMCS_CTRL_PROC_EXEC_CONTROLS2
461	* Set required bits to one and zero according to the MSR capabilities.
462	*/
463	val = pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0;
464	val \|= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT;
465
466	#ifdef HWACCM_VTX_WITH_EPT
467	if (pVM->hwaccm.s.fNestedPaging)
468	val \|= VMX_VMCS_CTRL_PROC_EXEC2_EPT;
469	#endif /* HWACCM_VTX_WITH_EPT */
470	#ifdef HWACCM_VTX_WITH_VPID
471	else
472	if (pVM->hwaccm.s.vmx.fVPID)
473	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VPID;
474	#endif /* HWACCM_VTX_WITH_VPID */
475
476	if (pVM->hwaccm.s.fHasIoApic)
477	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC;
478
479	if (pVM->hwaccm.s.vmx.fUnrestrictedGuest)
480	val \|= VMX_VMCS_CTRL_PROC_EXEC2_REAL_MODE;
481
482	/* Mask away the bits that the CPU doesn't support */
483	/** @todo make sure they don't conflict with the above requirements. */
484	val &= pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1;
485	pVCpu->hwaccm.s.vmx.proc_ctls2 = val;
486	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS2, val);
487	AssertRC(rc);
488	}
489
490	/* VMX_VMCS_CTRL_CR3_TARGET_COUNT
491	* Set required bits to one and zero according to the MSR capabilities.
492	*/
493	rc = VMXWriteVMCS(VMX_VMCS_CTRL_CR3_TARGET_COUNT, 0);
494	AssertRC(rc);
495
496	/* Forward all exception except #NM & #PF to the guest.
497	* We always need to check pagefaults since our shadow page table can be out of sync.
498	* And we always lazily sync the FPU & XMM state.
499	*/
500
501	/** @todo Possible optimization:
502	* Keep the FPU and XMM state current in the EM thread. That way there's no need to
503	* lazily sync anything, but the downside is that we can't use the FPU stack or XMM
504	* registers ourselves of course.
505	*
506	* Note: only possible if the current state is actually ours (X86_CR0_TS flag)
507	*/
508
509	/* Don't filter page faults; all of them should cause a switch. */
510	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PAGEFAULT_ERROR_MASK, 0);
511	rc \|= VMXWriteVMCS(VMX_VMCS_CTRL_PAGEFAULT_ERROR_MATCH, 0);
512	AssertRC(rc);
513
514	/* Init TSC offset to zero. */
515	rc = VMXWriteVMCS64(VMX_VMCS_CTRL_TSC_OFFSET_FULL, 0);
516	AssertRC(rc);
517
518	rc = VMXWriteVMCS64(VMX_VMCS_CTRL_IO_BITMAP_A_FULL, 0);
519	AssertRC(rc);
520
521	rc = VMXWriteVMCS64(VMX_VMCS_CTRL_IO_BITMAP_B_FULL, 0);
522	AssertRC(rc);
523
524	/* Set the MSR bitmap address. */
525	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
526	{
527	Assert(pVCpu->hwaccm.s.vmx.pMSRBitmapPhys);
528
529	rc = VMXWriteVMCS64(VMX_VMCS_CTRL_MSR_BITMAP_FULL, pVCpu->hwaccm.s.vmx.pMSRBitmapPhys);
530	AssertRC(rc);
531
532	/* Allow the guest to directly modify these MSRs; they are restored and saved automatically. */
533	hmR0VmxSetMSRPermission(pVCpu, MSR_IA32_SYSENTER_CS, true, true);
534	hmR0VmxSetMSRPermission(pVCpu, MSR_IA32_SYSENTER_ESP, true, true);
535	hmR0VmxSetMSRPermission(pVCpu, MSR_IA32_SYSENTER_EIP, true, true);
536	hmR0VmxSetMSRPermission(pVCpu, MSR_K8_LSTAR, true, true);
537	hmR0VmxSetMSRPermission(pVCpu, MSR_K6_STAR, true, true);
538	hmR0VmxSetMSRPermission(pVCpu, MSR_K8_SF_MASK, true, true);
539	hmR0VmxSetMSRPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, true, true);
540	hmR0VmxSetMSRPermission(pVCpu, MSR_K8_GS_BASE, true, true);
541	hmR0VmxSetMSRPermission(pVCpu, MSR_K8_FS_BASE, true, true);
542	}
543
544	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
545	/* Set the guest & host MSR load/store physical addresses. */
546	Assert(pVCpu->hwaccm.s.vmx.pGuestMSRPhys);
547	rc = VMXWriteVMCS64(VMX_VMCS_CTRL_VMENTRY_MSR_LOAD_FULL, pVCpu->hwaccm.s.vmx.pGuestMSRPhys);
548	AssertRC(rc);
549	rc = VMXWriteVMCS64(VMX_VMCS_CTRL_VMEXIT_MSR_STORE_FULL, pVCpu->hwaccm.s.vmx.pGuestMSRPhys);
550	AssertRC(rc);
551
552	Assert(pVCpu->hwaccm.s.vmx.pHostMSRPhys);
553	rc = VMXWriteVMCS64(VMX_VMCS_CTRL_VMEXIT_MSR_LOAD_FULL, pVCpu->hwaccm.s.vmx.pHostMSRPhys);
554	AssertRC(rc);
555	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
556
557	rc = VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
558	AssertRC(rc);
559
560	rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_MSR_STORE_COUNT, 0);
561	AssertRC(rc);
562
563	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
564	{
565	Assert(pVM->hwaccm.s.vmx.pMemObjAPIC);
566	/* Optional */
567	rc = VMXWriteVMCS(VMX_VMCS_CTRL_TPR_THRESHOLD, 0);
568	rc \|= VMXWriteVMCS64(VMX_VMCS_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hwaccm.s.vmx.HCPhysVAPIC);
569
570	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
571	rc \|= VMXWriteVMCS64(VMX_VMCS_CTRL_APIC_ACCESSADDR_FULL, pVM->hwaccm.s.vmx.pAPICPhys);
572
573	AssertRC(rc);
574	}
575
576	/* Set link pointer to -1. Not currently used. */
577	rc = VMXWriteVMCS64(VMX_VMCS_GUEST_LINK_PTR_FULL, 0xFFFFFFFFFFFFFFFFULL);
578	AssertRC(rc);
579
580	/* Clear VM Control Structure. Marking it inactive, clearing implementation specific data and writing back VMCS data to memory. */
581	rc = VMXClearVMCS(pVCpu->hwaccm.s.vmx.HCPhysVMCS);
582	AssertRC(rc);
583
584	/* Configure the VMCS read cache. */
585	PVMCSCACHE pCache = &pVCpu->hwaccm.s.vmx.VMCSCache;
586
587	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_RIP);
588	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_RSP);
589	VMXSetupCachedReadVMCS(pCache, VMX_VMCS_GUEST_RFLAGS);
590	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE);
591	VMXSetupCachedReadVMCS(pCache, VMX_VMCS_CTRL_CR0_READ_SHADOW);
592	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_CR0);
593	VMXSetupCachedReadVMCS(pCache, VMX_VMCS_CTRL_CR4_READ_SHADOW);
594	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_CR4);
595	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_DR7);
596	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_SYSENTER_CS);
597	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_SYSENTER_EIP);
598	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_SYSENTER_ESP);
599	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_GDTR_LIMIT);
600	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_GDTR_BASE);
601	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_IDTR_LIMIT);
602	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_IDTR_BASE);
603
604	VMX_SETUP_SELREG(ES, pCache);
605	VMX_SETUP_SELREG(SS, pCache);
606	VMX_SETUP_SELREG(CS, pCache);
607	VMX_SETUP_SELREG(DS, pCache);
608	VMX_SETUP_SELREG(FS, pCache);
609	VMX_SETUP_SELREG(GS, pCache);
610	VMX_SETUP_SELREG(LDTR, pCache);
611	VMX_SETUP_SELREG(TR, pCache);
612
613	/* Status code VMCS reads. */
614	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_REASON);
615	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_VM_INSTR_ERROR);
616	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INSTR_LENGTH);
617	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INTERRUPTION_ERRCODE);
618	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO);
619	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INSTR_INFO);
620	VMXSetupCachedReadVMCS(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
621	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_IDT_INFO);
622	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_IDT_ERRCODE);
623
624	if (pVM->hwaccm.s.fNestedPaging)
625	{
626	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_CR3);
627	VMXSetupCachedReadVMCS(pCache, VMX_VMCS_EXIT_PHYS_ADDR_FULL);
628	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
629	}
630	else
631	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
632	} /* for each VMCPU */
633
634	/* Choose the right TLB setup function. */
635	if (pVM->hwaccm.s.fNestedPaging)
636	{
637	pVM->hwaccm.s.vmx.pfnSetupTaggedTLB = hmR0VmxSetupTLBEPT;
638
639	/* Default values for flushing. */
640	pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_ALL_CONTEXTS;
641	pVM->hwaccm.s.vmx.enmFlushContext = VMX_FLUSH_ALL_CONTEXTS;
642
643	/* If the capabilities specify we can do more, then make use of it. */
644	if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVEPT_CAPS_INDIV)
645	pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_PAGE;
646	else
647	if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVEPT_CAPS_CONTEXT)
648	pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_SINGLE_CONTEXT;
649
650	if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVEPT_CAPS_CONTEXT)
651	pVM->hwaccm.s.vmx.enmFlushContext = VMX_FLUSH_SINGLE_CONTEXT;
652	}
653	#ifdef HWACCM_VTX_WITH_VPID
654	else
655	if (pVM->hwaccm.s.vmx.fVPID)
656	{
657	pVM->hwaccm.s.vmx.pfnSetupTaggedTLB = hmR0VmxSetupTLBVPID;
658
659	/* Default values for flushing. */
660	pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_ALL_CONTEXTS;
661	pVM->hwaccm.s.vmx.enmFlushContext = VMX_FLUSH_ALL_CONTEXTS;
662
663	/* If the capabilities specify we can do more, then make use of it. */
664	if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_INDIV)
665	pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_PAGE;
666	else
667	if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_CONTEXT)
668	pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_SINGLE_CONTEXT;
669
670	if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_CONTEXT)
671	pVM->hwaccm.s.vmx.enmFlushContext = VMX_FLUSH_SINGLE_CONTEXT;
672	}
673	#endif /* HWACCM_VTX_WITH_VPID */
674	else
675	pVM->hwaccm.s.vmx.pfnSetupTaggedTLB = hmR0VmxSetupTLBDummy;
676
677	vmx_end:
678	hmR0VmxCheckError(pVM, &pVM->aCpus[0], rc);
679	return rc;
680	}
681
682	/**
683	* Sets the permission bits for the specified MSR
684	*
685	* @param pVCpu The VMCPU to operate on.
686	* @param ulMSR MSR value
687	* @param fRead Reading allowed/disallowed
688	* @param fWrite Writing allowed/disallowed
689	*/
690	static void hmR0VmxSetMSRPermission(PVMCPU pVCpu, unsigned ulMSR, bool fRead, bool fWrite)
691	{
692	unsigned ulBit;
693	uint8_t pMSRBitmap = (uint8_t )pVCpu->hwaccm.s.vmx.pMSRBitmap;
694
695	/* Layout:
696	* 0x000 - 0x3ff - Low MSR read bits
697	* 0x400 - 0x7ff - High MSR read bits
698	* 0x800 - 0xbff - Low MSR write bits
699	* 0xc00 - 0xfff - High MSR write bits
700	*/
701	if (ulMSR <= 0x00001FFF)
702	{
703	/* Pentium-compatible MSRs */
704	ulBit = ulMSR;
705	}
706	else
707	if ( ulMSR >= 0xC0000000
708	&& ulMSR <= 0xC0001FFF)
709	{
710	/* AMD Sixth Generation x86 Processor MSRs */
711	ulBit = (ulMSR - 0xC0000000);
712	pMSRBitmap += 0x400;
713	}
714	else
715	{
716	AssertFailed();
717	return;
718	}
719
720	Assert(ulBit <= 0x1fff);
721	if (fRead)
722	ASMBitClear(pMSRBitmap, ulBit);
723	else
724	ASMBitSet(pMSRBitmap, ulBit);
725
726	if (fWrite)
727	ASMBitClear(pMSRBitmap + 0x800, ulBit);
728	else
729	ASMBitSet(pMSRBitmap + 0x800, ulBit);
730	}
731
732
733	/**
734	* Injects an event (trap or external interrupt)
735	*
736	* @returns VBox status code. Note that it may return VINF_EM_RESET to
737	* indicate a triple fault when injecting X86_XCPT_DF.
738	*
739	* @param pVM The VM to operate on.
740	* @param pVCpu The VMCPU to operate on.
741	* @param pCtx CPU Context
742	* @param intInfo VMX interrupt info
743	* @param cbInstr Opcode length of faulting instruction
744	* @param errCode Error code (optional)
745	*/
746	static int hmR0VmxInjectEvent(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, uint32_t intInfo, uint32_t cbInstr, uint32_t errCode)
747	{
748	int rc;
749	uint32_t iGate = VMX_EXIT_INTERRUPTION_INFO_VECTOR(intInfo);
750
751	#ifdef VBOX_WITH_STATISTICS
752	STAM_COUNTER_INC(&pVCpu->hwaccm.s.paStatInjectedIrqsR0[iGate & MASK_INJECT_IRQ_STAT]);
753	#endif
754
755	#ifdef VBOX_STRICT
756	if (iGate == 0xE)
757	LogFlow(("hmR0VmxInjectEvent: Injecting interrupt %d at %RGv error code=%08x CR2=%RGv intInfo=%08x\n", iGate, (RTGCPTR)pCtx->rip, errCode, pCtx->cr2, intInfo));
758	else
759	if (iGate < 0x20)
760	LogFlow(("hmR0VmxInjectEvent: Injecting interrupt %d at %RGv error code=%08x\n", iGate, (RTGCPTR)pCtx->rip, errCode));
761	else
762	{
763	LogFlow(("INJ-EI: %x at %RGv\n", iGate, (RTGCPTR)pCtx->rip));
764	Assert( VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW
765	\|\| !VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
766	Assert( VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW
767	\|\| pCtx->eflags.u32 & X86_EFL_IF);
768	}
769	#endif
770
771	if ( CPUMIsGuestInRealModeEx(pCtx)
772	&& pVM->hwaccm.s.vmx.pRealModeTSS)
773	{
774	RTGCPHYS GCPhysHandler;
775	uint16_t offset, ip;
776	RTSEL sel;
777
778	/* Injecting events doesn't work right with real mode emulation.
779	* (#GP if we try to inject external hardware interrupts)
780	* Inject the interrupt or trap directly instead.
781	*
782	* ASSUMES no access handlers for the bits we read or write below (should be safe).
783	*/
784	Log(("Manual interrupt/trap '%x' inject (real mode)\n", iGate));
785
786	/* Check if the interrupt handler is present. */
787	if (iGate * 4 + 3 > pCtx->idtr.cbIdt)
788	{
789	Log(("IDT cbIdt violation\n"));
790	if (iGate != X86_XCPT_DF)
791	{
792	uint32_t intInfo2;
793
794	intInfo2 = (iGate == X86_XCPT_GP) ? (uint32_t)X86_XCPT_DF : iGate;
795	intInfo2 \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
796	intInfo2 \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
797	intInfo2 \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
798
799	return hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo2, 0, 0 /* no error code according to the Intel docs */);
800	}
801	Log(("Triple fault -> reset the VM!\n"));
802	return VINF_EM_RESET;
803	}
804	if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW
805	\|\| iGate == 3 /* Both #BP and #OF point to the instruction after. */
806	\|\| iGate == 4)
807	{
808	ip = pCtx->ip + cbInstr;
809	}
810	else
811	ip = pCtx->ip;
812
813	/* Read the selector:offset pair of the interrupt handler. */
814	GCPhysHandler = (RTGCPHYS)pCtx->idtr.pIdt + iGate * 4;
815	rc = PGMPhysSimpleReadGCPhys(pVM, &offset, GCPhysHandler, sizeof(offset)); AssertRC(rc);
816	rc = PGMPhysSimpleReadGCPhys(pVM, &sel, GCPhysHandler + 2, sizeof(sel)); AssertRC(rc);
817
818	LogFlow(("IDT handler %04X:%04X\n", sel, offset));
819
820	/* Construct the stack frame. */
821	/** @todo should check stack limit. */
822	pCtx->sp -= 2;
823	LogFlow(("ss:sp %04X:%04X eflags=%x\n", pCtx->ss, pCtx->sp, pCtx->eflags.u));
824	rc = PGMPhysSimpleWriteGCPhys(pVM, pCtx->ssHid.u64Base + pCtx->sp, &pCtx->eflags, sizeof(uint16_t)); AssertRC(rc);
825	pCtx->sp -= 2;
826	LogFlow(("ss:sp %04X:%04X cs=%x\n", pCtx->ss, pCtx->sp, pCtx->cs));
827	rc = PGMPhysSimpleWriteGCPhys(pVM, pCtx->ssHid.u64Base + pCtx->sp, &pCtx->cs, sizeof(uint16_t)); AssertRC(rc);
828	pCtx->sp -= 2;
829	LogFlow(("ss:sp %04X:%04X ip=%x\n", pCtx->ss, pCtx->sp, ip));
830	rc = PGMPhysSimpleWriteGCPhys(pVM, pCtx->ssHid.u64Base + pCtx->sp, &ip, sizeof(ip)); AssertRC(rc);
831
832	/* Update the CPU state for executing the handler. */
833	pCtx->rip = offset;
834	pCtx->cs = sel;
835	pCtx->csHid.u64Base = sel << 4;
836	pCtx->eflags.u &= ~(X86_EFL_IF\|X86_EFL_TF\|X86_EFL_RF\|X86_EFL_AC);
837
838	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_SEGMENT_REGS;
839	return VINF_SUCCESS;
840	}
841
842	/* Set event injection state. */
843	rc = VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_IRQ_INFO, intInfo \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT));
844
845	rc \|= VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
846	rc \|= VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_EXCEPTION_ERRCODE, errCode);
847
848	AssertRC(rc);
849	return rc;
850	}
851
852
853	/**
854	* Checks for pending guest interrupts and injects them
855	*
856	* @returns VBox status code.
857	* @param pVM The VM to operate on.
858	* @param pVCpu The VMCPU to operate on.
859	* @param pCtx CPU Context
860	*/
861	static int hmR0VmxCheckPendingInterrupt(PVM pVM, PVMCPU pVCpu, CPUMCTX *pCtx)
862	{
863	int rc;
864
865	/* Dispatch any pending interrupts. (injected before, but a VM exit occurred prematurely) */
866	if (pVCpu->hwaccm.s.Event.fPending)
867	{
868	Log(("CPU%d: Reinjecting event %RX64 %08x at %RGv cr2=%RX64\n", pVCpu->idCpu, pVCpu->hwaccm.s.Event.intInfo, pVCpu->hwaccm.s.Event.errCode, (RTGCPTR)pCtx->rip, pCtx->cr2));
869	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatIntReinject);
870	rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, pVCpu->hwaccm.s.Event.intInfo, 0, pVCpu->hwaccm.s.Event.errCode);
871	AssertRC(rc);
872
873	pVCpu->hwaccm.s.Event.fPending = false;
874	return VINF_SUCCESS;
875	}
876
877	/* If an active trap is already pending, then we must forward it first! */
878	if (!TRPMHasTrap(pVCpu))
879	{
880	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI))
881	{
882	RTGCUINTPTR intInfo;
883
884	Log(("CPU%d: injecting #NMI\n", pVCpu->idCpu));
885
886	intInfo = X86_XCPT_NMI;
887	intInfo \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
888	intInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
889
890	rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo, 0, 0);
891	AssertRC(rc);
892
893	return VINF_SUCCESS;
894	}
895
896	/* @todo SMI interrupts. */
897
898	/* When external interrupts are pending, we should exit the VM when IF is set. */
899	if (VMCPU_FF_ISPENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC\|VMCPU_FF_INTERRUPT_PIC)))
900	{
901	if (!(pCtx->eflags.u32 & X86_EFL_IF))
902	{
903	if (!(pVCpu->hwaccm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_IRQ_WINDOW_EXIT))
904	{
905	LogFlow(("Enable irq window exit!\n"));
906	pVCpu->hwaccm.s.vmx.proc_ctls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_IRQ_WINDOW_EXIT;
907	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
908	AssertRC(rc);
909	}
910	/* else nothing to do but wait */
911	}
912	else
913	if (!VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
914	{
915	uint8_t u8Interrupt;
916
917	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
918	Log(("CPU%d: Dispatch interrupt: u8Interrupt=%x (%d) rc=%Rrc cs:rip=%04X:%RGv\n", pVCpu->idCpu, u8Interrupt, u8Interrupt, rc, pCtx->cs, (RTGCPTR)pCtx->rip));
919	if (RT_SUCCESS(rc))
920	{
921	rc = TRPMAssertTrap(pVCpu, u8Interrupt, TRPM_HARDWARE_INT);
922	AssertRC(rc);
923	}
924	else
925	{
926	/* Can only happen in rare cases where a pending interrupt is cleared behind our back */
927	Assert(!VMCPU_FF_ISPENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC\|VMCPU_FF_INTERRUPT_PIC)));
928	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatSwitchGuestIrq);
929	/* Just continue */
930	}
931	}
932	else
933	Log(("Pending interrupt blocked at %RGv by VM_FF_INHIBIT_INTERRUPTS!!\n", (RTGCPTR)pCtx->rip));
934	}
935	}
936
937	#ifdef VBOX_STRICT
938	if (TRPMHasTrap(pVCpu))
939	{
940	uint8_t u8Vector;
941	rc = TRPMQueryTrapAll(pVCpu, &u8Vector, 0, 0, 0);
942	AssertRC(rc);
943	}
944	#endif
945
946	if ( (pCtx->eflags.u32 & X86_EFL_IF)
947	&& (!VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
948	&& TRPMHasTrap(pVCpu)
949	)
950	{
951	uint8_t u8Vector;
952	TRPMEVENT enmType;
953	RTGCUINTPTR intInfo;
954	RTGCUINT errCode;
955
956	/* If a new event is pending, then dispatch it now. */
957	rc = TRPMQueryTrapAll(pVCpu, &u8Vector, &enmType, &errCode, 0);
958	AssertRC(rc);
959	Assert(pCtx->eflags.Bits.u1IF == 1 \|\| enmType == TRPM_TRAP);
960	Assert(enmType != TRPM_SOFTWARE_INT);
961
962	/* Clear the pending trap. */
963	rc = TRPMResetTrap(pVCpu);
964	AssertRC(rc);
965
966	intInfo = u8Vector;
967	intInfo \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
968
969	if (enmType == TRPM_TRAP)
970	{
971	switch (u8Vector) {
972	case X86_XCPT_DF:
973	case X86_XCPT_TS:
974	case X86_XCPT_NP:
975	case X86_XCPT_SS:
976	case X86_XCPT_GP:
977	case X86_XCPT_PF:
978	case X86_XCPT_AC:
979	/* Valid error codes. */
980	intInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
981	break;
982	default:
983	break;
984	}
985	if (u8Vector == X86_XCPT_BP \|\| u8Vector == X86_XCPT_OF)
986	intInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
987	else
988	intInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
989	}
990	else
991	intInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
992
993	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatIntInject);
994	rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo, 0, errCode);
995	AssertRC(rc);
996	} /* if (interrupts can be dispatched) */
997
998	return VINF_SUCCESS;
999	}
1000
1001	/**
1002	* Save the host state
1003	*
1004	* @returns VBox status code.
1005	* @param pVM The VM to operate on.
1006	* @param pVCpu The VMCPU to operate on.
1007	*/
1008	VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
1009	{
1010	int rc = VINF_SUCCESS;
1011	NOREF(pVM);
1012
1013	/*
1014	* Host CPU Context
1015	*/
1016	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_HOST_CONTEXT)
1017	{
1018	RTIDTR idtr;
1019	RTGDTR gdtr;
1020	RTSEL SelTR;
1021	PCX86DESCHC pDesc;
1022	uintptr_t trBase;
1023	RTSEL cs;
1024	RTSEL ss;
1025	uint64_t cr3;
1026
1027	/* Control registers */
1028	rc = VMXWriteVMCS(VMX_VMCS_HOST_CR0, ASMGetCR0());
1029	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1030	if (VMX_IS_64BIT_HOST_MODE())
1031	{
1032	cr3 = hwaccmR0Get64bitCR3();
1033	rc \|= VMXWriteVMCS64(VMX_VMCS_HOST_CR3, cr3);
1034	}
1035	else
1036	#endif
1037	{
1038	cr3 = ASMGetCR3();
1039	rc \|= VMXWriteVMCS(VMX_VMCS_HOST_CR3, cr3);
1040	}
1041	rc \|= VMXWriteVMCS(VMX_VMCS_HOST_CR4, ASMGetCR4());
1042	AssertRC(rc);
1043	Log2(("VMX_VMCS_HOST_CR0 %08x\n", ASMGetCR0()));
1044	Log2(("VMX_VMCS_HOST_CR3 %08RX64\n", cr3));
1045	Log2(("VMX_VMCS_HOST_CR4 %08x\n", ASMGetCR4()));
1046
1047	/* Selector registers. */
1048	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1049	if (VMX_IS_64BIT_HOST_MODE())
1050	{
1051	cs = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
1052	ss = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
1053	}
1054	else
1055	{
1056	/* sysenter loads LDT cs & ss, VMX doesn't like this. Load the GDT ones (safe). */
1057	cs = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
1058	ss = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
1059	}
1060	#else
1061	cs = ASMGetCS();
1062	ss = ASMGetSS();
1063	#endif
1064	Assert(!(cs & X86_SEL_LDT)); Assert((cs & X86_SEL_RPL) == 0);
1065	Assert(!(ss & X86_SEL_LDT)); Assert((ss & X86_SEL_RPL) == 0);
1066	rc = VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_CS, cs);
1067	/* Note: VMX is (again) very picky about the RPL of the selectors here; we'll restore them manually. */
1068	rc \|= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_DS, 0);
1069	rc \|= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_ES, 0);
1070	#if HC_ARCH_BITS == 32
1071	if (!VMX_IS_64BIT_HOST_MODE())
1072	{
1073	rc \|= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_FS, 0);
1074	rc \|= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_GS, 0);
1075	}
1076	#endif
1077	rc \|= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_SS, ss);
1078	SelTR = ASMGetTR();
1079	rc \|= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_TR, SelTR);
1080	AssertRC(rc);
1081	Log2(("VMX_VMCS_HOST_FIELD_CS %08x (%08x)\n", cs, ASMGetSS()));
1082	Log2(("VMX_VMCS_HOST_FIELD_DS 00000000 (%08x)\n", ASMGetDS()));
1083	Log2(("VMX_VMCS_HOST_FIELD_ES 00000000 (%08x)\n", ASMGetES()));
1084	Log2(("VMX_VMCS_HOST_FIELD_FS 00000000 (%08x)\n", ASMGetFS()));
1085	Log2(("VMX_VMCS_HOST_FIELD_GS 00000000 (%08x)\n", ASMGetGS()));
1086	Log2(("VMX_VMCS_HOST_FIELD_SS %08x (%08x)\n", ss, ASMGetSS()));
1087	Log2(("VMX_VMCS_HOST_FIELD_TR %08x\n", ASMGetTR()));
1088
1089	/* GDTR & IDTR */
1090	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1091	if (VMX_IS_64BIT_HOST_MODE())
1092	{
1093	X86XDTR64 gdtr64, idtr64;
1094	hwaccmR0Get64bitGDTRandIDTR(&gdtr64, &idtr64);
1095	rc = VMXWriteVMCS64(VMX_VMCS_HOST_GDTR_BASE, gdtr64.uAddr);
1096	rc \|= VMXWriteVMCS64(VMX_VMCS_HOST_IDTR_BASE, gdtr64.uAddr);
1097	AssertRC(rc);
1098	Log2(("VMX_VMCS_HOST_GDTR_BASE %RX64\n", gdtr64.uAddr));
1099	Log2(("VMX_VMCS_HOST_IDTR_BASE %RX64\n", idtr64.uAddr));
1100	gdtr.cbGdt = gdtr64.cb;
1101	gdtr.pGdt = (uintptr_t)gdtr64.uAddr;
1102	}
1103	else
1104	#endif
1105	{
1106	ASMGetGDTR(&gdtr);
1107	rc = VMXWriteVMCS(VMX_VMCS_HOST_GDTR_BASE, gdtr.pGdt);
1108	ASMGetIDTR(&idtr);
1109	rc \|= VMXWriteVMCS(VMX_VMCS_HOST_IDTR_BASE, idtr.pIdt);
1110	AssertRC(rc);
1111	Log2(("VMX_VMCS_HOST_GDTR_BASE %RHv\n", gdtr.pGdt));
1112	Log2(("VMX_VMCS_HOST_IDTR_BASE %RHv\n", idtr.pIdt));
1113	}
1114
1115	/* Save the base address of the TR selector. */
1116	if (SelTR > gdtr.cbGdt)
1117	{
1118	AssertMsgFailed(("Invalid TR selector %x. GDTR.cbGdt=%x\n", SelTR, gdtr.cbGdt));
1119	return VERR_VMX_INVALID_HOST_STATE;
1120	}
1121
1122	pDesc = (PCX86DESCHC)(gdtr.pGdt + (SelTR & X86_SEL_MASK));
1123	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1124	if (VMX_IS_64BIT_HOST_MODE())
1125	{
1126	uint64_t trBase64 = X86DESC64_BASE(*(PX86DESC64)pDesc);
1127	rc = VMXWriteVMCS64(VMX_VMCS_HOST_TR_BASE, trBase64);
1128	Log2(("VMX_VMCS_HOST_TR_BASE %RX64\n", trBase64));
1129	AssertRC(rc);
1130	}
1131	else
1132	#endif
1133	{
1134	#if HC_ARCH_BITS == 64
1135	trBase = X86DESC64_BASE(*pDesc);
1136	#else
1137	trBase = X86DESC_BASE(*pDesc);
1138	#endif
1139	rc = VMXWriteVMCS(VMX_VMCS_HOST_TR_BASE, trBase);
1140	AssertRC(rc);
1141	Log2(("VMX_VMCS_HOST_TR_BASE %RHv\n", trBase));
1142	}
1143
1144	/* FS and GS base. */
1145	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1146	if (VMX_IS_64BIT_HOST_MODE())
1147	{
1148	Log2(("MSR_K8_FS_BASE = %RX64\n", ASMRdMsr(MSR_K8_FS_BASE)));
1149	Log2(("MSR_K8_GS_BASE = %RX64\n", ASMRdMsr(MSR_K8_GS_BASE)));
1150	rc = VMXWriteVMCS64(VMX_VMCS_HOST_FS_BASE, ASMRdMsr(MSR_K8_FS_BASE));
1151	rc \|= VMXWriteVMCS64(VMX_VMCS_HOST_GS_BASE, ASMRdMsr(MSR_K8_GS_BASE));
1152	}
1153	#endif
1154	AssertRC(rc);
1155
1156	/* Sysenter MSRs. */
1157	/** @todo expensive!! */
1158	rc = VMXWriteVMCS(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
1159	Log2(("VMX_VMCS_HOST_SYSENTER_CS %08x\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_CS)));
1160	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1161	if (VMX_IS_64BIT_HOST_MODE())
1162	{
1163	Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_EIP)));
1164	Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_ESP)));
1165	rc \|= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
1166	rc \|= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
1167	}
1168	else
1169	{
1170	rc \|= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
1171	rc \|= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
1172	Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP)));
1173	Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP)));
1174	}
1175	#elif HC_ARCH_BITS == 32
1176	rc \|= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
1177	rc \|= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
1178	Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP)));
1179	Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP)));
1180	#else
1181	Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_EIP)));
1182	Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_ESP)));
1183	rc \|= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
1184	rc \|= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
1185	#endif
1186	AssertRC(rc);
1187
1188	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
1189	/* Store all host MSRs in the VM-Exit load area, so they will be reloaded after the world switch back to the host. */
1190	PVMXMSR pMsr = (PVMXMSR)pVCpu->hwaccm.s.vmx.pHostMSR;
1191	unsigned idxMsr = 0;
1192
1193	/* EFER MSR present? */
1194	if (ASMCpuId_EDX(0x80000001) & (X86_CPUID_AMD_FEATURE_EDX_NX\|X86_CPUID_AMD_FEATURE_EDX_LONG_MODE))
1195	{
1196	if (ASMCpuId_EDX(0x80000001) & X86_CPUID_AMD_FEATURE_EDX_SEP)
1197	{
1198	pMsr->u32IndexMSR = MSR_K6_STAR;
1199	pMsr->u32Reserved = 0;
1200	pMsr->u64Value = ASMRdMsr(MSR_K6_STAR); /* legacy syscall eip, cs & ss */
1201	pMsr++; idxMsr++;
1202	}
1203
1204	pMsr->u32IndexMSR = MSR_K6_EFER;
1205	pMsr->u32Reserved = 0;
1206	# if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1207	if (CPUMIsGuestInLongMode(pVCpu))
1208	{
1209	/* Must match the efer value in our 64 bits switcher. */
1210	pMsr->u64Value = ASMRdMsr(MSR_K6_EFER) \| MSR_K6_EFER_LME \| MSR_K6_EFER_SCE \| MSR_K6_EFER_NXE;
1211	}
1212	else
1213	# endif
1214	pMsr->u64Value = ASMRdMsr(MSR_K6_EFER);
1215	pMsr++; idxMsr++;
1216	}
1217
1218	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1219	if (VMX_IS_64BIT_HOST_MODE())
1220	{
1221	pMsr->u32IndexMSR = MSR_K8_LSTAR;
1222	pMsr->u32Reserved = 0;
1223	pMsr->u64Value = ASMRdMsr(MSR_K8_LSTAR); /* 64 bits mode syscall rip */
1224	pMsr++; idxMsr++;
1225	pMsr->u32IndexMSR = MSR_K8_SF_MASK;
1226	pMsr->u32Reserved = 0;
1227	pMsr->u64Value = ASMRdMsr(MSR_K8_SF_MASK); /* syscall flag mask */
1228	pMsr++; idxMsr++;
1229	pMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
1230	pMsr->u32Reserved = 0;
1231	pMsr->u64Value = ASMRdMsr(MSR_K8_KERNEL_GS_BASE); /* swapgs exchange value */
1232	pMsr++; idxMsr++;
1233	}
1234	# endif
1235	rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_MSR_LOAD_COUNT, idxMsr);
1236	AssertRC(rc);
1237	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
1238
1239	pVCpu->hwaccm.s.fContextUseFlags &= ~HWACCM_CHANGED_HOST_CONTEXT;
1240	}
1241	return rc;
1242	}
1243
1244	/**
1245	* Loads the 4 PDPEs into the guest state when nested paging is used and the
1246	* guest operates in PAE mode.
1247	*
1248	* @returns VINF_SUCCESS or fatal error.
1249	* @param pVCpu The VMCPU to operate on.
1250	* @param pCtx Guest context
1251	*/
1252	static int hmR0VmxLoadPaePdpes(PVMCPU pVCpu, PCPUMCTX pCtx)
1253	{
1254	if (CPUMIsGuestInPAEModeEx(pCtx))
1255	{
1256	X86PDPE aPdpes[4];
1257	int rc = PGMGstGetPaePdpes(pVCpu, &aPdpes[0]);
1258	AssertRCReturn(rc, rc);
1259
1260	rc = VMXWriteVMCS64(VMX_VMCS_GUEST_PDPTR0_FULL, aPdpes[0].u); AssertRCReturn(rc, rc);
1261	rc = VMXWriteVMCS64(VMX_VMCS_GUEST_PDPTR1_FULL, aPdpes[1].u); AssertRCReturn(rc, rc);
1262	rc = VMXWriteVMCS64(VMX_VMCS_GUEST_PDPTR2_FULL, aPdpes[2].u); AssertRCReturn(rc, rc);
1263	rc = VMXWriteVMCS64(VMX_VMCS_GUEST_PDPTR3_FULL, aPdpes[3].u); AssertRCReturn(rc, rc);
1264	}
1265	return VINF_SUCCESS;
1266	}
1267
1268	/**
1269	* Saves the 4 PDPEs into the guest state when nested paging is used and the
1270	* guest operates in PAE mode.
1271	*
1272	* @returns VINF_SUCCESS or fatal error.
1273	* @param pVCpu The VMCPU to operate on.
1274	* @param pCtx Guest context
1275	*
1276	* @remarks Tell PGM about CR3 changes before calling this helper.
1277	*/
1278	static int hmR0VmxSavePaePdpes(PVMCPU pVCpu, PCPUMCTX pCtx)
1279	{
1280	if (CPUMIsGuestInPAEModeEx(pCtx))
1281	{
1282	int rc;
1283	X86PDPE aPdpes[4];
1284	rc = VMXReadVMCS64(VMX_VMCS_GUEST_PDPTR0_FULL, &aPdpes[0].u); AssertRCReturn(rc, rc);
1285	rc = VMXReadVMCS64(VMX_VMCS_GUEST_PDPTR1_FULL, &aPdpes[1].u); AssertRCReturn(rc, rc);
1286	rc = VMXReadVMCS64(VMX_VMCS_GUEST_PDPTR2_FULL, &aPdpes[2].u); AssertRCReturn(rc, rc);
1287	rc = VMXReadVMCS64(VMX_VMCS_GUEST_PDPTR3_FULL, &aPdpes[3].u); AssertRCReturn(rc, rc);
1288
1289	rc = PGMGstUpdatePaePdpes(pVCpu, &aPdpes[0]);
1290	AssertRCReturn(rc, rc);
1291	}
1292	return VINF_SUCCESS;
1293	}
1294
1295
1296	/**
1297	* Update the exception bitmap according to the current CPU state
1298	*
1299	* @param pVM The VM to operate on.
1300	* @param pVCpu The VMCPU to operate on.
1301	* @param pCtx Guest context
1302	*/
1303	static void hmR0VmxUpdateExceptionBitmap(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1304	{
1305	uint32_t u32TrapMask;
1306	Assert(pCtx);
1307
1308	u32TrapMask = HWACCM_VMX_TRAP_MASK;
1309	#ifndef DEBUG
1310	if (pVM->hwaccm.s.fNestedPaging)
1311	u32TrapMask &= ~RT_BIT(X86_XCPT_PF); /* no longer need to intercept #PF. */
1312	#endif
1313
1314	/* Also catch floating point exceptions if we need to report them to the guest in a different way. */
1315	if (!(pCtx->cr0 & X86_CR0_NE))
1316	{
1317	u32TrapMask \|= RT_BIT(X86_XCPT_MF);
1318	}
1319
1320	#ifdef VBOX_STRICT
1321	Assert(u32TrapMask & RT_BIT(X86_XCPT_GP));
1322	#endif
1323
1324	/* Intercept all exceptions in real mode as none of them can be injected directly (#GP otherwise). */
1325	if ( CPUMIsGuestInRealModeEx(pCtx)
1326	&& pVM->hwaccm.s.vmx.pRealModeTSS)
1327	u32TrapMask \|= HWACCM_VMX_TRAP_MASK_REALMODE;
1328
1329	int rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXCEPTION_BITMAP, u32TrapMask);
1330	AssertRC(rc);
1331	}
1332
1333	/**
1334	* Loads a minimal guest state
1335	*
1336	* NOTE: Don't do anything here that can cause a jump back to ring 3!!!!!
1337	*
1338	* @param pVM The VM to operate on.
1339	* @param pVCpu The VMCPU to operate on.
1340	* @param pCtx Guest context
1341	*/
1342	VMMR0DECL(void) VMXR0LoadMinimalGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1343	{
1344	int rc;
1345	X86EFLAGS eflags;
1346
1347	Assert(!(pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_ALL_GUEST));
1348
1349	/* EIP, ESP and EFLAGS */
1350	rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_RIP, pCtx->rip);
1351	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_RSP, pCtx->rsp);
1352	AssertRC(rc);
1353
1354	/* Bits 22-31, 15, 5 & 3 must be zero. Bit 1 must be 1. */
1355	eflags = pCtx->eflags;
1356	eflags.u32 &= VMX_EFLAGS_RESERVED_0;
1357	eflags.u32 \|= VMX_EFLAGS_RESERVED_1;
1358
1359	/* Real mode emulation using v86 mode. */
1360	if ( CPUMIsGuestInRealModeEx(pCtx)
1361	&& pVM->hwaccm.s.vmx.pRealModeTSS)
1362	{
1363	pVCpu->hwaccm.s.vmx.RealMode.eflags = eflags;
1364
1365	eflags.Bits.u1VM = 1;
1366	eflags.Bits.u2IOPL = 0; /* must always be 0 or else certain instructions won't cause faults. */
1367	}
1368	rc = VMXWriteVMCS(VMX_VMCS_GUEST_RFLAGS, eflags.u32);
1369	AssertRC(rc);
1370	}
1371
1372	/**
1373	* Loads the guest state
1374	*
1375	* NOTE: Don't do anything here that can cause a jump back to ring 3!!!!!
1376	*
1377	* @returns VBox status code.
1378	* @param pVM The VM to operate on.
1379	* @param pVCpu The VMCPU to operate on.
1380	* @param pCtx Guest context
1381	*/
1382	VMMR0DECL(int) VMXR0LoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1383	{
1384	int rc = VINF_SUCCESS;
1385	RTGCUINTPTR val;
1386
1387	/* VMX_VMCS_CTRL_ENTRY_CONTROLS
1388	* Set required bits to one and zero according to the MSR capabilities.
1389	*/
1390	val = pVM->hwaccm.s.vmx.msr.vmx_entry.n.disallowed0;
1391	/* Load guest debug controls (dr7 & IA32_DEBUGCTL_MSR) (forced to 1 on the 'first' VT-x capable CPUs; this actually includes the newest Nehalem CPUs) */
1392	val \|= VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_DEBUG;
1393	/* 64 bits guest mode? */
1394	if (CPUMIsGuestInLongModeEx(pCtx))
1395	val \|= VMX_VMCS_CTRL_ENTRY_CONTROLS_IA64_MODE;
1396	/* else Must be zero when AMD64 is not available. */
1397
1398	/* Mask away the bits that the CPU doesn't support */
1399	val &= pVM->hwaccm.s.vmx.msr.vmx_entry.n.allowed1;
1400	rc = VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_CONTROLS, val);
1401	AssertRC(rc);
1402
1403	/* VMX_VMCS_CTRL_EXIT_CONTROLS
1404	* Set required bits to one and zero according to the MSR capabilities.
1405	*/
1406	val = pVM->hwaccm.s.vmx.msr.vmx_exit.n.disallowed0;
1407
1408	/* Save debug controls (dr7 & IA32_DEBUGCTL_MSR) (forced to 1 on the 'first' VT-x capable CPUs; this actually includes the newest Nehalem CPUs) */
1409	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_DEBUG;
1410
1411	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1412	if (VMX_IS_64BIT_HOST_MODE())
1413	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_AMD64;
1414	/* else: Must be zero when AMD64 is not available. */
1415	#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
1416	if (CPUMIsGuestInLongModeEx(pCtx))
1417	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_AMD64; /* our switcher goes to long mode */
1418	else
1419	Assert(!(val & VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_AMD64));
1420	#endif
1421	val &= pVM->hwaccm.s.vmx.msr.vmx_exit.n.allowed1;
1422	/* Don't acknowledge external interrupts on VM-exit. */
1423	rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_CONTROLS, val);
1424	AssertRC(rc);
1425
1426	/* Guest CPU context: ES, CS, SS, DS, FS, GS. */
1427	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_SEGMENT_REGS)
1428	{
1429	if (pVM->hwaccm.s.vmx.pRealModeTSS)
1430	{
1431	PGMMODE enmGuestMode = PGMGetGuestMode(pVCpu);
1432	if (pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode != enmGuestMode)
1433	{
1434	/* Correct weird requirements for switching to protected mode. */
1435	if ( pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode == PGMMODE_REAL
1436	&& enmGuestMode >= PGMMODE_PROTECTED)
1437	{
1438	#ifdef VBOX_WITH_REM
1439	/* Flush the recompiler code cache as it's not unlikely
1440	* the guest will rewrite code it will later execute in real
1441	* mode (OpenBSD 4.0 is one such example)
1442	*/
1443	REMFlushTBs(pVM);
1444	#endif
1445
1446	/* DPL of all hidden selector registers must match the current CPL (0). */
1447	pCtx->csHid.Attr.n.u2Dpl = 0;
1448	pCtx->csHid.Attr.n.u4Type = X86_SEL_TYPE_CODE \| X86_SEL_TYPE_RW_ACC;
1449
1450	pCtx->dsHid.Attr.n.u2Dpl = 0;
1451	pCtx->esHid.Attr.n.u2Dpl = 0;
1452	pCtx->fsHid.Attr.n.u2Dpl = 0;
1453	pCtx->gsHid.Attr.n.u2Dpl = 0;
1454	pCtx->ssHid.Attr.n.u2Dpl = 0;
1455	}
1456	pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode = enmGuestMode;
1457	}
1458	else
1459	/* VT-x will fail with a guest invalid state otherwise... (CPU state after a reset) */
1460	if ( CPUMIsGuestInRealModeEx(pCtx)
1461	&& pCtx->csHid.u64Base == 0xffff0000)
1462	{
1463	pCtx->csHid.u64Base = 0xf0000;
1464	pCtx->cs = 0xf000;
1465	}
1466	}
1467
1468	VMX_WRITE_SELREG(ES, es);
1469	AssertRC(rc);
1470
1471	VMX_WRITE_SELREG(CS, cs);
1472	AssertRC(rc);
1473
1474	VMX_WRITE_SELREG(SS, ss);
1475	AssertRC(rc);
1476
1477	VMX_WRITE_SELREG(DS, ds);
1478	AssertRC(rc);
1479
1480	VMX_WRITE_SELREG(FS, fs);
1481	AssertRC(rc);
1482
1483	VMX_WRITE_SELREG(GS, gs);
1484	AssertRC(rc);
1485	}
1486
1487	/* Guest CPU context: LDTR. */
1488	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_LDTR)
1489	{
1490	if (pCtx->ldtr == 0)
1491	{
1492	rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_LDTR, 0);
1493	rc \|= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_LIMIT, 0);
1494	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_LDTR_BASE, 0);
1495	/* Note: vmlaunch will fail with 0 or just 0x02. No idea why. */
1496	rc \|= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, 0x82 /* present, LDT */);
1497	}
1498	else
1499	{
1500	rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_LDTR, pCtx->ldtr);
1501	rc \|= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtrHid.u32Limit);
1502	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_LDTR_BASE, pCtx->ldtrHid.u64Base);
1503	rc \|= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, pCtx->ldtrHid.Attr.u);
1504	}
1505	AssertRC(rc);
1506	}
1507	/* Guest CPU context: TR. */
1508	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_TR)
1509	{
1510	/* Real mode emulation using v86 mode with CR4.VME (interrupt redirection using the int bitmap in the TSS) */
1511	if ( CPUMIsGuestInRealModeEx(pCtx)
1512	&& pVM->hwaccm.s.vmx.pRealModeTSS)
1513	{
1514	RTGCPHYS GCPhys;
1515
1516	/* We convert it here every time as pci regions could be reconfigured. */
1517	rc = PDMVMMDevHeapR3ToGCPhys(pVM, pVM->hwaccm.s.vmx.pRealModeTSS, &GCPhys);
1518	AssertRC(rc);
1519
1520	rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_TR, 0);
1521	rc \|= VMXWriteVMCS(VMX_VMCS32_GUEST_TR_LIMIT, HWACCM_VTX_TSS_SIZE);
1522	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_TR_BASE, GCPhys /* phys = virt in this mode */);
1523
1524	X86DESCATTR attr;
1525
1526	attr.u = 0;
1527	attr.n.u1Present = 1;
1528	attr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
1529	val = attr.u;
1530	}
1531	else
1532	{
1533	rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_TR, pCtx->tr);
1534	rc \|= VMXWriteVMCS(VMX_VMCS32_GUEST_TR_LIMIT, pCtx->trHid.u32Limit);
1535	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_TR_BASE, pCtx->trHid.u64Base);
1536
1537	val = pCtx->trHid.Attr.u;
1538
1539	/* The TSS selector must be busy (REM bugs? see defect #XXXX). */
1540	if (!(val & X86_SEL_TYPE_SYS_TSS_BUSY_MASK))
1541	{
1542	if (val & 0xf)
1543	val \|= X86_SEL_TYPE_SYS_TSS_BUSY_MASK;
1544	else
1545	/* Default if no TR selector has been set (otherwise vmlaunch will fail!) */
1546	val = (val & ~0xF) \| X86_SEL_TYPE_SYS_386_TSS_BUSY;
1547	}
1548	AssertMsg((val & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY \|\| (val & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("%#x\n", val));
1549	}
1550	rc \|= VMXWriteVMCS(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, val);
1551	AssertRC(rc);
1552	}
1553	/* Guest CPU context: GDTR. */
1554	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_GDTR)
1555	{
1556	rc = VMXWriteVMCS(VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt);
1557	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_GDTR_BASE, pCtx->gdtr.pGdt);
1558	AssertRC(rc);
1559	}
1560	/* Guest CPU context: IDTR. */
1561	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_IDTR)
1562	{
1563	rc = VMXWriteVMCS(VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt);
1564	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_IDTR_BASE, pCtx->idtr.pIdt);
1565	AssertRC(rc);
1566	}
1567
1568	/*
1569	* Sysenter MSRs
1570	*/
1571	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_MSR)
1572	{
1573	rc = VMXWriteVMCS(VMX_VMCS32_GUEST_SYSENTER_CS, pCtx->SysEnter.cs);
1574	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_SYSENTER_EIP, pCtx->SysEnter.eip);
1575	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_SYSENTER_ESP, pCtx->SysEnter.esp);
1576	AssertRC(rc);
1577	}
1578
1579	/* Control registers */
1580	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_CR0)
1581	{
1582	val = pCtx->cr0;
1583	rc = VMXWriteVMCS(VMX_VMCS_CTRL_CR0_READ_SHADOW, val);
1584	Log2(("Guest CR0-shadow %08x\n", val));
1585	if (CPUMIsGuestFPUStateActive(pVCpu) == false)
1586	{
1587	/* Always use #NM exceptions to load the FPU/XMM state on demand. */
1588	val \|= X86_CR0_TS \| X86_CR0_ET \| X86_CR0_NE \| X86_CR0_MP;
1589	}
1590	else
1591	{
1592	/** @todo check if we support the old style mess correctly. */
1593	if (!(val & X86_CR0_NE))
1594	Log(("Forcing X86_CR0_NE!!!\n"));
1595
1596	val \|= X86_CR0_NE; /* always turn on the native mechanism to report FPU errors (old style uses interrupts) */
1597	}
1598	/* Note: protected mode & paging are always enabled; we use them for emulating real and protected mode without paging too. */
1599	if (!pVM->hwaccm.s.vmx.fUnrestrictedGuest)
1600	val \|= X86_CR0_PE \| X86_CR0_PG;
1601
1602	if (pVM->hwaccm.s.fNestedPaging)
1603	{
1604	if (CPUMIsGuestInPagedProtectedModeEx(pCtx))
1605	{
1606	/* Disable cr3 read/write monitoring as we don't need it for EPT. */
1607	pVCpu->hwaccm.s.vmx.proc_ctls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
1608	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT);
1609	}
1610	else
1611	{
1612	/* Reenable cr3 read/write monitoring as our identity mapped page table is active. */
1613	pVCpu->hwaccm.s.vmx.proc_ctls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
1614	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
1615	}
1616	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1617	AssertRC(rc);
1618	}
1619	else
1620	{
1621	/* Note: We must also set this as we rely on protecting various pages for which supervisor writes must be caught. */
1622	val \|= X86_CR0_WP;
1623	}
1624
1625	/* Always enable caching. */
1626	val &= ~(X86_CR0_CD\|X86_CR0_NW);
1627
1628	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_CR0, val);
1629	Log2(("Guest CR0 %08x\n", val));
1630	/* CR0 flags owned by the host; if the guests attempts to change them, then
1631	* the VM will exit.
1632	*/
1633	val = X86_CR0_PE /* Must monitor this bit (assumptions are made for real mode emulation) */
1634	\| X86_CR0_WP /* Must monitor this bit (it must always be enabled). */
1635	\| X86_CR0_PG /* Must monitor this bit (assumptions are made for real mode & protected mode without paging emulation) */
1636	\| X86_CR0_CD /* Bit not restored during VM-exit! */
1637	\| X86_CR0_NW /* Bit not restored during VM-exit! */
1638	\| X86_CR0_NE;
1639
1640	/* When the guest's FPU state is active, then we no longer care about
1641	* the FPU related bits.
1642	*/
1643	if (CPUMIsGuestFPUStateActive(pVCpu) == false)
1644	val \|= X86_CR0_TS \| X86_CR0_ET \| X86_CR0_MP;
1645
1646	pVCpu->hwaccm.s.vmx.cr0_mask = val;
1647
1648	rc \|= VMXWriteVMCS(VMX_VMCS_CTRL_CR0_MASK, val);
1649	Log2(("Guest CR0-mask %08x\n", val));
1650	AssertRC(rc);
1651	}
1652	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_CR4)
1653	{
1654	/* CR4 */
1655	rc = VMXWriteVMCS(VMX_VMCS_CTRL_CR4_READ_SHADOW, pCtx->cr4);
1656	Log2(("Guest CR4-shadow %08x\n", pCtx->cr4));
1657	/* Set the required bits in cr4 too (currently X86_CR4_VMXE). */
1658	val = pCtx->cr4 \| (uint32_t)pVM->hwaccm.s.vmx.msr.vmx_cr4_fixed0;
1659
1660	if (!pVM->hwaccm.s.fNestedPaging)
1661	{
1662	switch(pVCpu->hwaccm.s.enmShadowMode)
1663	{
1664	case PGMMODE_REAL: /* Real mode -> emulated using v86 mode */
1665	case PGMMODE_PROTECTED: /* Protected mode, no paging -> emulated using identity mapping. */
1666	case PGMMODE_32_BIT: /* 32-bit paging. */
1667	val &= ~X86_CR4_PAE;
1668	break;
1669
1670	case PGMMODE_PAE: /* PAE paging. */
1671	case PGMMODE_PAE_NX: /* PAE paging with NX enabled. */
1672	/** Must use PAE paging as we could use physical memory > 4 GB */
1673	val \|= X86_CR4_PAE;
1674	break;
1675
1676	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
1677	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
1678	#ifdef VBOX_ENABLE_64_BITS_GUESTS
1679	break;
1680	#else
1681	AssertFailed();
1682	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1683	#endif
1684	default: /* shut up gcc */
1685	AssertFailed();
1686	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1687	}
1688	}
1689	else
1690	if ( !CPUMIsGuestInPagedProtectedModeEx(pCtx)
1691	&& !pVM->hwaccm.s.vmx.fUnrestrictedGuest)
1692	{
1693	/* We use 4 MB pages in our identity mapping page table for real and protected mode without paging. */
1694	val \|= X86_CR4_PSE;
1695	/* Our identity mapping is a 32 bits page directory. */
1696	val &= ~X86_CR4_PAE;
1697	}
1698
1699	/* Turn off VME if we're in emulated real mode. */
1700	if ( CPUMIsGuestInRealModeEx(pCtx)
1701	&& pVM->hwaccm.s.vmx.pRealModeTSS)
1702	val &= ~X86_CR4_VME;
1703
1704	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_CR4, val);
1705	Log2(("Guest CR4 %08x\n", val));
1706	/* CR4 flags owned by the host; if the guests attempts to change them, then
1707	* the VM will exit.
1708	*/
1709	val = 0
1710	\| X86_CR4_VME
1711	\| X86_CR4_PAE
1712	\| X86_CR4_PGE
1713	\| X86_CR4_PSE
1714	\| X86_CR4_VMXE;
1715	pVCpu->hwaccm.s.vmx.cr4_mask = val;
1716
1717	rc \|= VMXWriteVMCS(VMX_VMCS_CTRL_CR4_MASK, val);
1718	Log2(("Guest CR4-mask %08x\n", val));
1719	AssertRC(rc);
1720	}
1721
1722	#if 0
1723	/* Enable single stepping if requested and CPU supports it. */
1724	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG)
1725	if (DBGFIsStepping(pVCpu))
1726	{
1727	pVCpu->hwaccm.s.vmx.proc_ctls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG;
1728	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1729	AssertRC(rc);
1730	}
1731	#endif
1732
1733	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_CR3)
1734	{
1735	if (pVM->hwaccm.s.fNestedPaging)
1736	{
1737	Assert(PGMGetHyperCR3(pVCpu));
1738	pVCpu->hwaccm.s.vmx.GCPhysEPTP = PGMGetHyperCR3(pVCpu);
1739
1740	Assert(!(pVCpu->hwaccm.s.vmx.GCPhysEPTP & 0xfff));
1741	/** @todo Check the IA32_VMX_EPT_VPID_CAP MSR for other supported memory types. */
1742	pVCpu->hwaccm.s.vmx.GCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
1743	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
1744
1745	rc = VMXWriteVMCS64(VMX_VMCS_CTRL_EPTP_FULL, pVCpu->hwaccm.s.vmx.GCPhysEPTP);
1746	AssertRC(rc);
1747
1748	if ( !CPUMIsGuestInPagedProtectedModeEx(pCtx)
1749	&& !pVM->hwaccm.s.vmx.fUnrestrictedGuest)
1750	{
1751	RTGCPHYS GCPhys;
1752
1753	/* We convert it here every time as pci regions could be reconfigured. */
1754	rc = PDMVMMDevHeapR3ToGCPhys(pVM, pVM->hwaccm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
1755	AssertMsgRC(rc, ("pNonPagingModeEPTPageTable = %RGv\n", pVM->hwaccm.s.vmx.pNonPagingModeEPTPageTable));
1756
1757	/* We use our identity mapping page table here as we need to map guest virtual to guest physical addresses; EPT will
1758	* take care of the translation to host physical addresses.
1759	*/
1760	val = GCPhys;
1761	}
1762	else
1763	{
1764	/* Save the real guest CR3 in VMX_VMCS_GUEST_CR3 */
1765	val = pCtx->cr3;
1766	rc = hmR0VmxLoadPaePdpes(pVCpu, pCtx);
1767	AssertRCReturn(rc, rc);
1768	}
1769	}
1770	else
1771	{
1772	val = PGMGetHyperCR3(pVCpu);
1773	Assert(val \|\| VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL));
1774	}
1775
1776	/* Save our shadow CR3 register. */
1777	rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_CR3, val);
1778	AssertRC(rc);
1779	}
1780
1781	/* Debug registers. */
1782	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_DEBUG)
1783	{
1784	pCtx->dr[6] \|= X86_DR6_INIT_VAL; /* set all reserved bits to 1. */
1785	pCtx->dr[6] &= ~RT_BIT(12); /* must be zero. */
1786
1787	pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
1788	pCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* must be zero */
1789	pCtx->dr[7] \|= 0x400; /* must be one */
1790
1791	/* Resync DR7 */
1792	rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_DR7, pCtx->dr[7]);
1793	AssertRC(rc);
1794
1795	#ifdef DEBUG
1796	/* Sync the hypervisor debug state now if any breakpoint is armed. */
1797	if ( CPUMGetHyperDR7(pVCpu) & (X86_DR7_ENABLED_MASK\|X86_DR7_GD)
1798	&& !CPUMIsHyperDebugStateActive(pVCpu)
1799	&& !DBGFIsStepping(pVCpu))
1800	{
1801	/* Save the host and load the hypervisor debug state. */
1802	rc = CPUMR0LoadHyperDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
1803	AssertRC(rc);
1804
1805	/* DRx intercepts remain enabled. */
1806
1807	/* Override dr7 with the hypervisor value. */
1808	rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_DR7, CPUMGetHyperDR7(pVCpu));
1809	AssertRC(rc);
1810	}
1811	else
1812	#endif
1813	/* Sync the debug state now if any breakpoint is armed. */
1814	if ( (pCtx->dr[7] & (X86_DR7_ENABLED_MASK\|X86_DR7_GD))
1815	&& !CPUMIsGuestDebugStateActive(pVCpu)
1816	&& !DBGFIsStepping(pVCpu))
1817	{
1818	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatDRxArmed);
1819
1820	/* Disable drx move intercepts. */
1821	pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
1822	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1823	AssertRC(rc);
1824
1825	/* Save the host and load the guest debug state. */
1826	rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
1827	AssertRC(rc);
1828	}
1829
1830	/* IA32_DEBUGCTL MSR. */
1831	rc = VMXWriteVMCS64(VMX_VMCS_GUEST_DEBUGCTL_FULL, 0);
1832	AssertRC(rc);
1833
1834	/** @todo do we really ever need this? */
1835	rc \|= VMXWriteVMCS(VMX_VMCS_GUEST_DEBUG_EXCEPTIONS, 0);
1836	AssertRC(rc);
1837	}
1838
1839	/* 64 bits guest mode? */
1840	if (CPUMIsGuestInLongModeEx(pCtx))
1841	{
1842	#if !defined(VBOX_ENABLE_64_BITS_GUESTS)
1843	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1844	#elif HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1845	pVCpu->hwaccm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
1846	#else
1847	# ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1848	if (!pVM->hwaccm.s.fAllow64BitGuests)
1849	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1850	# endif
1851	pVCpu->hwaccm.s.vmx.pfnStartVM = VMXR0StartVM64;
1852	#endif
1853	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_MSR)
1854	{
1855	/* Update these as wrmsr might have changed them. */
1856	rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_FS_BASE, pCtx->fsHid.u64Base);
1857	AssertRC(rc);
1858	rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_GS_BASE, pCtx->gsHid.u64Base);
1859	AssertRC(rc);
1860	}
1861	}
1862	else
1863	{
1864	pVCpu->hwaccm.s.vmx.pfnStartVM = VMXR0StartVM32;
1865	}
1866
1867	hmR0VmxUpdateExceptionBitmap(pVM, pVCpu, pCtx);
1868
1869	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
1870	/* Store all guest MSRs in the VM-Entry load area, so they will be loaded during the world switch. */
1871	PVMXMSR pMsr = (PVMXMSR)pVCpu->hwaccm.s.vmx.pGuestMSR;
1872	unsigned idxMsr = 0;
1873
1874	uint32_t ulEdx;
1875	uint32_t ulTemp;
1876	CPUMGetGuestCpuId(pVCpu, 0x80000001, &ulTemp, &ulTemp, &ulTemp, &ulEdx);
1877	/* EFER MSR present? */
1878	if (ulEdx & (X86_CPUID_AMD_FEATURE_EDX_NX\|X86_CPUID_AMD_FEATURE_EDX_LONG_MODE))
1879	{
1880	pMsr->u32IndexMSR = MSR_K6_EFER;
1881	pMsr->u32Reserved = 0;
1882	pMsr->u64Value = pCtx->msrEFER;
1883	/* VT-x will complain if only MSR_K6_EFER_LME is set. */
1884	if (!CPUMIsGuestInLongModeEx(pCtx))
1885	pMsr->u64Value &= ~(MSR_K6_EFER_LMA\|MSR_K6_EFER_LME);
1886	pMsr++; idxMsr++;
1887
1888	if (ulEdx & X86_CPUID_AMD_FEATURE_EDX_LONG_MODE)
1889	{
1890	pMsr->u32IndexMSR = MSR_K8_LSTAR;
1891	pMsr->u32Reserved = 0;
1892	pMsr->u64Value = pCtx->msrLSTAR; /* 64 bits mode syscall rip */
1893	pMsr++; idxMsr++;
1894	pMsr->u32IndexMSR = MSR_K6_STAR;
1895	pMsr->u32Reserved = 0;
1896	pMsr->u64Value = pCtx->msrSTAR; /* legacy syscall eip, cs & ss */
1897	pMsr++; idxMsr++;
1898	pMsr->u32IndexMSR = MSR_K8_SF_MASK;
1899	pMsr->u32Reserved = 0;
1900	pMsr->u64Value = pCtx->msrSFMASK; /* syscall flag mask */
1901	pMsr++; idxMsr++;
1902	pMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
1903	pMsr->u32Reserved = 0;
1904	pMsr->u64Value = pCtx->msrKERNELGSBASE; /* swapgs exchange value */
1905	pMsr++; idxMsr++;
1906	}
1907	}
1908	pVCpu->hwaccm.s.vmx.cCachedMSRs = idxMsr;
1909
1910	rc = VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_MSR_LOAD_COUNT, idxMsr);
1911	AssertRC(rc);
1912
1913	rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_MSR_STORE_COUNT, idxMsr);
1914	AssertRC(rc);
1915	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
1916
1917	bool fOffsettedTsc;
1918	if (pVM->hwaccm.s.vmx.fUsePreemptTimer)
1919	{
1920	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &pVCpu->hwaccm.s.vmx.u64TSCOffset);
1921
1922	/* Make sure the returned values have sane upper and lower boundaries. */
1923	uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
1924
1925	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64 of a second */
1926	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
1927
1928	cTicksToDeadline >>= pVM->hwaccm.s.vmx.cPreemptTimerShift;
1929	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
1930	rc = VMXWriteVMCS(VMX_VMCS32_GUEST_PREEMPTION_TIMER_VALUE, cPreemptionTickCount);
1931	AssertRC(rc);
1932	}
1933	else
1934	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hwaccm.s.vmx.u64TSCOffset);
1935	if (fOffsettedTsc)
1936	{
1937	uint64_t u64CurTSC = ASMReadTSC();
1938	if (u64CurTSC + pVCpu->hwaccm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
1939	{
1940	/* Note: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT takes precedence over TSC_OFFSET */
1941	rc = VMXWriteVMCS64(VMX_VMCS_CTRL_TSC_OFFSET_FULL, pVCpu->hwaccm.s.vmx.u64TSCOffset);
1942	AssertRC(rc);
1943
1944	pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
1945	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1946	AssertRC(rc);
1947	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTSCOffset);
1948	}
1949	else
1950	{
1951	/* Fall back to rdtsc emulation as we would otherwise pass decreasing tsc values to the guest. */
1952	LogFlow(("TSC %RX64 offset %RX64 time=%RX64 last=%RX64 (diff=%RX64, virt_tsc=%RX64)\n", u64CurTSC, pVCpu->hwaccm.s.vmx.u64TSCOffset, u64CurTSC + pVCpu->hwaccm.s.vmx.u64TSCOffset, TMCpuTickGetLastSeen(pVCpu), TMCpuTickGetLastSeen(pVCpu) - u64CurTSC - pVCpu->hwaccm.s.vmx.u64TSCOffset, TMCpuTickGet(pVCpu)));
1953	pVCpu->hwaccm.s.vmx.proc_ctls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
1954	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1955	AssertRC(rc);
1956	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTSCInterceptOverFlow);
1957	}
1958	}
1959	else
1960	{
1961	pVCpu->hwaccm.s.vmx.proc_ctls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
1962	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1963	AssertRC(rc);
1964	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTSCIntercept);
1965	}
1966
1967	/* Done with the major changes */
1968	pVCpu->hwaccm.s.fContextUseFlags &= ~HWACCM_CHANGED_ALL_GUEST;
1969
1970	/* Minimal guest state update (esp, eip, eflags mostly) */
1971	VMXR0LoadMinimalGuestState(pVM, pVCpu, pCtx);
1972	return rc;
1973	}
1974
1975	/**
1976	* Syncs back the guest state
1977	*
1978	* @returns VBox status code.
1979	* @param pVM The VM to operate on.
1980	* @param pVCpu The VMCPU to operate on.
1981	* @param pCtx Guest context
1982	*/
1983	DECLINLINE(int) VMXR0SaveGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1984	{
1985	RTGCUINTREG val, valShadow;
1986	RTGCUINTPTR uInterruptState;
1987	int rc;
1988
1989	/* Let's first sync back eip, esp, and eflags. */
1990	rc = VMXReadCachedVMCS(VMX_VMCS64_GUEST_RIP, &val);
1991	AssertRC(rc);
1992	pCtx->rip = val;
1993	rc = VMXReadCachedVMCS(VMX_VMCS64_GUEST_RSP, &val);
1994	AssertRC(rc);
1995	pCtx->rsp = val;
1996	rc = VMXReadCachedVMCS(VMX_VMCS_GUEST_RFLAGS, &val);
1997	AssertRC(rc);
1998	pCtx->eflags.u32 = val;
1999
2000	/* Take care of instruction fusing (sti, mov ss) */
2001	rc \|= VMXReadCachedVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &val);
2002	uInterruptState = val;
2003	if (uInterruptState != 0)
2004	{
2005	Assert(uInterruptState <= 2); /* only sti & mov ss */
2006	Log(("uInterruptState %x eip=%RGv\n", (uint32_t)uInterruptState, pCtx->rip));
2007	EMSetInhibitInterruptsPC(pVCpu, pCtx->rip);
2008	}
2009	else
2010	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2011
2012	/* Control registers. */
2013	VMXReadCachedVMCS(VMX_VMCS_CTRL_CR0_READ_SHADOW, &valShadow);
2014	VMXReadCachedVMCS(VMX_VMCS64_GUEST_CR0, &val);
2015	val = (valShadow & pVCpu->hwaccm.s.vmx.cr0_mask) \| (val & ~pVCpu->hwaccm.s.vmx.cr0_mask);
2016	CPUMSetGuestCR0(pVCpu, val);
2017
2018	VMXReadCachedVMCS(VMX_VMCS_CTRL_CR4_READ_SHADOW, &valShadow);
2019	VMXReadCachedVMCS(VMX_VMCS64_GUEST_CR4, &val);
2020	val = (valShadow & pVCpu->hwaccm.s.vmx.cr4_mask) \| (val & ~pVCpu->hwaccm.s.vmx.cr4_mask);
2021	CPUMSetGuestCR4(pVCpu, val);
2022
2023	/* Note: no reason to sync back the CRx registers. They can't be changed by the guest. */
2024	/* Note: only in the nested paging case can CR3 & CR4 be changed by the guest. */
2025	if ( pVM->hwaccm.s.fNestedPaging
2026	&& CPUMIsGuestInPagedProtectedModeEx(pCtx)) /** @todo check if we will always catch mode switches and such... */
2027	{
2028	PVMCSCACHE pCache = &pVCpu->hwaccm.s.vmx.VMCSCache;
2029
2030	/* Can be updated behind our back in the nested paging case. */
2031	CPUMSetGuestCR2(pVCpu, pCache->cr2);
2032
2033	VMXReadCachedVMCS(VMX_VMCS64_GUEST_CR3, &val);
2034
2035	if (val != pCtx->cr3)
2036	{
2037	CPUMSetGuestCR3(pVCpu, val);
2038	PGMUpdateCR3(pVCpu, val);
2039	}
2040	rc = hmR0VmxSavePaePdpes(pVCpu, pCtx);
2041	AssertRCReturn(rc, rc);
2042	}
2043
2044	/* Sync back DR7 here. */
2045	VMXReadCachedVMCS(VMX_VMCS64_GUEST_DR7, &val);
2046	pCtx->dr[7] = val;
2047
2048	/* Guest CPU context: ES, CS, SS, DS, FS, GS. */
2049	VMX_READ_SELREG(ES, es);
2050	VMX_READ_SELREG(SS, ss);
2051	VMX_READ_SELREG(CS, cs);
2052	VMX_READ_SELREG(DS, ds);
2053	VMX_READ_SELREG(FS, fs);
2054	VMX_READ_SELREG(GS, gs);
2055
2056	/*
2057	* System MSRs
2058	*/
2059	VMXReadCachedVMCS(VMX_VMCS32_GUEST_SYSENTER_CS, &val);
2060	pCtx->SysEnter.cs = val;
2061	VMXReadCachedVMCS(VMX_VMCS64_GUEST_SYSENTER_EIP, &val);
2062	pCtx->SysEnter.eip = val;
2063	VMXReadCachedVMCS(VMX_VMCS64_GUEST_SYSENTER_ESP, &val);
2064	pCtx->SysEnter.esp = val;
2065
2066	/* Misc. registers; must sync everything otherwise we can get out of sync when jumping to ring 3. */
2067	VMX_READ_SELREG(LDTR, ldtr);
2068
2069	VMXReadCachedVMCS(VMX_VMCS32_GUEST_GDTR_LIMIT, &val);
2070	pCtx->gdtr.cbGdt = val;
2071	VMXReadCachedVMCS(VMX_VMCS64_GUEST_GDTR_BASE, &val);
2072	pCtx->gdtr.pGdt = val;
2073
2074	VMXReadCachedVMCS(VMX_VMCS32_GUEST_IDTR_LIMIT, &val);
2075	pCtx->idtr.cbIdt = val;
2076	VMXReadCachedVMCS(VMX_VMCS64_GUEST_IDTR_BASE, &val);
2077	pCtx->idtr.pIdt = val;
2078
2079	/* Real mode emulation using v86 mode. */
2080	if ( CPUMIsGuestInRealModeEx(pCtx)
2081	&& pVM->hwaccm.s.vmx.pRealModeTSS)
2082	{
2083	/* Hide our emulation flags */
2084	pCtx->eflags.Bits.u1VM = 0;
2085
2086	/* Restore original IOPL setting as we always use 0. */
2087	pCtx->eflags.Bits.u2IOPL = pVCpu->hwaccm.s.vmx.RealMode.eflags.Bits.u2IOPL;
2088
2089	/* Force a TR resync every time in case we switch modes. */
2090	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_TR;
2091	}
2092	else
2093	{
2094	/* In real mode we have a fake TSS, so only sync it back when it's supposed to be valid. */
2095	VMX_READ_SELREG(TR, tr);
2096	}
2097
2098	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
2099	/* Save the possibly changed MSRs that we automatically restore and save during a world switch. */
2100	for (unsigned i = 0; i < pVCpu->hwaccm.s.vmx.cCachedMSRs; i++)
2101	{
2102	PVMXMSR pMsr = (PVMXMSR)pVCpu->hwaccm.s.vmx.pGuestMSR;
2103	pMsr += i;
2104
2105	switch (pMsr->u32IndexMSR)
2106	{
2107	case MSR_K8_LSTAR:
2108	pCtx->msrLSTAR = pMsr->u64Value;
2109	break;
2110	case MSR_K6_STAR:
2111	pCtx->msrSTAR = pMsr->u64Value;
2112	break;
2113	case MSR_K8_SF_MASK:
2114	pCtx->msrSFMASK = pMsr->u64Value;
2115	break;
2116	case MSR_K8_KERNEL_GS_BASE:
2117	pCtx->msrKERNELGSBASE = pMsr->u64Value;
2118	break;
2119	case MSR_K6_EFER:
2120	/* EFER can't be changed without causing a VM-exit. */
2121	// Assert(pCtx->msrEFER == pMsr->u64Value);
2122	break;
2123	default:
2124	AssertFailed();
2125	return VERR_HM_UNEXPECTED_LD_ST_MSR;
2126	}
2127	}
2128	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
2129	return VINF_SUCCESS;
2130	}
2131
2132	/**
2133	* Dummy placeholder
2134	*
2135	* @param pVM The VM to operate on.
2136	* @param pVCpu The VMCPU to operate on.
2137	*/
2138	static DECLCALLBACK(void) hmR0VmxSetupTLBDummy(PVM pVM, PVMCPU pVCpu)
2139	{
2140	NOREF(pVM);
2141	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
2142	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2143	pVCpu->hwaccm.s.TlbShootdown.cPages = 0;
2144	return;
2145	}
2146
2147	/**
2148	* Setup the tagged TLB for EPT
2149	*
2150	* @returns VBox status code.
2151	* @param pVM The VM to operate on.
2152	* @param pVCpu The VMCPU to operate on.
2153	*/
2154	static DECLCALLBACK(void) hmR0VmxSetupTLBEPT(PVM pVM, PVMCPU pVCpu)
2155	{
2156	PHMGLOBLCPUINFO pCpu;
2157
2158	Assert(pVM->hwaccm.s.fNestedPaging);
2159	Assert(!pVM->hwaccm.s.vmx.fVPID);
2160
2161	/* Deal with tagged TLBs if VPID or EPT is supported. */
2162	pCpu = HWACCMR0GetCurrentCpu();
2163	/* Force a TLB flush for the first world switch if the current cpu differs from the one we ran on last. */
2164	/* Note that this can happen both for start and resume due to long jumps back to ring 3. */
2165	if ( pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu
2166	/* if the tlb flush count has changed, another VM has flushed the TLB of this cpu, so we can't use our current ASID anymore. */
2167	\|\| pVCpu->hwaccm.s.cTLBFlushes != pCpu->cTLBFlushes)
2168	{
2169	/* Force a TLB flush on VM entry. */
2170	pVCpu->hwaccm.s.fForceTLBFlush = true;
2171	}
2172	/* Disabled because this has triggered every time I have suspended my
2173	* laptop with a VM running for the past three months or more. */
2174	// else
2175	// Assert(!pCpu->fFlushTLB);
2176
2177	/* Check for tlb shootdown flushes. */
2178	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2179	pVCpu->hwaccm.s.fForceTLBFlush = true;
2180
2181	pVCpu->hwaccm.s.idLastCpu = pCpu->idCpu;
2182	pCpu->fFlushTLB = false;
2183
2184	if (pVCpu->hwaccm.s.fForceTLBFlush)
2185	{
2186	hmR0VmxFlushEPT(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushContext, 0);
2187	}
2188	else
2189	if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2190	{
2191	/* Deal with pending TLB shootdown actions which were queued when we were not executing code. */
2192	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTlbShootdown);
2193
2194	for (unsigned i=0;i<pVCpu->hwaccm.s.TlbShootdown.cPages;i++)
2195	{
2196	/* aTlbShootdownPages contains physical addresses in this case. */
2197	hmR0VmxFlushEPT(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushPage, pVCpu->hwaccm.s.TlbShootdown.aPages[i]);
2198	}
2199	}
2200	pVCpu->hwaccm.s.TlbShootdown.cPages= 0;
2201	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2202
2203	#ifdef VBOX_WITH_STATISTICS
2204	if (pVCpu->hwaccm.s.fForceTLBFlush)
2205	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushTLBWorldSwitch);
2206	else
2207	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatNoFlushTLBWorldSwitch);
2208	#endif
2209	}
2210
2211	#ifdef HWACCM_VTX_WITH_VPID
2212	/**
2213	* Setup the tagged TLB for VPID
2214	*
2215	* @returns VBox status code.
2216	* @param pVM The VM to operate on.
2217	* @param pVCpu The VMCPU to operate on.
2218	*/
2219	static DECLCALLBACK(void) hmR0VmxSetupTLBVPID(PVM pVM, PVMCPU pVCpu)
2220	{
2221	PHMGLOBLCPUINFO pCpu;
2222
2223	Assert(pVM->hwaccm.s.vmx.fVPID);
2224	Assert(!pVM->hwaccm.s.fNestedPaging);
2225
2226	/* Deal with tagged TLBs if VPID or EPT is supported. */
2227	pCpu = HWACCMR0GetCurrentCpu();
2228	/* Force a TLB flush for the first world switch if the current cpu differs from the one we ran on last. */
2229	/* Note that this can happen both for start and resume due to long jumps back to ring 3. */
2230	if ( pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu
2231	/* if the tlb flush count has changed, another VM has flushed the TLB of this cpu, so we can't use our current ASID anymore. */
2232	\|\| pVCpu->hwaccm.s.cTLBFlushes != pCpu->cTLBFlushes)
2233	{
2234	/* Force a TLB flush on VM entry. */
2235	pVCpu->hwaccm.s.fForceTLBFlush = true;
2236	}
2237	else
2238	Assert(!pCpu->fFlushTLB);
2239
2240	pVCpu->hwaccm.s.idLastCpu = pCpu->idCpu;
2241
2242	/* Check for tlb shootdown flushes. */
2243	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2244	pVCpu->hwaccm.s.fForceTLBFlush = true;
2245
2246	/* Make sure we flush the TLB when required. Switch ASID to achieve the same thing, but without actually flushing the whole TLB (which is expensive). */
2247	if (pVCpu->hwaccm.s.fForceTLBFlush)
2248	{
2249	if ( ++pCpu->uCurrentASID >= pVM->hwaccm.s.uMaxASID
2250	\|\| pCpu->fFlushTLB)
2251	{
2252	pCpu->fFlushTLB = false;
2253	pCpu->uCurrentASID = 1; /* start at 1; host uses 0 */
2254	pCpu->cTLBFlushes++;
2255	hmR0VmxFlushVPID(pVM, pVCpu, VMX_FLUSH_ALL_CONTEXTS, 0);
2256	}
2257	else
2258	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushASID);
2259
2260	pVCpu->hwaccm.s.fForceTLBFlush = false;
2261	pVCpu->hwaccm.s.cTLBFlushes = pCpu->cTLBFlushes;
2262	pVCpu->hwaccm.s.uCurrentASID = pCpu->uCurrentASID;
2263	}
2264	else
2265	{
2266	Assert(!pCpu->fFlushTLB);
2267	Assert(pVCpu->hwaccm.s.uCurrentASID && pCpu->uCurrentASID);
2268
2269	if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2270	{
2271	/* Deal with pending TLB shootdown actions which were queued when we were not executing code. */
2272	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTlbShootdown);
2273	for (unsigned i = 0; i < pVCpu->hwaccm.s.TlbShootdown.cPages; i++)
2274	hmR0VmxFlushVPID(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushPage, pVCpu->hwaccm.s.TlbShootdown.aPages[i]);
2275	}
2276	}
2277	pVCpu->hwaccm.s.TlbShootdown.cPages = 0;
2278	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2279
2280	AssertMsg(pVCpu->hwaccm.s.cTLBFlushes == pCpu->cTLBFlushes, ("Flush count mismatch for cpu %d (%x vs %x)\n", pCpu->idCpu, pVCpu->hwaccm.s.cTLBFlushes, pCpu->cTLBFlushes));
2281	AssertMsg(pCpu->uCurrentASID >= 1 && pCpu->uCurrentASID < pVM->hwaccm.s.uMaxASID, ("cpu%d uCurrentASID = %x\n", pCpu->idCpu, pCpu->uCurrentASID));
2282	AssertMsg(pVCpu->hwaccm.s.uCurrentASID >= 1 && pVCpu->hwaccm.s.uCurrentASID < pVM->hwaccm.s.uMaxASID, ("cpu%d VM uCurrentASID = %x\n", pCpu->idCpu, pVCpu->hwaccm.s.uCurrentASID));
2283
2284	int rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hwaccm.s.uCurrentASID);
2285	AssertRC(rc);
2286
2287	if (pVCpu->hwaccm.s.fForceTLBFlush)
2288	hmR0VmxFlushVPID(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushContext, 0);
2289
2290	# ifdef VBOX_WITH_STATISTICS
2291	if (pVCpu->hwaccm.s.fForceTLBFlush)
2292	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushTLBWorldSwitch);
2293	else
2294	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatNoFlushTLBWorldSwitch);
2295	# endif
2296	}
2297	#endif /* HWACCM_VTX_WITH_VPID */
2298
2299	/**
2300	* Runs guest code in a VT-x VM.
2301	*
2302	* @returns VBox status code.
2303	* @param pVM The VM to operate on.
2304	* @param pVCpu The VMCPU to operate on.
2305	* @param pCtx Guest context
2306	*/
2307	VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2308	{
2309	STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatEntry, x);
2310	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hwaccm.s.StatExit1);
2311	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hwaccm.s.StatExit2);
2312
2313	VBOXSTRICTRC rc = VINF_SUCCESS;
2314	int rc2;
2315	RTGCUINTREG val;
2316	RTGCUINTREG exitReason = (RTGCUINTREG)VMX_EXIT_INVALID;
2317	RTGCUINTREG instrError, cbInstr;
2318	RTGCUINTPTR exitQualification = 0;
2319	RTGCUINTPTR intInfo = 0; /* shut up buggy gcc 4 */
2320	RTGCUINTPTR errCode, instrInfo;
2321	bool fSetupTPRCaching = false;
2322	uint64_t u64OldLSTAR = 0;
2323	uint8_t u8LastTPR = 0;
2324	RTCCUINTREG uOldEFlags = ~(RTCCUINTREG)0;
2325	unsigned cResume = 0;
2326	#ifdef VBOX_STRICT
2327	RTCPUID idCpuCheck;
2328	bool fWasInLongMode = false;
2329	#endif
2330	#ifdef VBOX_HIGH_RES_TIMERS_HACK_IN_RING0
2331	uint64_t u64LastTime = RTTimeMilliTS();
2332	#endif
2333
2334	Assert(!(pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC) \|\| (pVCpu->hwaccm.s.vmx.pbVAPIC && pVM->hwaccm.s.vmx.pAPIC));
2335
2336	/* Check if we need to use TPR shadowing. */
2337	if ( CPUMIsGuestInLongModeEx(pCtx)
2338	\|\| ( ((pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC) \|\| pVM->hwaccm.s.fTRPPatchingAllowed)
2339	&& pVM->hwaccm.s.fHasIoApic)
2340	)
2341	{
2342	fSetupTPRCaching = true;
2343	}
2344
2345	Log2(("\nE"));
2346
2347	#ifdef VBOX_STRICT
2348	{
2349	RTCCUINTREG val2;
2350
2351	rc2 = VMXReadVMCS(VMX_VMCS_CTRL_PIN_EXEC_CONTROLS, &val2);
2352	AssertRC(rc2);
2353	Log2(("VMX_VMCS_CTRL_PIN_EXEC_CONTROLS = %08x\n", val2));
2354
2355	/* allowed zero */
2356	if ((val2 & pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.disallowed0)
2357	Log(("Invalid VMX_VMCS_CTRL_PIN_EXEC_CONTROLS: zero\n"));
2358
2359	/* allowed one */
2360	if ((val2 & ~pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.allowed1) != 0)
2361	Log(("Invalid VMX_VMCS_CTRL_PIN_EXEC_CONTROLS: one\n"));
2362
2363	rc2 = VMXReadVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, &val2);
2364	AssertRC(rc2);
2365	Log2(("VMX_VMCS_CTRL_PROC_EXEC_CONTROLS = %08x\n", val2));
2366
2367	/* Must be set according to the MSR, but can be cleared in case of EPT. */
2368	if (pVM->hwaccm.s.fNestedPaging)
2369	val2 \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT
2370	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
2371	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
2372
2373	/* allowed zero */
2374	if ((val2 & pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.disallowed0)
2375	Log(("Invalid VMX_VMCS_CTRL_PROC_EXEC_CONTROLS: zero\n"));
2376
2377	/* allowed one */
2378	if ((val2 & ~pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1) != 0)
2379	Log(("Invalid VMX_VMCS_CTRL_PROC_EXEC_CONTROLS: one\n"));
2380
2381	rc2 = VMXReadVMCS(VMX_VMCS_CTRL_ENTRY_CONTROLS, &val2);
2382	AssertRC(rc2);
2383	Log2(("VMX_VMCS_CTRL_ENTRY_CONTROLS = %08x\n", val2));
2384
2385	/* allowed zero */
2386	if ((val2 & pVM->hwaccm.s.vmx.msr.vmx_entry.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_entry.n.disallowed0)
2387	Log(("Invalid VMX_VMCS_CTRL_ENTRY_CONTROLS: zero\n"));
2388
2389	/* allowed one */
2390	if ((val2 & ~pVM->hwaccm.s.vmx.msr.vmx_entry.n.allowed1) != 0)
2391	Log(("Invalid VMX_VMCS_CTRL_ENTRY_CONTROLS: one\n"));
2392
2393	rc2 = VMXReadVMCS(VMX_VMCS_CTRL_EXIT_CONTROLS, &val2);
2394	AssertRC(rc2);
2395	Log2(("VMX_VMCS_CTRL_EXIT_CONTROLS = %08x\n", val2));
2396
2397	/* allowed zero */
2398	if ((val2 & pVM->hwaccm.s.vmx.msr.vmx_exit.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_exit.n.disallowed0)
2399	Log(("Invalid VMX_VMCS_CTRL_EXIT_CONTROLS: zero\n"));
2400
2401	/* allowed one */
2402	if ((val2 & ~pVM->hwaccm.s.vmx.msr.vmx_exit.n.allowed1) != 0)
2403	Log(("Invalid VMX_VMCS_CTRL_EXIT_CONTROLS: one\n"));
2404	}
2405	fWasInLongMode = CPUMIsGuestInLongModeEx(pCtx);
2406	#endif /* VBOX_STRICT */
2407
2408	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2409	pVCpu->hwaccm.s.vmx.VMCSCache.u64TimeEntry = RTTimeNanoTS();
2410	#endif
2411
2412	/* We can jump to this point to resume execution after determining that a VM-exit is innocent.
2413	*/
2414	ResumeExecution:
2415	if (!STAM_REL_PROFILE_ADV_IS_RUNNING(&pVCpu->hwaccm.s.StatEntry))
2416	STAM_REL_PROFILE_ADV_STOP_START(&pVCpu->hwaccm.s.StatExit2, &pVCpu->hwaccm.s.StatEntry, x);
2417	AssertMsg(pVCpu->hwaccm.s.idEnteredCpu == RTMpCpuId(),
2418	("Expected %d, I'm %d; cResume=%d exitReason=%RGv exitQualification=%RGv\n",
2419	(int)pVCpu->hwaccm.s.idEnteredCpu, (int)RTMpCpuId(), cResume, exitReason, exitQualification));
2420	Assert(!HWACCMR0SuspendPending());
2421	/* Not allowed to switch modes without reloading the host state (32->64 switcher)!! */
2422	Assert(fWasInLongMode == CPUMIsGuestInLongModeEx(pCtx));
2423
2424	/* Safety precaution; looping for too long here can have a very bad effect on the host */
2425	if (RT_UNLIKELY(++cResume > pVM->hwaccm.s.cMaxResumeLoops))
2426	{
2427	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitMaxResume);
2428	rc = VINF_EM_RAW_INTERRUPT;
2429	goto end;
2430	}
2431
2432	/* Check for irq inhibition due to instruction fusing (sti, mov ss). */
2433	if (VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
2434	{
2435	Log(("VM_FF_INHIBIT_INTERRUPTS at %RGv successor %RGv\n", (RTGCPTR)pCtx->rip, EMGetInhibitInterruptsPC(pVCpu)));
2436	if (pCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
2437	{
2438	/* Note: we intentionally don't clear VM_FF_INHIBIT_INTERRUPTS here.
2439	* Before we are able to execute this instruction in raw mode (iret to guest code) an external interrupt might
2440	* force a world switch again. Possibly allowing a guest interrupt to be dispatched in the process. This could
2441	* break the guest. Sounds very unlikely, but such timing sensitive problems are not as rare as you might think.
2442	*/
2443	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2444	/* Irq inhibition is no longer active; clear the corresponding VMX state. */
2445	rc2 = VMXWriteVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, 0);
2446	AssertRC(rc2);
2447	}
2448	}
2449	else
2450	{
2451	/* Irq inhibition is no longer active; clear the corresponding VMX state. */
2452	rc2 = VMXWriteVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, 0);
2453	AssertRC(rc2);
2454	}
2455
2456	#ifdef VBOX_HIGH_RES_TIMERS_HACK_IN_RING0
2457	if (RT_UNLIKELY((cResume & 0xf) == 0))
2458	{
2459	uint64_t u64CurTime = RTTimeMilliTS();
2460
2461	if (RT_UNLIKELY(u64CurTime > u64LastTime))
2462	{
2463	u64LastTime = u64CurTime;
2464	TMTimerPollVoid(pVM, pVCpu);
2465	}
2466	}
2467	#endif
2468
2469	/* Check for pending actions that force us to go back to ring 3. */
2470	if ( VM_FF_ISPENDING(pVM, VM_FF_HWACCM_TO_R3_MASK \| VM_FF_REQUEST \| VM_FF_PGM_POOL_FLUSH_PENDING \| VM_FF_PDM_DMA)
2471	\|\| VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_HWACCM_TO_R3_MASK \| VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL \| VMCPU_FF_REQUEST))
2472	{
2473	/* Check if a sync operation is pending. */
2474	if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
2475	{
2476	rc = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4, VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
2477	if (rc != VINF_SUCCESS)
2478	{
2479	AssertRC(VBOXSTRICTRC_VAL(rc));
2480	Log(("Pending pool sync is forcing us back to ring 3; rc=%d\n", VBOXSTRICTRC_VAL(rc)));
2481	goto end;
2482	}
2483	}
2484
2485	#ifdef DEBUG
2486	/* Intercept X86_XCPT_DB if stepping is enabled */
2487	if (!DBGFIsStepping(pVCpu))
2488	#endif
2489	{
2490	if ( VM_FF_ISPENDING(pVM, VM_FF_HWACCM_TO_R3_MASK)
2491	\|\| VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_HWACCM_TO_R3_MASK))
2492	{
2493	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatSwitchToR3);
2494	rc = RT_UNLIKELY(VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
2495	goto end;
2496	}
2497	}
2498
2499	/* Pending request packets might contain actions that need immediate attention, such as pending hardware interrupts. */
2500	if ( VM_FF_ISPENDING(pVM, VM_FF_REQUEST)
2501	\|\| VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_REQUEST))
2502	{
2503	rc = VINF_EM_PENDING_REQUEST;
2504	goto end;
2505	}
2506
2507	/* Check if a pgm pool flush is in progress. */
2508	if (VM_FF_ISPENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
2509	{
2510	rc = VINF_PGM_POOL_FLUSH_PENDING;
2511	goto end;
2512	}
2513
2514	/* Check if DMA work is pending (2nd+ run). */
2515	if (VM_FF_ISPENDING(pVM, VM_FF_PDM_DMA) && cResume > 1)
2516	{
2517	rc = VINF_EM_RAW_TO_R3;
2518	goto end;
2519	}
2520	}
2521
2522	#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
2523	/*
2524	* Exit to ring-3 preemption/work is pending.
2525	*
2526	* Interrupts are disabled before the call to make sure we don't miss any interrupt
2527	* that would flag preemption (IPI, timer tick, ++). (Would've been nice to do this
2528	* further down, but hmR0VmxCheckPendingInterrupt makes that impossible.)
2529	*
2530	* Note! Interrupts must be disabled done before we check for TLB flushes; TLB
2531	* shootdowns rely on this.
2532	*/
2533	uOldEFlags = ASMIntDisableFlags();
2534	if (RTThreadPreemptIsPending(NIL_RTTHREAD))
2535	{
2536	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitPreemptPending);
2537	rc = VINF_EM_RAW_INTERRUPT;
2538	goto end;
2539	}
2540	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
2541	#endif
2542
2543	/* When external interrupts are pending, we should exit the VM when IF is set. */
2544	/* Note! After VM_FF_INHIBIT_INTERRUPTS check!!! */
2545	rc = hmR0VmxCheckPendingInterrupt(pVM, pVCpu, pCtx);
2546	if (RT_FAILURE(rc))
2547	goto end;
2548
2549	/** @todo check timers?? */
2550
2551	/* TPR caching using CR8 is only available in 64 bits mode */
2552	/* Note the 32 bits exception for AMD (X86_CPUID_AMD_FEATURE_ECX_CR8L), but that appears missing in Intel CPUs */
2553	/* Note: we can't do this in LoadGuestState as PDMApicGetTPR can jump back to ring 3 (lock)!!!!! (no longer true) */
2554	/**
2555	* @todo query and update the TPR only when it could have been changed (mmio access & wrmsr (x2apic))
2556	*/
2557	if (fSetupTPRCaching)
2558	{
2559	/* TPR caching in CR8 */
2560	bool fPending;
2561
2562	rc2 = PDMApicGetTPR(pVCpu, &u8LastTPR, &fPending);
2563	AssertRC(rc2);
2564	/* The TPR can be found at offset 0x80 in the APIC mmio page. */
2565	pVCpu->hwaccm.s.vmx.pbVAPIC[0x80] = u8LastTPR;
2566
2567	/* Two options here:
2568	* - external interrupt pending, but masked by the TPR value.
2569	* -> a CR8 update that lower the current TPR value should cause an exit
2570	* - no pending interrupts
2571	* -> We don't need to be explicitely notified. There are enough world switches for detecting pending interrupts.
2572	*/
2573	rc = VMXWriteVMCS(VMX_VMCS_CTRL_TPR_THRESHOLD, (fPending) ? (u8LastTPR >> 4) : 0); /* cr8 bits 3-0 correspond to bits 7-4 of the task priority mmio register. */
2574	AssertRC(VBOXSTRICTRC_VAL(rc));
2575
2576	if (pVM->hwaccm.s.fTPRPatchingActive)
2577	{
2578	Assert(!CPUMIsGuestInLongModeEx(pCtx));
2579	/* Our patch code uses LSTAR for TPR caching. */
2580	pCtx->msrLSTAR = u8LastTPR;
2581
2582	if (fPending)
2583	{
2584	/* A TPR change could activate a pending interrupt, so catch lstar writes. */
2585	hmR0VmxSetMSRPermission(pVCpu, MSR_K8_LSTAR, true, false);
2586	}
2587	else
2588	{
2589	/* No interrupts are pending, so we don't need to be explicitely notified.
2590	* There are enough world switches for detecting pending interrupts.
2591	*/
2592	hmR0VmxSetMSRPermission(pVCpu, MSR_K8_LSTAR, true, true);
2593	}
2594	}
2595	}
2596
2597	#if defined(HWACCM_VTX_WITH_EPT) && defined(LOG_ENABLED)
2598	if ( pVM->hwaccm.s.fNestedPaging
2599	# ifdef HWACCM_VTX_WITH_VPID
2600	\|\| pVM->hwaccm.s.vmx.fVPID
2601	# endif /* HWACCM_VTX_WITH_VPID */
2602	)
2603	{
2604	PHMGLOBLCPUINFO pCpu;
2605
2606	pCpu = HWACCMR0GetCurrentCpu();
2607	if ( pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu
2608	\|\| pVCpu->hwaccm.s.cTLBFlushes != pCpu->cTLBFlushes)
2609	{
2610	if (pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu)
2611	LogFlow(("Force TLB flush due to rescheduling to a different cpu (%d vs %d)\n", pVCpu->hwaccm.s.idLastCpu, pCpu->idCpu));
2612	else
2613	LogFlow(("Force TLB flush due to changed TLB flush count (%x vs %x)\n", pVCpu->hwaccm.s.cTLBFlushes, pCpu->cTLBFlushes));
2614	}
2615	if (pCpu->fFlushTLB)
2616	LogFlow(("Force TLB flush: first time cpu %d is used -> flush\n", pCpu->idCpu));
2617	else
2618	if (pVCpu->hwaccm.s.fForceTLBFlush)
2619	LogFlow(("Manual TLB flush\n"));
2620	}
2621	#endif
2622	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
2623	PGMRZDynMapFlushAutoSet(pVCpu);
2624	#endif
2625
2626	/*
2627	* NOTE: DO NOT DO ANYTHING AFTER THIS POINT THAT MIGHT JUMP BACK TO RING 3!
2628	* (until the actual world switch)
2629	*/
2630	#ifdef VBOX_STRICT
2631	idCpuCheck = RTMpCpuId();
2632	#endif
2633	#ifdef LOG_ENABLED
2634	VMMR0LogFlushDisable(pVCpu);
2635	#endif
2636	/* Save the host state first. */
2637	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_HOST_CONTEXT)
2638	{
2639	rc = VMXR0SaveHostState(pVM, pVCpu);
2640	if (RT_UNLIKELY(rc != VINF_SUCCESS))
2641	{
2642	VMMR0LogFlushEnable(pVCpu);
2643	goto end;
2644	}
2645	}
2646
2647	/* Load the guest state */
2648	if (!pVCpu->hwaccm.s.fContextUseFlags)
2649	{
2650	VMXR0LoadMinimalGuestState(pVM, pVCpu, pCtx);
2651	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatLoadMinimal);
2652	}
2653	else
2654	{
2655	rc = VMXR0LoadGuestState(pVM, pVCpu, pCtx);
2656	if (RT_UNLIKELY(rc != VINF_SUCCESS))
2657	{
2658	VMMR0LogFlushEnable(pVCpu);
2659	goto end;
2660	}
2661	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatLoadFull);
2662	}
2663
2664	#ifndef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
2665	/* Disable interrupts to make sure a poke will interrupt execution.
2666	* This must be done before we check for TLB flushes; TLB shootdowns rely on this.
2667	*/
2668	uOldEFlags = ASMIntDisableFlags();
2669	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
2670	#endif
2671
2672	/* Non-register state Guest Context */
2673	/** @todo change me according to cpu state */
2674	rc2 = VMXWriteVMCS(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_CMS_GUEST_ACTIVITY_ACTIVE);
2675	AssertRC(rc2);
2676
2677	/* Set TLB flush state as checked until we return from the world switch. */
2678	ASMAtomicWriteBool(&pVCpu->hwaccm.s.fCheckedTLBFlush, true);
2679	/* Deal with tagged TLB setup and invalidation. */
2680	pVM->hwaccm.s.vmx.pfnSetupTaggedTLB(pVM, pVCpu);
2681
2682	/* Manual save and restore:
2683	* - General purpose registers except RIP, RSP
2684	*
2685	* Trashed:
2686	* - CR2 (we don't care)
2687	* - LDTR (reset to 0)
2688	* - DRx (presumably not changed at all)
2689	* - DR7 (reset to 0x400)
2690	* - EFLAGS (reset to RT_BIT(1); not relevant)
2691	*
2692	*/
2693
2694	/* All done! Let's start VM execution. */
2695	STAM_PROFILE_ADV_STOP_START(&pVCpu->hwaccm.s.StatEntry, &pVCpu->hwaccm.s.StatInGC, x);
2696	Assert(idCpuCheck == RTMpCpuId());
2697
2698	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2699	pVCpu->hwaccm.s.vmx.VMCSCache.cResume = cResume;
2700	pVCpu->hwaccm.s.vmx.VMCSCache.u64TimeSwitch = RTTimeNanoTS();
2701	#endif
2702
2703	/* Save the current TPR value in the LSTAR msr so our patches can access it. */
2704	if (pVM->hwaccm.s.fTPRPatchingActive)
2705	{
2706	Assert(pVM->hwaccm.s.fTPRPatchingActive);
2707	u64OldLSTAR = ASMRdMsr(MSR_K8_LSTAR);
2708	ASMWrMsr(MSR_K8_LSTAR, u8LastTPR);
2709	}
2710
2711	TMNotifyStartOfExecution(pVCpu);
2712	#ifdef VBOX_WITH_KERNEL_USING_XMM
2713	rc = hwaccmR0VMXStartVMWrapXMM(pVCpu->hwaccm.s.fResumeVM, pCtx, &pVCpu->hwaccm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hwaccm.s.vmx.pfnStartVM);
2714	#else
2715	rc = pVCpu->hwaccm.s.vmx.pfnStartVM(pVCpu->hwaccm.s.fResumeVM, pCtx, &pVCpu->hwaccm.s.vmx.VMCSCache, pVM, pVCpu);
2716	#endif
2717	ASMAtomicWriteBool(&pVCpu->hwaccm.s.fCheckedTLBFlush, false);
2718	ASMAtomicIncU32(&pVCpu->hwaccm.s.cWorldSwitchExits);
2719	/* Possibly the last TSC value seen by the guest (too high) (only when we're in tsc offset mode). */
2720	if (!(pVCpu->hwaccm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT))
2721	TMCpuTickSetLastSeen(pVCpu, ASMReadTSC() + pVCpu->hwaccm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
2722
2723	TMNotifyEndOfExecution(pVCpu);
2724	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED);
2725	Assert(!(ASMGetFlags() & X86_EFL_IF));
2726
2727	/* Restore the host LSTAR msr if the guest could have changed it. */
2728	if (pVM->hwaccm.s.fTPRPatchingActive)
2729	{
2730	Assert(pVM->hwaccm.s.fTPRPatchingActive);
2731	pVCpu->hwaccm.s.vmx.pbVAPIC[0x80] = pCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
2732	ASMWrMsr(MSR_K8_LSTAR, u64OldLSTAR);
2733	}
2734
2735	STAM_PROFILE_ADV_STOP_START(&pVCpu->hwaccm.s.StatInGC, &pVCpu->hwaccm.s.StatExit1, x);
2736	ASMSetFlags(uOldEFlags);
2737	#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
2738	uOldEFlags = ~(RTCCUINTREG)0;
2739	#endif
2740
2741	AssertMsg(!pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries, ("pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries=%d\n", pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries));
2742
2743	/* In case we execute a goto ResumeExecution later on. */
2744	pVCpu->hwaccm.s.fResumeVM = true;
2745	pVCpu->hwaccm.s.fForceTLBFlush = false;
2746
2747	/*
2748	* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
2749	* IMPORTANT: WE CAN'T DO ANY LOGGING OR OPERATIONS THAT CAN DO A LONGJMP BACK TO RING 3 BEFORE WE'VE SYNCED BACK (MOST OF) THE GUEST STATE
2750	* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
2751	*/
2752
2753	if (RT_UNLIKELY(rc != VINF_SUCCESS))
2754	{
2755	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx);
2756	VMMR0LogFlushEnable(pVCpu);
2757	goto end;
2758	}
2759
2760	/* Success. Query the guest state and figure out what has happened. */
2761
2762	/* Investigate why there was a VM-exit. */
2763	rc2 = VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_REASON, &exitReason);
2764	STAM_COUNTER_INC(&pVCpu->hwaccm.s.paStatExitReasonR0[exitReason & MASK_EXITREASON_STAT]);
2765
2766	exitReason &= 0xffff; /* bit 0-15 contain the exit code. */
2767	rc2 \|= VMXReadCachedVMCS(VMX_VMCS32_RO_VM_INSTR_ERROR, &instrError);
2768	rc2 \|= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &cbInstr);
2769	rc2 \|= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &intInfo);
2770	/* might not be valid; depends on VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID. */
2771	rc2 \|= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERRCODE, &errCode);
2772	rc2 \|= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INSTR_INFO, &instrInfo);
2773	rc2 \|= VMXReadCachedVMCS(VMX_VMCS_RO_EXIT_QUALIFICATION, &exitQualification);
2774	AssertRC(rc2);
2775
2776	/* Sync back the guest state */
2777	rc2 = VMXR0SaveGuestState(pVM, pVCpu, pCtx);
2778	AssertRC(rc2);
2779
2780	/* Note! NOW IT'S SAFE FOR LOGGING! */
2781	VMMR0LogFlushEnable(pVCpu);
2782	Log2(("Raw exit reason %08x\n", exitReason));
2783	#if ARCH_BITS == 64 /* for the time being */
2784	VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, pCtx, exitReason);
2785	#endif
2786
2787	/* Check if an injected event was interrupted prematurely. */
2788	rc2 = VMXReadCachedVMCS(VMX_VMCS32_RO_IDT_INFO, &val);
2789	AssertRC(rc2);
2790	pVCpu->hwaccm.s.Event.intInfo = VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(val);
2791	if ( VMX_EXIT_INTERRUPTION_INFO_VALID(pVCpu->hwaccm.s.Event.intInfo)
2792	/* Ignore 'int xx' as they'll be restarted anyway. */
2793	&& VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hwaccm.s.Event.intInfo) != VMX_EXIT_INTERRUPTION_INFO_TYPE_SW
2794	/* Ignore software exceptions (such as int3) as they'll reoccur when we restart the instruction anyway. */
2795	&& VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hwaccm.s.Event.intInfo) != VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT)
2796	{
2797	Assert(!pVCpu->hwaccm.s.Event.fPending);
2798	pVCpu->hwaccm.s.Event.fPending = true;
2799	/* Error code present? */
2800	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVCpu->hwaccm.s.Event.intInfo))
2801	{
2802	rc2 = VMXReadCachedVMCS(VMX_VMCS32_RO_IDT_ERRCODE, &val);
2803	AssertRC(rc2);
2804	pVCpu->hwaccm.s.Event.errCode = val;
2805	Log(("Pending inject %RX64 at %RGv exit=%08x intInfo=%08x exitQualification=%RGv pending error=%RX64\n", pVCpu->hwaccm.s.Event.intInfo, (RTGCPTR)pCtx->rip, exitReason, intInfo, exitQualification, val));
2806	}
2807	else
2808	{
2809	Log(("Pending inject %RX64 at %RGv exit=%08x intInfo=%08x exitQualification=%RGv\n", pVCpu->hwaccm.s.Event.intInfo, (RTGCPTR)pCtx->rip, exitReason, intInfo, exitQualification));
2810	pVCpu->hwaccm.s.Event.errCode = 0;
2811	}
2812	}
2813	#ifdef VBOX_STRICT
2814	else
2815	if ( VMX_EXIT_INTERRUPTION_INFO_VALID(pVCpu->hwaccm.s.Event.intInfo)
2816	/* Ignore software exceptions (such as int3) as they're reoccur when we restart the instruction anyway. */
2817	&& VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hwaccm.s.Event.intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT)
2818	{
2819	Log(("Ignore pending inject %RX64 at %RGv exit=%08x intInfo=%08x exitQualification=%RGv\n", pVCpu->hwaccm.s.Event.intInfo, (RTGCPTR)pCtx->rip, exitReason, intInfo, exitQualification));
2820	}
2821
2822	if (exitReason == VMX_EXIT_ERR_INVALID_GUEST_STATE)
2823	HWACCMDumpRegs(pVM, pVCpu, pCtx);
2824	#endif
2825
2826	Log2(("E%d: New EIP=%x:%RGv\n", (uint32_t)exitReason, pCtx->cs, (RTGCPTR)pCtx->rip));
2827	Log2(("Exit reason %d, exitQualification %RGv\n", (uint32_t)exitReason, exitQualification));
2828	Log2(("instrInfo=%d instrError=%d instr length=%d\n", (uint32_t)instrInfo, (uint32_t)instrError, (uint32_t)cbInstr));
2829	Log2(("Interruption error code %d\n", (uint32_t)errCode));
2830	Log2(("IntInfo = %08x\n", (uint32_t)intInfo));
2831
2832	/* Sync back the TPR if it was changed. */
2833	if ( fSetupTPRCaching
2834	&& u8LastTPR != pVCpu->hwaccm.s.vmx.pbVAPIC[0x80])
2835	{
2836	rc2 = PDMApicSetTPR(pVCpu, pVCpu->hwaccm.s.vmx.pbVAPIC[0x80]);
2837	AssertRC(rc2);
2838	}
2839
2840	#ifdef DBGFTRACE_ENABLED /** @todo DTrace later. */
2841	RTTraceBufAddMsgF(pVM->CTX_SUFF(hTraceBuf), "vmexit %08x %016RX64 at %04:%08RX64 %RX64",
2842	exitReason, (uint64_t)exitQualification, pCtx->cs, pCtx->rip, (uint64_t)intInfo);
2843	#endif
2844	STAM_PROFILE_ADV_STOP_START(&pVCpu->hwaccm.s.StatExit1, &pVCpu->hwaccm.s.StatExit2, x);
2845
2846	/* Some cases don't need a complete resync of the guest CPU state; handle them here. */
2847	Assert(rc == VINF_SUCCESS); /* might consider VERR_IPE_UNINITIALIZED_STATUS here later... */
2848	switch (exitReason)
2849	{
2850	case VMX_EXIT_EXCEPTION: /* 0 Exception or non-maskable interrupt (NMI). */
2851	case VMX_EXIT_EXTERNAL_IRQ: /* 1 External interrupt. */
2852	{
2853	uint32_t vector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(intInfo);
2854
2855	if (!VMX_EXIT_INTERRUPTION_INFO_VALID(intInfo))
2856	{
2857	Assert(exitReason == VMX_EXIT_EXTERNAL_IRQ);
2858	#if 0 //def VBOX_WITH_VMMR0_DISABLE_PREEMPTION
2859	if ( RTThreadPreemptIsPendingTrusty()
2860	&& !RTThreadPreemptIsPending(NIL_RTTHREAD))
2861	goto ResumeExecution;
2862	#endif
2863	/* External interrupt; leave to allow it to be dispatched again. */
2864	rc = VINF_EM_RAW_INTERRUPT;
2865	break;
2866	}
2867	STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2868	switch (VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo))
2869	{
2870	case VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI: /* Non-maskable interrupt. */
2871	/* External interrupt; leave to allow it to be dispatched again. */
2872	rc = VINF_EM_RAW_INTERRUPT;
2873	break;
2874
2875	case VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT: /* External hardware interrupt. */
2876	AssertFailed(); /* can't come here; fails the first check. */
2877	break;
2878
2879	case VMX_EXIT_INTERRUPTION_INFO_TYPE_DBEXCPT: /* Unknown why we get this type for #DB */
2880	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT: /* Software exception. (#BP or #OF) */
2881	Assert(vector == 1 \|\| vector == 3 \|\| vector == 4);
2882	/* no break */
2883	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT: /* Hardware exception. */
2884	Log2(("Hardware/software interrupt %d\n", vector));
2885	switch (vector)
2886	{
2887	case X86_XCPT_NM:
2888	{
2889	Log(("#NM fault at %RGv error code %x\n", (RTGCPTR)pCtx->rip, errCode));
2890
2891	/** @todo don't intercept #NM exceptions anymore when we've activated the guest FPU state. */
2892	/* If we sync the FPU/XMM state on-demand, then we can continue execution as if nothing has happened. */
2893	rc = CPUMR0LoadGuestFPU(pVM, pVCpu, pCtx);
2894	if (rc == VINF_SUCCESS)
2895	{
2896	Assert(CPUMIsGuestFPUStateActive(pVCpu));
2897
2898	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitShadowNM);
2899
2900	/* Continue execution. */
2901	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR0;
2902
2903	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2904	goto ResumeExecution;
2905	}
2906
2907	Log(("Forward #NM fault to the guest\n"));
2908	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestNM);
2909	rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, 0);
2910	AssertRC(rc2);
2911	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2912	goto ResumeExecution;
2913	}
2914
2915	case X86_XCPT_PF: /* Page fault */
2916	{
2917	#ifdef DEBUG
2918	if (pVM->hwaccm.s.fNestedPaging)
2919	{ /* A genuine pagefault.
2920	* Forward the trap to the guest by injecting the exception and resuming execution.
2921	*/
2922	Log(("Guest page fault at %RGv cr2=%RGv error code %RGv rsp=%RGv\n", (RTGCPTR)pCtx->rip, exitQualification, errCode, (RTGCPTR)pCtx->rsp));
2923
2924	Assert(CPUMIsGuestInPagedProtectedModeEx(pCtx));
2925
2926	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestPF);
2927
2928	/* Now we must update CR2. */
2929	pCtx->cr2 = exitQualification;
2930	rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
2931	AssertRC(rc2);
2932
2933	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2934	goto ResumeExecution;
2935	}
2936	#endif
2937	Assert(!pVM->hwaccm.s.fNestedPaging);
2938
2939	#ifdef VBOX_HWACCM_WITH_GUEST_PATCHING
2940	/* Shortcut for APIC TPR reads and writes; 32 bits guests only */
2941	if ( pVM->hwaccm.s.fTRPPatchingAllowed
2942	&& pVM->hwaccm.s.pGuestPatchMem
2943	&& (exitQualification & 0xfff) == 0x080
2944	&& !(errCode & X86_TRAP_PF_P) /* not present */
2945	&& CPUMGetGuestCPL(pVCpu, CPUMCTX2CORE(pCtx)) == 0
2946	&& !CPUMIsGuestInLongModeEx(pCtx)
2947	&& pVM->hwaccm.s.cPatches < RT_ELEMENTS(pVM->hwaccm.s.aPatches))
2948	{
2949	RTGCPHYS GCPhysApicBase, GCPhys;
2950	PDMApicGetBase(pVM, &GCPhysApicBase); /* @todo cache this */
2951	GCPhysApicBase &= PAGE_BASE_GC_MASK;
2952
2953	rc = PGMGstGetPage(pVCpu, (RTGCPTR)exitQualification, NULL, &GCPhys);
2954	if ( rc == VINF_SUCCESS
2955	&& GCPhys == GCPhysApicBase)
2956	{
2957	/* Only attempt to patch the instruction once. */
2958	PHWACCMTPRPATCH pPatch = (PHWACCMTPRPATCH)RTAvloU32Get(&pVM->hwaccm.s.PatchTree, (AVLOU32KEY)pCtx->eip);
2959	if (!pPatch)
2960	{
2961	rc = VINF_EM_HWACCM_PATCH_TPR_INSTR;
2962	break;
2963	}
2964	}
2965	}
2966	#endif
2967
2968	Log2(("Page fault at %RGv error code %x\n", exitQualification, errCode));
2969	/* Exit qualification contains the linear address of the page fault. */
2970	TRPMAssertTrap(pVCpu, X86_XCPT_PF, TRPM_TRAP);
2971	TRPMSetErrorCode(pVCpu, errCode);
2972	TRPMSetFaultAddress(pVCpu, exitQualification);
2973
2974	/* Shortcut for APIC TPR reads and writes. */
2975	if ( (exitQualification & 0xfff) == 0x080
2976	&& !(errCode & X86_TRAP_PF_P) /* not present */
2977	&& fSetupTPRCaching
2978	&& (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
2979	{
2980	RTGCPHYS GCPhysApicBase, GCPhys;
2981	PDMApicGetBase(pVM, &GCPhysApicBase); /* @todo cache this */
2982	GCPhysApicBase &= PAGE_BASE_GC_MASK;
2983
2984	rc = PGMGstGetPage(pVCpu, (RTGCPTR)exitQualification, NULL, &GCPhys);
2985	if ( rc == VINF_SUCCESS
2986	&& GCPhys == GCPhysApicBase)
2987	{
2988	Log(("Enable VT-x virtual APIC access filtering\n"));
2989	rc2 = IOMMMIOMapMMIOHCPage(pVM, GCPhysApicBase, pVM->hwaccm.s.vmx.pAPICPhys, X86_PTE_RW \| X86_PTE_P);
2990	AssertRC(rc2);
2991	}
2992	}
2993
2994	/* Forward it to our trap handler first, in case our shadow pages are out of sync. */
2995	rc = PGMTrap0eHandler(pVCpu, errCode, CPUMCTX2CORE(pCtx), (RTGCPTR)exitQualification);
2996	Log2(("PGMTrap0eHandler %RGv returned %Rrc\n", (RTGCPTR)pCtx->rip, VBOXSTRICTRC_VAL(rc)));
2997
2998	if (rc == VINF_SUCCESS)
2999	{ /* We've successfully synced our shadow pages, so let's just continue execution. */
3000	Log2(("Shadow page fault at %RGv cr2=%RGv error code %x\n", (RTGCPTR)pCtx->rip, exitQualification ,errCode));
3001	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitShadowPF);
3002
3003	TRPMResetTrap(pVCpu);
3004	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3005	goto ResumeExecution;
3006	}
3007	else
3008	if (rc == VINF_EM_RAW_GUEST_TRAP)
3009	{ /* A genuine pagefault.
3010	* Forward the trap to the guest by injecting the exception and resuming execution.
3011	*/
3012	Log2(("Forward page fault to the guest\n"));
3013
3014	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestPF);
3015	/* The error code might have been changed. */
3016	errCode = TRPMGetErrorCode(pVCpu);
3017
3018	TRPMResetTrap(pVCpu);
3019
3020	/* Now we must update CR2. */
3021	pCtx->cr2 = exitQualification;
3022	rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
3023	AssertRC(rc2);
3024
3025	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3026	goto ResumeExecution;
3027	}
3028	#ifdef VBOX_STRICT
3029	if (rc != VINF_EM_RAW_EMULATE_INSTR && rc != VINF_EM_RAW_EMULATE_IO_BLOCK)
3030	Log2(("PGMTrap0eHandler failed with %d\n", VBOXSTRICTRC_VAL(rc)));
3031	#endif
3032	/* Need to go back to the recompiler to emulate the instruction. */
3033	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitShadowPFEM);
3034	TRPMResetTrap(pVCpu);
3035	break;
3036	}
3037
3038	case X86_XCPT_MF: /* Floating point exception. */
3039	{
3040	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestMF);
3041	if (!(pCtx->cr0 & X86_CR0_NE))
3042	{
3043	/* old style FPU error reporting needs some extra work. */
3044	/** @todo don't fall back to the recompiler, but do it manually. */
3045	rc = VINF_EM_RAW_EMULATE_INSTR;
3046	break;
3047	}
3048	Log(("Trap %x at %04X:%RGv\n", vector, pCtx->cs, (RTGCPTR)pCtx->rip));
3049	rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
3050	AssertRC(rc2);
3051
3052	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3053	goto ResumeExecution;
3054	}
3055
3056	case X86_XCPT_DB: /* Debug exception. */
3057	{
3058	uint64_t uDR6;
3059
3060	/* DR6, DR7.GD and IA32_DEBUGCTL.LBR are not updated yet.
3061	*
3062	* Exit qualification bits:
3063	* 3:0 B0-B3 which breakpoint condition was met
3064	* 12:4 Reserved (0)
3065	* 13 BD - debug register access detected
3066	* 14 BS - single step execution or branch taken
3067	* 63:15 Reserved (0)
3068	*/
3069	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestDB);
3070
3071	/* Note that we don't support guest and host-initiated debugging at the same time. */
3072
3073	uDR6 = X86_DR6_INIT_VAL;
3074	uDR6 \|= (exitQualification & (X86_DR6_B0\|X86_DR6_B1\|X86_DR6_B2\|X86_DR6_B3\|X86_DR6_BD\|X86_DR6_BS));
3075	rc = DBGFRZTrap01Handler(pVM, pVCpu, CPUMCTX2CORE(pCtx), uDR6);
3076	if (rc == VINF_EM_RAW_GUEST_TRAP)
3077	{
3078	/* Update DR6 here. */
3079	pCtx->dr[6] = uDR6;
3080
3081	/* Resync DR6 if the debug state is active. */
3082	if (CPUMIsGuestDebugStateActive(pVCpu))
3083	ASMSetDR6(pCtx->dr[6]);
3084
3085	/* X86_DR7_GD will be cleared if drx accesses should be trapped inside the guest. */
3086	pCtx->dr[7] &= ~X86_DR7_GD;
3087
3088	/* Paranoia. */
3089	pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
3090	pCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* must be zero */
3091	pCtx->dr[7] \|= 0x400; /* must be one */
3092
3093	/* Resync DR7 */
3094	rc2 = VMXWriteVMCS64(VMX_VMCS64_GUEST_DR7, pCtx->dr[7]);
3095	AssertRC(rc2);
3096
3097	Log(("Trap %x (debug) at %RGv exit qualification %RX64 dr6=%x dr7=%x\n", vector, (RTGCPTR)pCtx->rip, exitQualification, (uint32_t)pCtx->dr[6], (uint32_t)pCtx->dr[7]));
3098	rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
3099	AssertRC(rc2);
3100
3101	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3102	goto ResumeExecution;
3103	}
3104	/* Return to ring 3 to deal with the debug exit code. */
3105	Log(("Debugger hardware BP at %04x:%RGv (rc=%Rrc)\n", pCtx->cs, pCtx->rip, VBOXSTRICTRC_VAL(rc)));
3106	break;
3107	}
3108
3109	case X86_XCPT_BP: /* Breakpoint. */
3110	{
3111	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestBP);
3112	rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pCtx));
3113	if (rc == VINF_EM_RAW_GUEST_TRAP)
3114	{
3115	Log(("Guest #BP at %04x:%RGv\n", pCtx->cs, pCtx->rip));
3116	rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
3117	AssertRC(rc2);
3118	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3119	goto ResumeExecution;
3120	}
3121	if (rc == VINF_SUCCESS)
3122	{
3123	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3124	goto ResumeExecution;
3125	}
3126	Log(("Debugger BP at %04x:%RGv (rc=%Rrc)\n", pCtx->cs, pCtx->rip, VBOXSTRICTRC_VAL(rc)));
3127	break;
3128	}
3129
3130	case X86_XCPT_GP: /* General protection failure exception.*/
3131	{
3132	uint32_t cbOp;
3133	PDISCPUSTATE pDis = &pVCpu->hwaccm.s.DisState;
3134
3135	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestGP);
3136	#ifdef VBOX_STRICT
3137	if ( !CPUMIsGuestInRealModeEx(pCtx)
3138	\|\| !pVM->hwaccm.s.vmx.pRealModeTSS)
3139	{
3140	Log(("Trap %x at %04X:%RGv errorCode=%RGv\n", vector, pCtx->cs, (RTGCPTR)pCtx->rip, errCode));
3141	rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
3142	AssertRC(rc2);
3143	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3144	goto ResumeExecution;
3145	}
3146	#endif
3147	Assert(CPUMIsGuestInRealModeEx(pCtx));
3148
3149	LogFlow(("Real mode X86_XCPT_GP instruction emulation at %x:%RGv\n", pCtx->cs, (RTGCPTR)pCtx->rip));
3150
3151	rc2 = EMInterpretDisasOne(pVM, pVCpu, CPUMCTX2CORE(pCtx), pDis, &cbOp);
3152	if (RT_SUCCESS(rc2))
3153	{
3154	bool fUpdateRIP = true;
3155
3156	rc = VINF_SUCCESS;
3157	Assert(cbOp == pDis->opsize);
3158	switch (pDis->pCurInstr->opcode)
3159	{
3160	case OP_CLI:
3161	pCtx->eflags.Bits.u1IF = 0;
3162	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCli);
3163	break;
3164
3165	case OP_STI:
3166	pCtx->eflags.Bits.u1IF = 1;
3167	EMSetInhibitInterruptsPC(pVCpu, pCtx->rip + pDis->opsize);
3168	Assert(VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
3169	rc2 = VMXWriteVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
3170	AssertRC(rc2);
3171	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitSti);
3172	break;
3173
3174	case OP_HLT:
3175	fUpdateRIP = false;
3176	rc = VINF_EM_HALT;
3177	pCtx->rip += pDis->opsize;
3178	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitHlt);
3179	break;
3180
3181	case OP_POPF:
3182	{
3183	RTGCPTR GCPtrStack;
3184	uint32_t cbParm;
3185	uint32_t uMask;
3186	X86EFLAGS eflags;
3187
3188	if (pDis->prefix & PREFIX_OPSIZE)
3189	{
3190	cbParm = 4;
3191	uMask = 0xffffffff;
3192	}
3193	else
3194	{
3195	cbParm = 2;
3196	uMask = 0xffff;
3197	}
3198
3199	rc2 = SELMToFlatEx(pVCpu, DIS_SELREG_SS, CPUMCTX2CORE(pCtx), pCtx->esp & uMask, 0, &GCPtrStack);
3200	if (RT_FAILURE(rc2))
3201	{
3202	rc = VERR_EM_INTERPRETER;
3203	break;
3204	}
3205	eflags.u = 0;
3206	rc2 = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &eflags.u, cbParm);
3207	if (RT_FAILURE(rc2))
3208	{
3209	rc = VERR_EM_INTERPRETER;
3210	break;
3211	}
3212	LogFlow(("POPF %x -> %RGv mask=%x\n", eflags.u, pCtx->rsp, uMask));
3213	pCtx->eflags.u = (pCtx->eflags.u & ~(X86_EFL_POPF_BITS & uMask)) \| (eflags.u & X86_EFL_POPF_BITS & uMask);
3214	/* RF cleared when popped in real mode; see pushf description in AMD manual. */
3215	pCtx->eflags.Bits.u1RF = 0;
3216	pCtx->esp += cbParm;
3217	pCtx->esp &= uMask;
3218
3219	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitPopf);
3220	break;
3221	}
3222
3223	case OP_PUSHF:
3224	{
3225	RTGCPTR GCPtrStack;
3226	uint32_t cbParm;
3227	uint32_t uMask;
3228	X86EFLAGS eflags;
3229
3230	if (pDis->prefix & PREFIX_OPSIZE)
3231	{
3232	cbParm = 4;
3233	uMask = 0xffffffff;
3234	}
3235	else
3236	{
3237	cbParm = 2;
3238	uMask = 0xffff;
3239	}
3240
3241	rc2 = SELMToFlatEx(pVCpu, DIS_SELREG_SS, CPUMCTX2CORE(pCtx), (pCtx->esp - cbParm) & uMask, 0, &GCPtrStack);
3242	if (RT_FAILURE(rc2))
3243	{
3244	rc = VERR_EM_INTERPRETER;
3245	break;
3246	}
3247	eflags = pCtx->eflags;
3248	/* RF & VM cleared when pushed in real mode; see pushf description in AMD manual. */
3249	eflags.Bits.u1RF = 0;
3250	eflags.Bits.u1VM = 0;
3251
3252	rc2 = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &eflags.u, cbParm);
3253	if (RT_FAILURE(rc2))
3254	{
3255	rc = VERR_EM_INTERPRETER;
3256	break;
3257	}
3258	LogFlow(("PUSHF %x -> %RGv\n", eflags.u, GCPtrStack));
3259	pCtx->esp -= cbParm;
3260	pCtx->esp &= uMask;
3261	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitPushf);
3262	break;
3263	}
3264
3265	case OP_IRET:
3266	{
3267	RTGCPTR GCPtrStack;
3268	uint32_t uMask = 0xffff;
3269	uint16_t aIretFrame[3];
3270
3271	if (pDis->prefix & (PREFIX_OPSIZE \| PREFIX_ADDRSIZE))
3272	{
3273	rc = VERR_EM_INTERPRETER;
3274	break;
3275	}
3276
3277	rc2 = SELMToFlatEx(pVCpu, DIS_SELREG_SS, CPUMCTX2CORE(pCtx), pCtx->esp & uMask, 0, &GCPtrStack);
3278	if (RT_FAILURE(rc2))
3279	{
3280	rc = VERR_EM_INTERPRETER;
3281	break;
3282	}
3283	rc2 = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
3284	if (RT_FAILURE(rc2))
3285	{
3286	rc = VERR_EM_INTERPRETER;
3287	break;
3288	}
3289	pCtx->ip = aIretFrame[0];
3290	pCtx->cs = aIretFrame[1];
3291	pCtx->csHid.u64Base = pCtx->cs << 4;
3292	pCtx->eflags.u = (pCtx->eflags.u & ~(X86_EFL_POPF_BITS & uMask)) \| (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
3293	pCtx->sp += sizeof(aIretFrame);
3294
3295	LogFlow(("iret to %04x:%x\n", pCtx->cs, pCtx->ip));
3296	fUpdateRIP = false;
3297	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIret);
3298	break;
3299	}
3300
3301	case OP_INT:
3302	{
3303	uint32_t intInfo2;
3304
3305	LogFlow(("Realmode: INT %x\n", pDis->param1.parval & 0xff));
3306	intInfo2 = pDis->param1.parval & 0xff;
3307	intInfo2 \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
3308	intInfo2 \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
3309
3310	rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo2, cbOp, 0);
3311	AssertRC(VBOXSTRICTRC_VAL(rc));
3312	fUpdateRIP = false;
3313	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInt);
3314	break;
3315	}
3316
3317	case OP_INTO:
3318	{
3319	if (pCtx->eflags.Bits.u1OF)
3320	{
3321	uint32_t intInfo2;
3322
3323	LogFlow(("Realmode: INTO\n"));
3324	intInfo2 = X86_XCPT_OF;
3325	intInfo2 \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
3326	intInfo2 \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
3327
3328	rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo2, cbOp, 0);
3329	AssertRC(VBOXSTRICTRC_VAL(rc));
3330	fUpdateRIP = false;
3331	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInt);
3332	}
3333	break;
3334	}
3335
3336	case OP_INT3:
3337	{
3338	uint32_t intInfo2;
3339
3340	LogFlow(("Realmode: INT 3\n"));
3341	intInfo2 = 3;
3342	intInfo2 \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
3343	intInfo2 \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
3344
3345	rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo2, cbOp, 0);
3346	AssertRC(VBOXSTRICTRC_VAL(rc));
3347	fUpdateRIP = false;
3348	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInt);
3349	break;
3350	}
3351
3352	default:
3353	rc = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pCtx), 0, EMCODETYPE_SUPERVISOR);
3354	fUpdateRIP = false;
3355	break;
3356	}
3357
3358	if (rc == VINF_SUCCESS)
3359	{
3360	if (fUpdateRIP)
3361	pCtx->rip += cbOp; /* Move on to the next instruction. */
3362
3363	/* lidt, lgdt can end up here. In the future crx changes as well. Just reload the whole context to be done with it. */
3364	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_ALL;
3365
3366	/* Only resume if successful. */
3367	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3368	goto ResumeExecution;
3369	}
3370	}
3371	else
3372	rc = VERR_EM_INTERPRETER;
3373
3374	AssertMsg(rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_EM_HALT, ("Unexpected rc=%Rrc\n", VBOXSTRICTRC_VAL(rc)));
3375	break;
3376	}
3377
3378	#ifdef VBOX_STRICT
3379	case X86_XCPT_XF: /* SIMD exception. */
3380	case X86_XCPT_DE: /* Divide error. */
3381	case X86_XCPT_UD: /* Unknown opcode exception. */
3382	case X86_XCPT_SS: /* Stack segment exception. */
3383	case X86_XCPT_NP: /* Segment not present exception. */
3384	{
3385	switch(vector)
3386	{
3387	case X86_XCPT_DE:
3388	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestDE);
3389	break;
3390	case X86_XCPT_UD:
3391	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestUD);
3392	break;
3393	case X86_XCPT_SS:
3394	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestSS);
3395	break;
3396	case X86_XCPT_NP:
3397	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestNP);
3398	break;
3399	case X86_XCPT_XF:
3400	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestXF);
3401	break;
3402	}
3403
3404	Log(("Trap %x at %04X:%RGv\n", vector, pCtx->cs, (RTGCPTR)pCtx->rip));
3405	rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
3406	AssertRC(rc2);
3407
3408	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3409	goto ResumeExecution;
3410	}
3411	#endif
3412	default:
3413	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestXcpUnk);
3414	if ( CPUMIsGuestInRealModeEx(pCtx)
3415	&& pVM->hwaccm.s.vmx.pRealModeTSS)
3416	{
3417	Log(("Real Mode Trap %x at %04x:%04X error code %x\n", vector, pCtx->cs, pCtx->eip, errCode));
3418	rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
3419	AssertRC(VBOXSTRICTRC_VAL(rc)); /* Strict RC check below. */
3420
3421	/* Go back to ring 3 in case of a triple fault. */
3422	if ( vector == X86_XCPT_DF
3423	&& rc == VINF_EM_RESET)
3424	break;
3425
3426	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3427	goto ResumeExecution;
3428	}
3429	AssertMsgFailed(("Unexpected vm-exit caused by exception %x\n", vector));
3430	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
3431	break;
3432	} /* switch (vector) */
3433
3434	break;
3435
3436	default:
3437	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_CODE;
3438	AssertMsgFailed(("Unexpected interruption code %x\n", intInfo));
3439	break;
3440	}
3441
3442	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3443	break;
3444	}
3445
3446	case VMX_EXIT_EPT_VIOLATION: /* 48 EPT violation. An attempt to access memory with a guest-physical address was disallowed by the configuration of the EPT paging structures. */
3447	{
3448	RTGCPHYS GCPhys;
3449
3450	Assert(pVM->hwaccm.s.fNestedPaging);
3451
3452	rc2 = VMXReadVMCS64(VMX_VMCS_EXIT_PHYS_ADDR_FULL, &GCPhys);
3453	AssertRC(rc2);
3454	Assert(((exitQualification >> 7) & 3) != 2);
3455
3456	/* Determine the kind of violation. */
3457	errCode = 0;
3458	if (exitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
3459	errCode \|= X86_TRAP_PF_ID;
3460
3461	if (exitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
3462	errCode \|= X86_TRAP_PF_RW;
3463
3464	/* If the page is present, then it's a page level protection fault. */
3465	if (exitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
3466	{
3467	errCode \|= X86_TRAP_PF_P;
3468	}
3469	else
3470	{
3471	/* Shortcut for APIC TPR reads and writes. */
3472	if ( (GCPhys & 0xfff) == 0x080
3473	&& GCPhys > 0x1000000 /* to skip VGA frame buffer accesses */
3474	&& fSetupTPRCaching
3475	&& (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
3476	{
3477	RTGCPHYS GCPhysApicBase;
3478	PDMApicGetBase(pVM, &GCPhysApicBase); /* @todo cache this */
3479	GCPhysApicBase &= PAGE_BASE_GC_MASK;
3480	if (GCPhys == GCPhysApicBase + 0x80)
3481	{
3482	Log(("Enable VT-x virtual APIC access filtering\n"));
3483	rc2 = IOMMMIOMapMMIOHCPage(pVM, GCPhysApicBase, pVM->hwaccm.s.vmx.pAPICPhys, X86_PTE_RW \| X86_PTE_P);
3484	AssertRC(rc2);
3485	}
3486	}
3487	}
3488	Log(("EPT Page fault %x at %RGp error code %x\n", (uint32_t)exitQualification, GCPhys, errCode));
3489
3490	/* GCPhys contains the guest physical address of the page fault. */
3491	TRPMAssertTrap(pVCpu, X86_XCPT_PF, TRPM_TRAP);
3492	TRPMSetErrorCode(pVCpu, errCode);
3493	TRPMSetFaultAddress(pVCpu, GCPhys);
3494
3495	/* Handle the pagefault trap for the nested shadow table. */
3496	rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, errCode, CPUMCTX2CORE(pCtx), GCPhys);
3497	if (rc == VINF_SUCCESS)
3498	{ /* We've successfully synced our shadow pages, so let's just continue execution. */
3499	Log2(("Shadow page fault at %RGv cr2=%RGp error code %x\n", (RTGCPTR)pCtx->rip, exitQualification , errCode));
3500	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitReasonNPF);
3501
3502	TRPMResetTrap(pVCpu);
3503	goto ResumeExecution;
3504	}
3505	/** @todo We probably should handle failure to get the instruction page
3506	* (VERR_PAGE_NOT_PRESENT, VERR_PAGE_TABLE_NOT_PRESENT). See
3507	* @bugref{6043}. */
3508
3509	#ifdef VBOX_STRICT
3510	if (rc != VINF_EM_RAW_EMULATE_INSTR)
3511	LogFlow(("PGMTrap0eHandlerNestedPaging at %RGv failed with %Rrc\n", (RTGCPTR)pCtx->rip, VBOXSTRICTRC_VAL(rc)));
3512	#endif
3513	/* Need to go back to the recompiler to emulate the instruction. */
3514	TRPMResetTrap(pVCpu);
3515	break;
3516	}
3517
3518	case VMX_EXIT_EPT_MISCONFIG:
3519	{
3520	RTGCPHYS GCPhys;
3521
3522	Assert(pVM->hwaccm.s.fNestedPaging);
3523
3524	rc2 = VMXReadVMCS64(VMX_VMCS_EXIT_PHYS_ADDR_FULL, &GCPhys);
3525	AssertRC(rc2);
3526	Log(("VMX_EXIT_EPT_MISCONFIG for %RGp\n", GCPhys));
3527
3528	/* Shortcut for APIC TPR reads and writes. */
3529	if ( (GCPhys & 0xfff) == 0x080
3530	&& GCPhys > 0x1000000 /* to skip VGA frame buffer accesses */
3531	&& fSetupTPRCaching
3532	&& (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
3533	{
3534	RTGCPHYS GCPhysApicBase;
3535	PDMApicGetBase(pVM, &GCPhysApicBase); /* @todo cache this */
3536	GCPhysApicBase &= PAGE_BASE_GC_MASK;
3537	if (GCPhys == GCPhysApicBase + 0x80)
3538	{
3539	Log(("Enable VT-x virtual APIC access filtering\n"));
3540	rc2 = IOMMMIOMapMMIOHCPage(pVM, GCPhysApicBase, pVM->hwaccm.s.vmx.pAPICPhys, X86_PTE_RW \| X86_PTE_P);
3541	AssertRC(rc2);
3542	}
3543	}
3544
3545	rc = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pCtx), GCPhys, UINT32_MAX);
3546	if (rc == VINF_SUCCESS)
3547	{
3548	Log2(("PGMR0Trap0eHandlerNPMisconfig(,,,%RGp) at %RGv -> resume\n", GCPhys, (RTGCPTR)pCtx->rip));
3549	goto ResumeExecution;
3550	}
3551	/** @todo We probably should handle failure to get the instruction page
3552	* (VERR_PAGE_NOT_PRESENT, VERR_PAGE_TABLE_NOT_PRESENT). See
3553	* @bugref{6043}. */
3554
3555	Log2(("PGMR0Trap0eHandlerNPMisconfig(,,,%RGp) at %RGv -> %Rrc\n", GCPhys, (RTGCPTR)pCtx->rip, VBOXSTRICTRC_VAL(rc)));
3556	break;
3557	}
3558
3559	case VMX_EXIT_IRQ_WINDOW: /* 7 Interrupt window. */
3560	/* Clear VM-exit on IF=1 change. */
3561	LogFlow(("VMX_EXIT_IRQ_WINDOW %RGv pending=%d IF=%d\n", (RTGCPTR)pCtx->rip, VMCPU_FF_ISPENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC\|VMCPU_FF_INTERRUPT_PIC)), pCtx->eflags.Bits.u1IF));
3562	pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_IRQ_WINDOW_EXIT;
3563	rc2 = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
3564	AssertRC(rc2);
3565	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIrqWindow);
3566	goto ResumeExecution; /* we check for pending guest interrupts there */
3567
3568	case VMX_EXIT_WBINVD: /* 54 Guest software attempted to execute WBINVD. (conditional) */
3569	case VMX_EXIT_INVD: /* 13 Guest software attempted to execute INVD. (unconditional) */
3570	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInvd);
3571	/* Skip instruction and continue directly. */
3572	pCtx->rip += cbInstr;
3573	/* Continue execution.*/
3574	goto ResumeExecution;
3575
3576	case VMX_EXIT_CPUID: /* 10 Guest software attempted to execute CPUID. */
3577	{
3578	Log2(("VMX: Cpuid %x\n", pCtx->eax));
3579	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCpuid);
3580	rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pCtx));
3581	if (rc == VINF_SUCCESS)
3582	{
3583	/* Update EIP and continue execution. */
3584	Assert(cbInstr == 2);
3585	pCtx->rip += cbInstr;
3586	goto ResumeExecution;
3587	}
3588	AssertMsgFailed(("EMU: cpuid failed with %Rrc\n", VBOXSTRICTRC_VAL(rc)));
3589	rc = VINF_EM_RAW_EMULATE_INSTR;
3590	break;
3591	}
3592
3593	case VMX_EXIT_RDPMC: /* 15 Guest software attempted to execute RDPMC. */
3594	{
3595	Log2(("VMX: Rdpmc %x\n", pCtx->ecx));
3596	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitRdpmc);
3597	rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pCtx));
3598	if (rc == VINF_SUCCESS)
3599	{
3600	/* Update EIP and continue execution. */
3601	Assert(cbInstr == 2);
3602	pCtx->rip += cbInstr;
3603	goto ResumeExecution;
3604	}
3605	rc = VINF_EM_RAW_EMULATE_INSTR;
3606	break;
3607	}
3608
3609	case VMX_EXIT_RDTSC: /* 16 Guest software attempted to execute RDTSC. */
3610	{
3611	Log2(("VMX: Rdtsc\n"));
3612	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitRdtsc);
3613	rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pCtx));
3614	if (rc == VINF_SUCCESS)
3615	{
3616	/* Update EIP and continue execution. */
3617	Assert(cbInstr == 2);
3618	pCtx->rip += cbInstr;
3619	goto ResumeExecution;
3620	}
3621	rc = VINF_EM_RAW_EMULATE_INSTR;
3622	break;
3623	}
3624
3625	case VMX_EXIT_INVPG: /* 14 Guest software attempted to execute INVPG. */
3626	{
3627	Log2(("VMX: invlpg\n"));
3628	Assert(!pVM->hwaccm.s.fNestedPaging);
3629
3630	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInvpg);
3631	rc = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pCtx), exitQualification);
3632	if (rc == VINF_SUCCESS)
3633	{
3634	/* Update EIP and continue execution. */
3635	pCtx->rip += cbInstr;
3636	goto ResumeExecution;
3637	}
3638	AssertMsg(rc == VERR_EM_INTERPRETER, ("EMU: invlpg %RGv failed with %Rrc\n", exitQualification, VBOXSTRICTRC_VAL(rc)));
3639	break;
3640	}
3641
3642	case VMX_EXIT_MONITOR: /* 39 Guest software attempted to execute MONITOR. */
3643	{
3644	Log2(("VMX: monitor\n"));
3645
3646	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitMonitor);
3647	rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pCtx));
3648	if (rc == VINF_SUCCESS)
3649	{
3650	/* Update EIP and continue execution. */
3651	pCtx->rip += cbInstr;
3652	goto ResumeExecution;
3653	}
3654	AssertMsg(rc == VERR_EM_INTERPRETER, ("EMU: monitor failed with %Rrc\n", VBOXSTRICTRC_VAL(rc)));
3655	break;
3656	}
3657
3658	case VMX_EXIT_WRMSR: /* 32 WRMSR. Guest software attempted to execute WRMSR. */
3659	/* When an interrupt is pending, we'll let MSR_K8_LSTAR writes fault in our TPR patch code. */
3660	if ( pVM->hwaccm.s.fTPRPatchingActive
3661	&& pCtx->ecx == MSR_K8_LSTAR)
3662	{
3663	Assert(!CPUMIsGuestInLongModeEx(pCtx));
3664	if ((pCtx->eax & 0xff) != u8LastTPR)
3665	{
3666	Log(("VMX: Faulting MSR_K8_LSTAR write with new TPR value %x\n", pCtx->eax & 0xff));
3667
3668	/* Our patch code uses LSTAR for TPR caching. */
3669	rc2 = PDMApicSetTPR(pVCpu, pCtx->eax & 0xff);
3670	AssertRC(rc2);
3671	}
3672
3673	/* Skip the instruction and continue. */
3674	pCtx->rip += cbInstr; /* wrmsr = [0F 30] */
3675
3676	/* Only resume if successful. */
3677	goto ResumeExecution;
3678	}
3679	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_MSR;
3680	/* no break */
3681	case VMX_EXIT_RDMSR: /* 31 RDMSR. Guest software attempted to execute RDMSR. */
3682	{
3683	STAM_COUNTER_INC((exitReason == VMX_EXIT_RDMSR) ? &pVCpu->hwaccm.s.StatExitRdmsr : &pVCpu->hwaccm.s.StatExitWrmsr);
3684
3685	/* Note: the intel manual claims there's a REX version of RDMSR that's slightly different, so we play safe by completely disassembling the instruction. */
3686	Log2(("VMX: %s\n", (exitReason == VMX_EXIT_RDMSR) ? "rdmsr" : "wrmsr"));
3687	rc = EMInterpretInstruction(pVCpu, CPUMCTX2CORE(pCtx), 0);
3688	if (rc == VINF_SUCCESS)
3689	{
3690	/* EIP has been updated already. */
3691
3692	/* Only resume if successful. */
3693	goto ResumeExecution;
3694	}
3695	AssertMsg(rc == VERR_EM_INTERPRETER, ("EMU: %s failed with %Rrc\n", (exitReason == VMX_EXIT_RDMSR) ? "rdmsr" : "wrmsr", VBOXSTRICTRC_VAL(rc)));
3696	break;
3697	}
3698
3699	case VMX_EXIT_CRX_MOVE: /* 28 Control-register accesses. */
3700	{
3701	STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatExit2Sub2, y2);
3702
3703	switch (VMX_EXIT_QUALIFICATION_CRX_ACCESS(exitQualification))
3704	{
3705	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE:
3706	Log2(("VMX: %RGv mov cr%d, x\n", (RTGCPTR)pCtx->rip, VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification)));
3707	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification)]);
3708	rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
3709	VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification),
3710	VMX_EXIT_QUALIFICATION_CRX_GENREG(exitQualification));
3711
3712	switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification))
3713	{
3714	case 0:
3715	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR0 \| HWACCM_CHANGED_GUEST_CR3;
3716	break;
3717	case 2:
3718	break;
3719	case 3:
3720	Assert(!pVM->hwaccm.s.fNestedPaging \|\| !CPUMIsGuestInPagedProtectedModeEx(pCtx));
3721	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR3;
3722	break;
3723	case 4:
3724	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR4;
3725	break;
3726	case 8:
3727	/* CR8 contains the APIC TPR */
3728	Assert(!(pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
3729	break;
3730
3731	default:
3732	AssertFailed();
3733	break;
3734	}
3735	break;
3736
3737	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ:
3738	Log2(("VMX: mov x, crx\n"));
3739	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification)]);
3740
3741	Assert(!pVM->hwaccm.s.fNestedPaging \|\| !CPUMIsGuestInPagedProtectedModeEx(pCtx) \|\| VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification) != USE_REG_CR3);
3742
3743	/* CR8 reads only cause an exit when the TPR shadow feature isn't present. */
3744	Assert(VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification) != 8 \|\| !(pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
3745
3746	rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
3747	VMX_EXIT_QUALIFICATION_CRX_GENREG(exitQualification),
3748	VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification));
3749	break;
3750
3751	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS:
3752	Log2(("VMX: clts\n"));
3753	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCLTS);
3754	rc = EMInterpretCLTS(pVM, pVCpu);
3755	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR0;
3756	break;
3757
3758	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW:
3759	Log2(("VMX: lmsw %x\n", VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(exitQualification)));
3760	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitLMSW);
3761	rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(exitQualification));
3762	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR0;
3763	break;
3764	}
3765
3766	/* Update EIP if no error occurred. */
3767	if (RT_SUCCESS(rc))
3768	pCtx->rip += cbInstr;
3769
3770	if (rc == VINF_SUCCESS)
3771	{
3772	/* Only resume if successful. */
3773	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub2, y2);
3774	goto ResumeExecution;
3775	}
3776	Assert(rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3);
3777	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub2, y2);
3778	break;
3779	}
3780
3781	case VMX_EXIT_DRX_MOVE: /* 29 Debug-register accesses. */
3782	{
3783	if ( !DBGFIsStepping(pVCpu)
3784	&& !CPUMIsHyperDebugStateActive(pVCpu))
3785	{
3786	/* Disable drx move intercepts. */
3787	pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
3788	rc2 = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
3789	AssertRC(rc2);
3790
3791	/* Save the host and load the guest debug state. */
3792	rc2 = CPUMR0LoadGuestDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
3793	AssertRC(rc2);
3794
3795	#ifdef LOG_ENABLED
3796	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
3797	Log(("VMX_EXIT_DRX_MOVE: write DR%d genreg %d\n", VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification), VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification)));
3798	else
3799	Log(("VMX_EXIT_DRX_MOVE: read DR%d\n", VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification)));
3800	#endif
3801
3802	#ifdef VBOX_WITH_STATISTICS
3803	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatDRxContextSwitch);
3804	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
3805	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxWrite);
3806	else
3807	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxRead);
3808	#endif
3809
3810	goto ResumeExecution;
3811	}
3812
3813	/** @todo clear VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT after the first time and restore drx registers afterwards */
3814	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
3815	{
3816	Log2(("VMX: mov drx%d, genreg%d\n", VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification), VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification)));
3817	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxWrite);
3818	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
3819	VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification),
3820	VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification));
3821	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_DEBUG;
3822	Log2(("DR7=%08x\n", pCtx->dr[7]));
3823	}
3824	else
3825	{
3826	Log2(("VMX: mov x, drx\n"));
3827	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxRead);
3828	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
3829	VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification),
3830	VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification));
3831	}
3832	/* Update EIP if no error occurred. */
3833	if (RT_SUCCESS(rc))
3834	pCtx->rip += cbInstr;
3835
3836	if (rc == VINF_SUCCESS)
3837	{
3838	/* Only resume if successful. */
3839	goto ResumeExecution;
3840	}
3841	Assert(rc == VERR_EM_INTERPRETER);
3842	break;
3843	}
3844
3845	/* Note: We'll get a #GP if the IO instruction isn't allowed (IOPL or TSS bitmap); no need to double check. */
3846	case VMX_EXIT_PORT_IO: /* 30 I/O instruction. */
3847	{
3848	STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3849	uint32_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(exitQualification);
3850	uint32_t uPort;
3851	bool fIOWrite = (VMX_EXIT_QUALIFICATION_IO_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
3852
3853	/** @todo necessary to make the distinction? */
3854	if (VMX_EXIT_QUALIFICATION_IO_ENCODING(exitQualification) == VMX_EXIT_QUALIFICATION_IO_ENCODING_DX)
3855	{
3856	uPort = pCtx->edx & 0xffff;
3857	}
3858	else
3859	uPort = VMX_EXIT_QUALIFICATION_IO_PORT(exitQualification); /* Immediate encoding. */
3860
3861	/* paranoia */
3862	if (RT_UNLIKELY(uIOWidth == 2 \|\| uIOWidth >= 4))
3863	{
3864	rc = fIOWrite ? VINF_IOM_R3_IOPORT_WRITE : VINF_IOM_R3_IOPORT_READ;
3865	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3866	break;
3867	}
3868
3869	uint32_t cbSize = g_aIOSize[uIOWidth];
3870
3871	if (VMX_EXIT_QUALIFICATION_IO_STRING(exitQualification))
3872	{
3873	/* ins/outs */
3874	PDISCPUSTATE pDis = &pVCpu->hwaccm.s.DisState;
3875
3876	/* Disassemble manually to deal with segment prefixes. */
3877	/** @todo VMX_VMCS_EXIT_GUEST_LINEAR_ADDR contains the flat pointer operand of the instruction. */
3878	/** @todo VMX_VMCS32_RO_EXIT_INSTR_INFO also contains segment prefix info. */
3879	rc2 = EMInterpretDisasOne(pVM, pVCpu, CPUMCTX2CORE(pCtx), pDis, NULL);
3880	if (RT_SUCCESS(rc))
3881	{
3882	if (fIOWrite)
3883	{
3884	Log2(("IOMInterpretOUTSEx %RGv %x size=%d\n", (RTGCPTR)pCtx->rip, uPort, cbSize));
3885	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIOStringWrite);
3886	rc = IOMInterpretOUTSEx(pVM, CPUMCTX2CORE(pCtx), uPort, pDis->prefix, pDis->addrmode, cbSize);
3887	}
3888	else
3889	{
3890	Log2(("IOMInterpretINSEx %RGv %x size=%d\n", (RTGCPTR)pCtx->rip, uPort, cbSize));
3891	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIOStringRead);
3892	rc = IOMInterpretINSEx(pVM, CPUMCTX2CORE(pCtx), uPort, pDis->prefix, pDis->addrmode, cbSize);
3893	}
3894	}
3895	else
3896	rc = VINF_EM_RAW_EMULATE_INSTR;
3897	}
3898	else
3899	{
3900	/* normal in/out */
3901	uint32_t uAndVal = g_aIOOpAnd[uIOWidth];
3902
3903	Assert(!VMX_EXIT_QUALIFICATION_IO_REP(exitQualification));
3904
3905	if (fIOWrite)
3906	{
3907	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIOWrite);
3908	rc = IOMIOPortWrite(pVM, uPort, pCtx->eax & uAndVal, cbSize);
3909	if (rc == VINF_IOM_R3_IOPORT_WRITE)
3910	HWACCMR0SavePendingIOPortWrite(pVCpu, pCtx->rip, pCtx->rip + cbInstr, uPort, uAndVal, cbSize);
3911	}
3912	else
3913	{
3914	uint32_t u32Val = 0;
3915
3916	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIORead);
3917	rc = IOMIOPortRead(pVM, uPort, &u32Val, cbSize);
3918	if (IOM_SUCCESS(rc))
3919	{
3920	/* Write back to the EAX register. */
3921	pCtx->eax = (pCtx->eax & ~uAndVal) \| (u32Val & uAndVal);
3922	}
3923	else
3924	if (rc == VINF_IOM_R3_IOPORT_READ)
3925	HWACCMR0SavePendingIOPortRead(pVCpu, pCtx->rip, pCtx->rip + cbInstr, uPort, uAndVal, cbSize);
3926	}
3927	}
3928	/*
3929	* Handled the I/O return codes.
3930	* (The unhandled cases end up with rc == VINF_EM_RAW_EMULATE_INSTR.)
3931	*/
3932	if (IOM_SUCCESS(rc))
3933	{
3934	/* Update EIP and continue execution. */
3935	pCtx->rip += cbInstr;
3936	if (RT_LIKELY(rc == VINF_SUCCESS))
3937	{
3938	/* If any IO breakpoints are armed, then we should check if a debug trap needs to be generated. */
3939	if (pCtx->dr[7] & X86_DR7_ENABLED_MASK)
3940	{
3941	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatDRxIOCheck);
3942	for (unsigned i=0;i<4;i++)
3943	{
3944	unsigned uBPLen = g_aIOSize[X86_DR7_GET_LEN(pCtx->dr[7], i)];
3945
3946	if ( (uPort >= pCtx->dr[i] && uPort < pCtx->dr[i] + uBPLen)
3947	&& (pCtx->dr[7] & (X86_DR7_L(i) \| X86_DR7_G(i)))
3948	&& (pCtx->dr[7] & X86_DR7_RW(i, X86_DR7_RW_IO)) == X86_DR7_RW(i, X86_DR7_RW_IO))
3949	{
3950	uint64_t uDR6;
3951
3952	Assert(CPUMIsGuestDebugStateActive(pVCpu));
3953
3954	uDR6 = ASMGetDR6();
3955
3956	/* Clear all breakpoint status flags and set the one we just hit. */
3957	uDR6 &= ~(X86_DR6_B0\|X86_DR6_B1\|X86_DR6_B2\|X86_DR6_B3);
3958	uDR6 \|= (uint64_t)RT_BIT(i);
3959
3960	/* Note: AMD64 Architecture Programmer's Manual 13.1:
3961	* Bits 15:13 of the DR6 register is never cleared by the processor and must be cleared by software after
3962	* the contents have been read.
3963	*/
3964	ASMSetDR6(uDR6);
3965
3966	/* X86_DR7_GD will be cleared if drx accesses should be trapped inside the guest. */
3967	pCtx->dr[7] &= ~X86_DR7_GD;
3968
3969	/* Paranoia. */
3970	pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
3971	pCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* must be zero */
3972	pCtx->dr[7] \|= 0x400; /* must be one */
3973
3974	/* Resync DR7 */
3975	rc2 = VMXWriteVMCS64(VMX_VMCS64_GUEST_DR7, pCtx->dr[7]);
3976	AssertRC(rc2);
3977
3978	/* Construct inject info. */
3979	intInfo = X86_XCPT_DB;
3980	intInfo \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
3981	intInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
3982
3983	Log(("Inject IO debug trap at %RGv\n", (RTGCPTR)pCtx->rip));
3984	rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), 0, 0);
3985	AssertRC(rc2);
3986
3987	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3988	goto ResumeExecution;
3989	}
3990	}
3991	}
3992	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3993	goto ResumeExecution;
3994	}
3995	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3996	break;
3997	}
3998
3999	#ifdef VBOX_STRICT
4000	if (rc == VINF_IOM_R3_IOPORT_READ)
4001	Assert(!fIOWrite);
4002	else if (rc == VINF_IOM_R3_IOPORT_WRITE)
4003	Assert(fIOWrite);
4004	else
4005	AssertMsg(RT_FAILURE(rc) \|\| rc == VINF_EM_RAW_EMULATE_INSTR \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rc)));
4006	#endif
4007	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
4008	break;
4009	}
4010
4011	case VMX_EXIT_TPR: /* 43 TPR below threshold. Guest software executed MOV to CR8. */
4012	LogFlow(("VMX_EXIT_TPR\n"));
4013	/* RIP is already set to the next instruction and the TPR has been synced back. Just resume. */
4014	goto ResumeExecution;
4015
4016	case VMX_EXIT_APIC_ACCESS: /* 44 APIC access. Guest software attempted to access memory at a physical address on the APIC-access page. */
4017	{
4018	LogFlow(("VMX_EXIT_APIC_ACCESS\n"));
4019	unsigned uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(exitQualification);
4020
4021	switch(uAccessType)
4022	{
4023	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
4024	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
4025	{
4026	RTGCPHYS GCPhys;
4027	PDMApicGetBase(pVM, &GCPhys);
4028	GCPhys &= PAGE_BASE_GC_MASK;
4029	GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(exitQualification);
4030
4031	LogFlow(("Apic access at %RGp\n", GCPhys));
4032	rc = IOMMMIOPhysHandler(pVM, (uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW, CPUMCTX2CORE(pCtx), GCPhys);
4033	if (rc == VINF_SUCCESS)
4034	goto ResumeExecution; /* rip already updated */
4035	break;
4036	}
4037
4038	default:
4039	rc = VINF_EM_RAW_EMULATE_INSTR;
4040	break;
4041	}
4042	break;
4043	}
4044
4045	case VMX_EXIT_PREEMPTION_TIMER: /* 52 VMX-preemption timer expired. The preemption timer counted down to zero. */
4046	if (!TMTimerPollBool(pVM, pVCpu))
4047	goto ResumeExecution;
4048	rc = VINF_EM_RAW_TIMER_PENDING;
4049	break;
4050
4051	default:
4052	/* The rest is handled after syncing the entire CPU state. */
4053	break;
4054	}
4055
4056	/* Note: the guest state isn't entirely synced back at this stage. */
4057
4058	/* Investigate why there was a VM-exit. (part 2) */
4059	switch (exitReason)
4060	{
4061	case VMX_EXIT_EXCEPTION: /* 0 Exception or non-maskable interrupt (NMI). */
4062	case VMX_EXIT_EXTERNAL_IRQ: /* 1 External interrupt. */
4063	case VMX_EXIT_EPT_VIOLATION:
4064	case VMX_EXIT_EPT_MISCONFIG: /* 49 EPT misconfig is used by the PGM/MMIO optimizations. */
4065	case VMX_EXIT_PREEMPTION_TIMER: /* 52 VMX-preemption timer expired. The preemption timer counted down to zero. */
4066	/* Already handled above. */
4067	break;
4068
4069	case VMX_EXIT_TRIPLE_FAULT: /* 2 Triple fault. */
4070	rc = VINF_EM_RESET; /* Triple fault equals a reset. */
4071	break;
4072
4073	case VMX_EXIT_INIT_SIGNAL: /* 3 INIT signal. */
4074	case VMX_EXIT_SIPI: /* 4 Start-up IPI (SIPI). */
4075	rc = VINF_EM_RAW_INTERRUPT;
4076	AssertFailed(); /* Can't happen. Yet. */
4077	break;
4078
4079	case VMX_EXIT_IO_SMI_IRQ: /* 5 I/O system-management interrupt (SMI). */
4080	case VMX_EXIT_SMI_IRQ: /* 6 Other SMI. */
4081	rc = VINF_EM_RAW_INTERRUPT;
4082	AssertFailed(); /* Can't happen afaik. */
4083	break;
4084
4085	case VMX_EXIT_TASK_SWITCH: /* 9 Task switch: too complicated to emulate, so fall back to the recompiler */
4086	Log(("VMX_EXIT_TASK_SWITCH: exit=%RX64\n", exitQualification));
4087	if ( (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(exitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
4088	&& pVCpu->hwaccm.s.Event.fPending)
4089	{
4090	/* Caused by an injected interrupt. */
4091	pVCpu->hwaccm.s.Event.fPending = false;
4092
4093	Log(("VMX_EXIT_TASK_SWITCH: reassert trap %d\n", VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVCpu->hwaccm.s.Event.intInfo)));
4094	Assert(!VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVCpu->hwaccm.s.Event.intInfo));
4095	rc2 = TRPMAssertTrap(pVCpu, VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVCpu->hwaccm.s.Event.intInfo), TRPM_HARDWARE_INT);
4096	AssertRC(rc2);
4097	}
4098	/* else Exceptions and software interrupts can just be restarted. */
4099	rc = VERR_EM_INTERPRETER;
4100	break;
4101
4102	case VMX_EXIT_HLT: /* 12 Guest software attempted to execute HLT. */
4103	/* Check if external interrupts are pending; if so, don't switch back. */
4104	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitHlt);
4105	pCtx->rip++; /* skip hlt */
4106	if (EMShouldContinueAfterHalt(pVCpu, pCtx))
4107	goto ResumeExecution;
4108
4109	rc = VINF_EM_HALT;
4110	break;
4111
4112	case VMX_EXIT_MWAIT: /* 36 Guest software executed MWAIT. */
4113	Log2(("VMX: mwait\n"));
4114	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitMwait);
4115	rc = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pCtx));
4116	if ( rc == VINF_EM_HALT
4117	\|\| rc == VINF_SUCCESS)
4118	{
4119	/* Update EIP and continue execution. */
4120	pCtx->rip += cbInstr;
4121
4122	/* Check if external interrupts are pending; if so, don't switch back. */
4123	if ( rc == VINF_SUCCESS
4124	\|\| ( rc == VINF_EM_HALT
4125	&& EMShouldContinueAfterHalt(pVCpu, pCtx))
4126	)
4127	goto ResumeExecution;
4128	}
4129	AssertMsg(rc == VERR_EM_INTERPRETER \|\| rc == VINF_EM_HALT, ("EMU: mwait failed with %Rrc\n", VBOXSTRICTRC_VAL(rc)));
4130	break;
4131
4132	case VMX_EXIT_RSM: /* 17 Guest software attempted to execute RSM in SMM. */
4133	AssertFailed(); /* can't happen. */
4134	rc = VERR_EM_INTERPRETER;
4135	break;
4136
4137	case VMX_EXIT_MTF: /* 37 Exit due to Monitor Trap Flag. */
4138	LogFlow(("VMX_EXIT_MTF at %RGv\n", (RTGCPTR)pCtx->rip));
4139	pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG;
4140	rc2 = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
4141	AssertRC(rc2);
4142	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitMTF);
4143	#if 0
4144	DBGFDoneStepping(pVCpu);
4145	#endif
4146	rc = VINF_EM_DBG_STOP;
4147	break;
4148
4149	case VMX_EXIT_VMCALL: /* 18 Guest software executed VMCALL. */
4150	case VMX_EXIT_VMCLEAR: /* 19 Guest software executed VMCLEAR. */
4151	case VMX_EXIT_VMLAUNCH: /* 20 Guest software executed VMLAUNCH. */
4152	case VMX_EXIT_VMPTRLD: /* 21 Guest software executed VMPTRLD. */
4153	case VMX_EXIT_VMPTRST: /* 22 Guest software executed VMPTRST. */
4154	case VMX_EXIT_VMREAD: /* 23 Guest software executed VMREAD. */
4155	case VMX_EXIT_VMRESUME: /* 24 Guest software executed VMRESUME. */
4156	case VMX_EXIT_VMWRITE: /* 25 Guest software executed VMWRITE. */
4157	case VMX_EXIT_VMXOFF: /* 26 Guest software executed VMXOFF. */
4158	case VMX_EXIT_VMXON: /* 27 Guest software executed VMXON. */
4159	/** @todo inject #UD immediately */
4160	rc = VERR_EM_INTERPRETER;
4161	break;
4162
4163	case VMX_EXIT_CPUID: /* 10 Guest software attempted to execute CPUID. */
4164	case VMX_EXIT_RDTSC: /* 16 Guest software attempted to execute RDTSC. */
4165	case VMX_EXIT_INVPG: /* 14 Guest software attempted to execute INVPG. */
4166	case VMX_EXIT_CRX_MOVE: /* 28 Control-register accesses. */
4167	case VMX_EXIT_DRX_MOVE: /* 29 Debug-register accesses. */
4168	case VMX_EXIT_PORT_IO: /* 30 I/O instruction. */
4169	case VMX_EXIT_RDPMC: /* 15 Guest software attempted to execute RDPMC. */
4170	/* already handled above */
4171	AssertMsg( rc == VINF_PGM_CHANGE_MODE
4172	\|\| rc == VINF_EM_RAW_INTERRUPT
4173	\|\| rc == VERR_EM_INTERPRETER
4174	\|\| rc == VINF_EM_RAW_EMULATE_INSTR
4175	\|\| rc == VINF_PGM_SYNC_CR3
4176	\|\| rc == VINF_IOM_R3_IOPORT_READ
4177	\|\| rc == VINF_IOM_R3_IOPORT_WRITE
4178	\|\| rc == VINF_EM_RAW_GUEST_TRAP
4179	\|\| rc == VINF_TRPM_XCPT_DISPATCHED
4180	\|\| rc == VINF_EM_RESCHEDULE_REM,
4181	("rc = %d\n", VBOXSTRICTRC_VAL(rc)));
4182	break;
4183
4184	case VMX_EXIT_TPR: /* 43 TPR below threshold. Guest software executed MOV to CR8. */
4185	case VMX_EXIT_APIC_ACCESS: /* 44 APIC access. Guest software attempted to access memory at a physical address on the APIC-access page. */
4186	case VMX_EXIT_RDMSR: /* 31 RDMSR. Guest software attempted to execute RDMSR. */
4187	case VMX_EXIT_WRMSR: /* 32 WRMSR. Guest software attempted to execute WRMSR. */
4188	case VMX_EXIT_PAUSE: /* 40 Guest software attempted to execute PAUSE. */
4189	case VMX_EXIT_MONITOR: /* 39 Guest software attempted to execute MONITOR. */
4190	/* Note: If we decide to emulate them here, then we must sync the MSRs that could have been changed (sysenter, fs/gs base)!!! */
4191	rc = VERR_EM_INTERPRETER;
4192	break;
4193
4194	case VMX_EXIT_IRQ_WINDOW: /* 7 Interrupt window. */
4195	Assert(rc == VINF_EM_RAW_INTERRUPT);
4196	break;
4197
4198	case VMX_EXIT_ERR_INVALID_GUEST_STATE: /* 33 VM-entry failure due to invalid guest state. */
4199	{
4200	#ifdef VBOX_STRICT
4201	RTCCUINTREG val2 = 0;
4202
4203	Log(("VMX_EXIT_ERR_INVALID_GUEST_STATE\n"));
4204
4205	VMXReadVMCS(VMX_VMCS64_GUEST_RIP, &val2);
4206	Log(("Old eip %RGv new %RGv\n", (RTGCPTR)pCtx->rip, (RTGCPTR)val2));
4207
4208	VMXReadVMCS(VMX_VMCS64_GUEST_CR0, &val2);
4209	Log(("VMX_VMCS_GUEST_CR0 %RX64\n", (uint64_t)val2));
4210
4211	VMXReadVMCS(VMX_VMCS64_GUEST_CR3, &val2);
4212	Log(("VMX_VMCS_GUEST_CR3 %RX64\n", (uint64_t)val2));
4213
4214	VMXReadVMCS(VMX_VMCS64_GUEST_CR4, &val2);
4215	Log(("VMX_VMCS_GUEST_CR4 %RX64\n", (uint64_t)val2));
4216
4217	VMXReadVMCS(VMX_VMCS_GUEST_RFLAGS, &val2);
4218	Log(("VMX_VMCS_GUEST_RFLAGS %08x\n", val2));
4219
4220	VMX_LOG_SELREG(CS, "CS", val2);
4221	VMX_LOG_SELREG(DS, "DS", val2);
4222	VMX_LOG_SELREG(ES, "ES", val2);
4223	VMX_LOG_SELREG(FS, "FS", val2);
4224	VMX_LOG_SELREG(GS, "GS", val2);
4225	VMX_LOG_SELREG(SS, "SS", val2);
4226	VMX_LOG_SELREG(TR, "TR", val2);
4227	VMX_LOG_SELREG(LDTR, "LDTR", val2);
4228
4229	VMXReadVMCS(VMX_VMCS64_GUEST_GDTR_BASE, &val2);
4230	Log(("VMX_VMCS_GUEST_GDTR_BASE %RX64\n", (uint64_t)val2));
4231	VMXReadVMCS(VMX_VMCS64_GUEST_IDTR_BASE, &val2);
4232	Log(("VMX_VMCS_GUEST_IDTR_BASE %RX64\n", (uint64_t)val2));
4233	#endif /* VBOX_STRICT */
4234	rc = VERR_VMX_INVALID_GUEST_STATE;
4235	break;
4236	}
4237
4238	case VMX_EXIT_ERR_MSR_LOAD: /* 34 VM-entry failure due to MSR loading. */
4239	case VMX_EXIT_ERR_MACHINE_CHECK: /* 41 VM-entry failure due to machine-check. */
4240	default:
4241	rc = VERR_VMX_UNEXPECTED_EXIT_CODE;
4242	AssertMsgFailed(("Unexpected exit code %d\n", exitReason)); /* Can't happen. */
4243	break;
4244
4245	}
4246	end:
4247
4248	/* We now going back to ring-3, so clear the action flag. */
4249	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
4250
4251	/* Signal changes for the recompiler. */
4252	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR \| CPUM_CHANGED_LDTR \| CPUM_CHANGED_GDTR \| CPUM_CHANGED_IDTR \| CPUM_CHANGED_TR \| CPUM_CHANGED_HIDDEN_SEL_REGS);
4253
4254	/* If we executed vmlaunch/vmresume and an external irq was pending, then we don't have to do a full sync the next time. */
4255	if ( exitReason == VMX_EXIT_EXTERNAL_IRQ
4256	&& !VMX_EXIT_INTERRUPTION_INFO_VALID(intInfo))
4257	{
4258	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatPendingHostIrq);
4259	/* On the next entry we'll only sync the host context. */
4260	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_HOST_CONTEXT;
4261	}
4262	else
4263	{
4264	/* On the next entry we'll sync everything. */
4265	/** @todo we can do better than this */
4266	/* Not in the VINF_PGM_CHANGE_MODE though! */
4267	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_ALL;
4268	}
4269
4270	/* translate into a less severe return code */
4271	if (rc == VERR_EM_INTERPRETER)
4272	rc = VINF_EM_RAW_EMULATE_INSTR;
4273	else
4274	/* Try to extract more information about what might have gone wrong here. */
4275	if (rc == VERR_VMX_INVALID_VMCS_PTR)
4276	{
4277	VMXGetActivateVMCS(&pVCpu->hwaccm.s.vmx.lasterror.u64VMCSPhys);
4278	pVCpu->hwaccm.s.vmx.lasterror.ulVMCSRevision = (uint32_t )pVCpu->hwaccm.s.vmx.pvVMCS;
4279	pVCpu->hwaccm.s.vmx.lasterror.idEnteredCpu = pVCpu->hwaccm.s.idEnteredCpu;
4280	pVCpu->hwaccm.s.vmx.lasterror.idCurrentCpu = RTMpCpuId();
4281	}
4282
4283	/* Just set the correct state here instead of trying to catch every goto above. */
4284	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED, VMCPUSTATE_STARTED_EXEC);
4285
4286	#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
4287	/* Restore interrupts if we exitted after disabling them. */
4288	if (uOldEFlags != ~(RTCCUINTREG)0)
4289	ASMSetFlags(uOldEFlags);
4290	#endif
4291
4292	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2, x);
4293	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit1, x);
4294	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatEntry, x);
4295	Log2(("X"));
4296	return VBOXSTRICTRC_TODO(rc);
4297	}
4298
4299
4300	/**
4301	* Enters the VT-x session
4302	*
4303	* @returns VBox status code.
4304	* @param pVM The VM to operate on.
4305	* @param pVCpu The VMCPU to operate on.
4306	* @param pCpu CPU info struct
4307	*/
4308	VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBLCPUINFO pCpu)
4309	{
4310	Assert(pVM->hwaccm.s.vmx.fSupported);
4311	NOREF(pCpu);
4312
4313	unsigned cr4 = ASMGetCR4();
4314	if (!(cr4 & X86_CR4_VMXE))
4315	{
4316	AssertMsgFailed(("X86_CR4_VMXE should be set!\n"));
4317	return VERR_VMX_X86_CR4_VMXE_CLEARED;
4318	}
4319
4320	/* Activate the VM Control Structure. */
4321	int rc = VMXActivateVMCS(pVCpu->hwaccm.s.vmx.HCPhysVMCS);
4322	if (RT_FAILURE(rc))
4323	return rc;
4324
4325	pVCpu->hwaccm.s.fResumeVM = false;
4326	return VINF_SUCCESS;
4327	}
4328
4329
4330	/**
4331	* Leaves the VT-x session
4332	*
4333	* @returns VBox status code.
4334	* @param pVM The VM to operate on.
4335	* @param pVCpu The VMCPU to operate on.
4336	* @param pCtx CPU context
4337	*/
4338	VMMR0DECL(int) VMXR0Leave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4339	{
4340	Assert(pVM->hwaccm.s.vmx.fSupported);
4341
4342	#ifdef DEBUG
4343	if (CPUMIsHyperDebugStateActive(pVCpu))
4344	{
4345	CPUMR0LoadHostDebugState(pVM, pVCpu);
4346	Assert(pVCpu->hwaccm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT);
4347	}
4348	else
4349	#endif
4350	/* Save the guest debug state if necessary. */
4351	if (CPUMIsGuestDebugStateActive(pVCpu))
4352	{
4353	CPUMR0SaveGuestDebugState(pVM, pVCpu, pCtx, true /* save DR6 */);
4354
4355	/* Enable drx move intercepts again. */
4356	pVCpu->hwaccm.s.vmx.proc_ctls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
4357	int rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
4358	AssertRC(rc);
4359
4360	/* Resync the debug registers the next time. */
4361	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_DEBUG;
4362	}
4363	else
4364	Assert(pVCpu->hwaccm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT);
4365
4366	/* Clear VM Control Structure. Marking it inactive, clearing implementation specific data and writing back VMCS data to memory. */
4367	int rc = VMXClearVMCS(pVCpu->hwaccm.s.vmx.HCPhysVMCS);
4368	AssertRC(rc);
4369
4370	return VINF_SUCCESS;
4371	}
4372
4373	/**
4374	* Flush the TLB (EPT)
4375	*
4376	* @returns VBox status code.
4377	* @param pVM The VM to operate on.
4378	* @param pVCpu The VM CPU to operate on.
4379	* @param enmFlush Type of flush
4380	* @param GCPhys Physical address of the page to flush
4381	*/
4382	static void hmR0VmxFlushEPT(PVM pVM, PVMCPU pVCpu, VMX_FLUSH enmFlush, RTGCPHYS GCPhys)
4383	{
4384	uint64_t descriptor[2];
4385
4386	LogFlow(("hmR0VmxFlushEPT %d %RGv\n", enmFlush, GCPhys));
4387	Assert(pVM->hwaccm.s.fNestedPaging);
4388	descriptor[0] = pVCpu->hwaccm.s.vmx.GCPhysEPTP;
4389	descriptor[1] = GCPhys;
4390	int rc = VMXR0InvEPT(enmFlush, &descriptor[0]);
4391	AssertRC(rc);
4392	}
4393
4394	#ifdef HWACCM_VTX_WITH_VPID
4395	/**
4396	* Flush the TLB (EPT)
4397	*
4398	* @returns VBox status code.
4399	* @param pVM The VM to operate on.
4400	* @param pVCpu The VM CPU to operate on.
4401	* @param enmFlush Type of flush
4402	* @param GCPtr Virtual address of the page to flush
4403	*/
4404	static void hmR0VmxFlushVPID(PVM pVM, PVMCPU pVCpu, VMX_FLUSH enmFlush, RTGCPTR GCPtr)
4405	{
4406	#if HC_ARCH_BITS == 32
4407	/* If we get a flush in 64 bits guest mode, then force a full TLB flush. Invvpid probably takes only 32 bits addresses. (@todo) */
4408	if ( CPUMIsGuestInLongMode(pVCpu)
4409	&& !VMX_IS_64BIT_HOST_MODE())
4410	{
4411	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
4412	}
4413	else
4414	#endif
4415	{
4416	uint64_t descriptor[2];
4417
4418	Assert(pVM->hwaccm.s.vmx.fVPID);
4419	descriptor[0] = pVCpu->hwaccm.s.uCurrentASID;
4420	descriptor[1] = GCPtr;
4421	int rc = VMXR0InvVPID(enmFlush, &descriptor[0]); NOREF(rc);
4422	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvVPID %x %x %RGv failed with %d\n", enmFlush, pVCpu->hwaccm.s.uCurrentASID, GCPtr, rc));
4423	}
4424	}
4425	#endif /* HWACCM_VTX_WITH_VPID */
4426
4427	/**
4428	* Invalidates a guest page
4429	*
4430	* @returns VBox status code.
4431	* @param pVM The VM to operate on.
4432	* @param pVCpu The VM CPU to operate on.
4433	* @param GCVirt Page to invalidate
4434	*/
4435	VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
4436	{
4437	bool fFlushPending = VMCPU_FF_ISSET(pVCpu, VMCPU_FF_TLB_FLUSH);
4438
4439	Log2(("VMXR0InvalidatePage %RGv\n", GCVirt));
4440
4441	/* Only relevant if we want to use VPID as otherwise every VMX transition
4442	* will flush the TLBs and paging-structure caches.
4443	* In the nested paging case we still see such calls, but
4444	* can safely ignore them. (e.g. after cr3 updates)
4445	*/
4446	#ifdef HWACCM_VTX_WITH_VPID
4447	/* Skip it if a TLB flush is already pending. */
4448	if ( !fFlushPending
4449	&& pVM->hwaccm.s.vmx.fVPID)
4450	hmR0VmxFlushVPID(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushPage, GCVirt);
4451	#endif /* HWACCM_VTX_WITH_VPID */
4452
4453	return VINF_SUCCESS;
4454	}
4455
4456	/**
4457	* Invalidates a guest page by physical address
4458	*
4459	* NOTE: Assumes the current instruction references this physical page though a virtual address!!
4460	*
4461	* @returns VBox status code.
4462	* @param pVM The VM to operate on.
4463	* @param pVCpu The VM CPU to operate on.
4464	* @param GCPhys Page to invalidate
4465	*/
4466	VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
4467	{
4468	bool fFlushPending = VMCPU_FF_ISSET(pVCpu, VMCPU_FF_TLB_FLUSH);
4469
4470	Assert(pVM->hwaccm.s.fNestedPaging);
4471
4472	LogFlow(("VMXR0InvalidatePhysPage %RGp\n", GCPhys));
4473
4474	/* Skip it if a TLB flush is already pending. */
4475	if (!fFlushPending)
4476	hmR0VmxFlushEPT(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushPage, GCPhys);
4477
4478	return VINF_SUCCESS;
4479	}
4480
4481	/**
4482	* Report world switch error and dump some useful debug info
4483	*
4484	* @param pVM The VM to operate on.
4485	* @param pVCpu The VMCPU to operate on.
4486	* @param rc Return code
4487	* @param pCtx Current CPU context (not updated)
4488	*/
4489	static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, VBOXSTRICTRC rc, PCPUMCTX pCtx)
4490	{
4491	NOREF(pVM);
4492
4493	switch (VBOXSTRICTRC_VAL(rc))
4494	{
4495	case VERR_VMX_INVALID_VMXON_PTR:
4496	AssertFailed();
4497	break;
4498
4499	case VERR_VMX_UNABLE_TO_START_VM:
4500	case VERR_VMX_UNABLE_TO_RESUME_VM:
4501	{
4502	int rc2;
4503	RTCCUINTREG exitReason, instrError;
4504
4505	rc2 = VMXReadVMCS(VMX_VMCS32_RO_EXIT_REASON, &exitReason);
4506	rc2 \|= VMXReadVMCS(VMX_VMCS32_RO_VM_INSTR_ERROR, &instrError);
4507	AssertRC(rc2);
4508	if (rc2 == VINF_SUCCESS)
4509	{
4510	Log(("Unable to start/resume VM for reason: %x. Instruction error %x\n", (uint32_t)exitReason, (uint32_t)instrError));
4511	Log(("Current stack %08x\n", &rc2));
4512
4513	pVCpu->hwaccm.s.vmx.lasterror.ulInstrError = instrError;
4514	pVCpu->hwaccm.s.vmx.lasterror.ulExitReason = exitReason;
4515
4516	#ifdef VBOX_STRICT
4517	RTGDTR gdtr;
4518	PCX86DESCHC pDesc;
4519	RTCCUINTREG val;
4520
4521	ASMGetGDTR(&gdtr);
4522
4523	VMXReadVMCS(VMX_VMCS64_GUEST_RIP, &val);
4524	Log(("Old eip %RGv new %RGv\n", (RTGCPTR)pCtx->rip, (RTGCPTR)val));
4525	VMXReadVMCS(VMX_VMCS_CTRL_PIN_EXEC_CONTROLS, &val);
4526	Log(("VMX_VMCS_CTRL_PIN_EXEC_CONTROLS %08x\n", val));
4527	VMXReadVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, &val);
4528	Log(("VMX_VMCS_CTRL_PROC_EXEC_CONTROLS %08x\n", val));
4529	VMXReadVMCS(VMX_VMCS_CTRL_ENTRY_CONTROLS, &val);
4530	Log(("VMX_VMCS_CTRL_ENTRY_CONTROLS %08x\n", val));
4531	VMXReadVMCS(VMX_VMCS_CTRL_EXIT_CONTROLS, &val);
4532	Log(("VMX_VMCS_CTRL_EXIT_CONTROLS %08x\n", val));
4533
4534	VMXReadVMCS(VMX_VMCS_HOST_CR0, &val);
4535	Log(("VMX_VMCS_HOST_CR0 %08x\n", val));
4536
4537	VMXReadVMCS(VMX_VMCS_HOST_CR3, &val);
4538	Log(("VMX_VMCS_HOST_CR3 %08x\n", val));
4539
4540	VMXReadVMCS(VMX_VMCS_HOST_CR4, &val);
4541	Log(("VMX_VMCS_HOST_CR4 %08x\n", val));
4542
4543	VMXReadVMCS(VMX_VMCS16_HOST_FIELD_CS, &val);
4544	Log(("VMX_VMCS_HOST_FIELD_CS %08x\n", val));
4545
4546	VMXReadVMCS(VMX_VMCS_GUEST_RFLAGS, &val);
4547	Log(("VMX_VMCS_GUEST_RFLAGS %08x\n", val));
4548
4549	if (val < gdtr.cbGdt)
4550	{
4551	pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
4552	HWACCMR0DumpDescriptor(pDesc, val, "CS: ");
4553	}
4554
4555	VMXReadVMCS(VMX_VMCS16_HOST_FIELD_DS, &val);
4556	Log(("VMX_VMCS_HOST_FIELD_DS %08x\n", val));
4557	if (val < gdtr.cbGdt)
4558	{
4559	pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
4560	HWACCMR0DumpDescriptor(pDesc, val, "DS: ");
4561	}
4562
4563	VMXReadVMCS(VMX_VMCS16_HOST_FIELD_ES, &val);
4564	Log(("VMX_VMCS_HOST_FIELD_ES %08x\n", val));
4565	if (val < gdtr.cbGdt)
4566	{
4567	pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
4568	HWACCMR0DumpDescriptor(pDesc, val, "ES: ");
4569	}
4570
4571	VMXReadVMCS(VMX_VMCS16_HOST_FIELD_FS, &val);
4572	Log(("VMX_VMCS16_HOST_FIELD_FS %08x\n", val));
4573	if (val < gdtr.cbGdt)
4574	{
4575	pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
4576	HWACCMR0DumpDescriptor(pDesc, val, "FS: ");
4577	}
4578
4579	VMXReadVMCS(VMX_VMCS16_HOST_FIELD_GS, &val);
4580	Log(("VMX_VMCS16_HOST_FIELD_GS %08x\n", val));
4581	if (val < gdtr.cbGdt)
4582	{
4583	pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
4584	HWACCMR0DumpDescriptor(pDesc, val, "GS: ");
4585	}
4586
4587	VMXReadVMCS(VMX_VMCS16_HOST_FIELD_SS, &val);
4588	Log(("VMX_VMCS16_HOST_FIELD_SS %08x\n", val));
4589	if (val < gdtr.cbGdt)
4590	{
4591	pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
4592	HWACCMR0DumpDescriptor(pDesc, val, "SS: ");
4593	}
4594
4595	VMXReadVMCS(VMX_VMCS16_HOST_FIELD_TR, &val);
4596	Log(("VMX_VMCS16_HOST_FIELD_TR %08x\n", val));
4597	if (val < gdtr.cbGdt)
4598	{
4599	pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
4600	HWACCMR0DumpDescriptor(pDesc, val, "TR: ");
4601	}
4602
4603	VMXReadVMCS(VMX_VMCS_HOST_TR_BASE, &val);
4604	Log(("VMX_VMCS_HOST_TR_BASE %RHv\n", val));
4605
4606	VMXReadVMCS(VMX_VMCS_HOST_GDTR_BASE, &val);
4607	Log(("VMX_VMCS_HOST_GDTR_BASE %RHv\n", val));
4608	VMXReadVMCS(VMX_VMCS_HOST_IDTR_BASE, &val);
4609	Log(("VMX_VMCS_HOST_IDTR_BASE %RHv\n", val));
4610
4611	VMXReadVMCS(VMX_VMCS32_HOST_SYSENTER_CS, &val);
4612	Log(("VMX_VMCS_HOST_SYSENTER_CS %08x\n", val));
4613
4614	VMXReadVMCS(VMX_VMCS_HOST_SYSENTER_EIP, &val);
4615	Log(("VMX_VMCS_HOST_SYSENTER_EIP %RHv\n", val));
4616
4617	VMXReadVMCS(VMX_VMCS_HOST_SYSENTER_ESP, &val);
4618	Log(("VMX_VMCS_HOST_SYSENTER_ESP %RHv\n", val));
4619
4620	VMXReadVMCS(VMX_VMCS_HOST_RSP, &val);
4621	Log(("VMX_VMCS_HOST_RSP %RHv\n", val));
4622	VMXReadVMCS(VMX_VMCS_HOST_RIP, &val);
4623	Log(("VMX_VMCS_HOST_RIP %RHv\n", val));
4624
4625	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4626	if (VMX_IS_64BIT_HOST_MODE())
4627	{
4628	Log(("MSR_K6_EFER = %RX64\n", ASMRdMsr(MSR_K6_EFER)));
4629	Log(("MSR_K6_STAR = %RX64\n", ASMRdMsr(MSR_K6_STAR)));
4630	Log(("MSR_K8_LSTAR = %RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
4631	Log(("MSR_K8_CSTAR = %RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
4632	Log(("MSR_K8_SF_MASK = %RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
4633	}
4634	# endif
4635	#endif /* VBOX_STRICT */
4636	}
4637	break;
4638	}
4639
4640	default:
4641	/* impossible */
4642	AssertMsgFailed(("%Rrc (%#x)\n", VBOXSTRICTRC_VAL(rc), VBOXSTRICTRC_VAL(rc)));
4643	break;
4644	}
4645	}
4646
4647	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4648
4649	/**
4650	* Prepares for and executes VMLAUNCH (64 bits guest mode)
4651	*
4652	* @returns VBox status code
4653	* @param fResume vmlauch/vmresume
4654	* @param pCtx Guest context
4655	* @param pCache VMCS cache
4656	* @param pVM The VM to operate on.
4657	* @param pVCpu The VMCPU to operate on.
4658	*/
4659	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
4660	{
4661	uint32_t aParam[6];
4662	PHMGLOBLCPUINFO pCpu;
4663	RTHCPHYS HCPhysCpuPage;
4664	int rc;
4665
4666	pCpu = HWACCMR0GetCurrentCpu();
4667	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4668
4669	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4670	pCache->uPos = 1;
4671	pCache->interPD = PGMGetInterPaeCR3(pVM);
4672	pCache->pSwitcher = (uint64_t)pVM->hwaccm.s.pfnHost32ToGuest64R0;
4673	#endif
4674
4675	#ifdef DEBUG
4676	pCache->TestIn.HCPhysCpuPage= 0;
4677	pCache->TestIn.HCPhysVMCS = 0;
4678	pCache->TestIn.pCache = 0;
4679	pCache->TestOut.HCPhysVMCS = 0;
4680	pCache->TestOut.pCache = 0;
4681	pCache->TestOut.pCtx = 0;
4682	pCache->TestOut.eflags = 0;
4683	#endif
4684
4685	aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
4686	aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
4687	aParam[2] = (uint32_t)(pVCpu->hwaccm.s.vmx.HCPhysVMCS); /* Param 2: VMCS physical address - Lo. */
4688	aParam[3] = (uint32_t)(pVCpu->hwaccm.s.vmx.HCPhysVMCS >> 32); /* Param 2: VMCS physical address - Hi. */
4689	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hwaccm.s.vmx.VMCSCache);
4690	aParam[5] = 0;
4691
4692	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4693	pCtx->dr[4] = pVM->hwaccm.s.vmx.pScratchPhys + 16 + 8;
4694	(uint32_t )(pVM->hwaccm.s.vmx.pScratch + 16 + 8) = 1;
4695	#endif
4696	rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, pVM->hwaccm.s.pfnVMXGCStartVM64, 6, &aParam[0]);
4697
4698	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4699	Assert((uint32_t )(pVM->hwaccm.s.vmx.pScratch + 16 + 8) == 5);
4700	Assert(pCtx->dr[4] == 10);
4701	(uint32_t )(pVM->hwaccm.s.vmx.pScratch + 16 + 8) = 0xff;
4702	#endif
4703
4704	#ifdef DEBUG
4705	AssertMsg(pCache->TestIn.HCPhysCpuPage== HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
4706	AssertMsg(pCache->TestIn.HCPhysVMCS == pVCpu->hwaccm.s.vmx.HCPhysVMCS, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVMCS, pVCpu->hwaccm.s.vmx.HCPhysVMCS));
4707	AssertMsg(pCache->TestIn.HCPhysVMCS == pCache->TestOut.HCPhysVMCS, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVMCS, pCache->TestOut.HCPhysVMCS));
4708	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache, pCache->TestOut.pCache));
4709	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hwaccm.s.vmx.VMCSCache), ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hwaccm.s.vmx.VMCSCache)));
4710	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx, pCache->TestOut.pCtx));
4711	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
4712	#endif
4713	return rc;
4714	}
4715
4716	# ifdef VBOX_STRICT
4717
4718	static bool hmR0VmxIsValidReadField(uint32_t idxField)
4719	{
4720	switch(idxField)
4721	{
4722	case VMX_VMCS64_GUEST_RIP:
4723	case VMX_VMCS64_GUEST_RSP:
4724	case VMX_VMCS_GUEST_RFLAGS:
4725	case VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE:
4726	case VMX_VMCS_CTRL_CR0_READ_SHADOW:
4727	case VMX_VMCS64_GUEST_CR0:
4728	case VMX_VMCS_CTRL_CR4_READ_SHADOW:
4729	case VMX_VMCS64_GUEST_CR4:
4730	case VMX_VMCS64_GUEST_DR7:
4731	case VMX_VMCS32_GUEST_SYSENTER_CS:
4732	case VMX_VMCS64_GUEST_SYSENTER_EIP:
4733	case VMX_VMCS64_GUEST_SYSENTER_ESP:
4734	case VMX_VMCS32_GUEST_GDTR_LIMIT:
4735	case VMX_VMCS64_GUEST_GDTR_BASE:
4736	case VMX_VMCS32_GUEST_IDTR_LIMIT:
4737	case VMX_VMCS64_GUEST_IDTR_BASE:
4738	case VMX_VMCS16_GUEST_FIELD_CS:
4739	case VMX_VMCS32_GUEST_CS_LIMIT:
4740	case VMX_VMCS64_GUEST_CS_BASE:
4741	case VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS:
4742	case VMX_VMCS16_GUEST_FIELD_DS:
4743	case VMX_VMCS32_GUEST_DS_LIMIT:
4744	case VMX_VMCS64_GUEST_DS_BASE:
4745	case VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS:
4746	case VMX_VMCS16_GUEST_FIELD_ES:
4747	case VMX_VMCS32_GUEST_ES_LIMIT:
4748	case VMX_VMCS64_GUEST_ES_BASE:
4749	case VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS:
4750	case VMX_VMCS16_GUEST_FIELD_FS:
4751	case VMX_VMCS32_GUEST_FS_LIMIT:
4752	case VMX_VMCS64_GUEST_FS_BASE:
4753	case VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS:
4754	case VMX_VMCS16_GUEST_FIELD_GS:
4755	case VMX_VMCS32_GUEST_GS_LIMIT:
4756	case VMX_VMCS64_GUEST_GS_BASE:
4757	case VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS:
4758	case VMX_VMCS16_GUEST_FIELD_SS:
4759	case VMX_VMCS32_GUEST_SS_LIMIT:
4760	case VMX_VMCS64_GUEST_SS_BASE:
4761	case VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS:
4762	case VMX_VMCS16_GUEST_FIELD_LDTR:
4763	case VMX_VMCS32_GUEST_LDTR_LIMIT:
4764	case VMX_VMCS64_GUEST_LDTR_BASE:
4765	case VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS:
4766	case VMX_VMCS16_GUEST_FIELD_TR:
4767	case VMX_VMCS32_GUEST_TR_LIMIT:
4768	case VMX_VMCS64_GUEST_TR_BASE:
4769	case VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS:
4770	case VMX_VMCS32_RO_EXIT_REASON:
4771	case VMX_VMCS32_RO_VM_INSTR_ERROR:
4772	case VMX_VMCS32_RO_EXIT_INSTR_LENGTH:
4773	case VMX_VMCS32_RO_EXIT_INTERRUPTION_ERRCODE:
4774	case VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO:
4775	case VMX_VMCS32_RO_EXIT_INSTR_INFO:
4776	case VMX_VMCS_RO_EXIT_QUALIFICATION:
4777	case VMX_VMCS32_RO_IDT_INFO:
4778	case VMX_VMCS32_RO_IDT_ERRCODE:
4779	case VMX_VMCS64_GUEST_CR3:
4780	case VMX_VMCS_EXIT_PHYS_ADDR_FULL:
4781	return true;
4782	}
4783	return false;
4784	}
4785
4786	static bool hmR0VmxIsValidWriteField(uint32_t idxField)
4787	{
4788	switch(idxField)
4789	{
4790	case VMX_VMCS64_GUEST_LDTR_BASE:
4791	case VMX_VMCS64_GUEST_TR_BASE:
4792	case VMX_VMCS64_GUEST_GDTR_BASE:
4793	case VMX_VMCS64_GUEST_IDTR_BASE:
4794	case VMX_VMCS64_GUEST_SYSENTER_EIP:
4795	case VMX_VMCS64_GUEST_SYSENTER_ESP:
4796	case VMX_VMCS64_GUEST_CR0:
4797	case VMX_VMCS64_GUEST_CR4:
4798	case VMX_VMCS64_GUEST_CR3:
4799	case VMX_VMCS64_GUEST_DR7:
4800	case VMX_VMCS64_GUEST_RIP:
4801	case VMX_VMCS64_GUEST_RSP:
4802	case VMX_VMCS64_GUEST_CS_BASE:
4803	case VMX_VMCS64_GUEST_DS_BASE:
4804	case VMX_VMCS64_GUEST_ES_BASE:
4805	case VMX_VMCS64_GUEST_FS_BASE:
4806	case VMX_VMCS64_GUEST_GS_BASE:
4807	case VMX_VMCS64_GUEST_SS_BASE:
4808	return true;
4809	}
4810	return false;
4811	}
4812
4813	# endif /* VBOX_STRICT */
4814
4815	/**
4816	* Executes the specified handler in 64 mode
4817	*
4818	* @returns VBox status code.
4819	* @param pVM The VM to operate on.
4820	* @param pVCpu The VMCPU to operate on.
4821	* @param pCtx Guest context
4822	* @param pfnHandler RC handler
4823	* @param cbParam Number of parameters
4824	* @param paParam Array of 32 bits parameters
4825	*/
4826	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, RTRCPTR pfnHandler, uint32_t cbParam, uint32_t *paParam)
4827	{
4828	int rc, rc2;
4829	PHMGLOBLCPUINFO pCpu;
4830	RTHCPHYS HCPhysCpuPage;
4831	RTHCUINTREG uOldEFlags;
4832
4833	AssertReturn(pVM->hwaccm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
4834	Assert(pfnHandler);
4835	Assert(pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hwaccm.s.vmx.VMCSCache.Write.aField));
4836	Assert(pVCpu->hwaccm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hwaccm.s.vmx.VMCSCache.Read.aField));
4837
4838	#ifdef VBOX_STRICT
4839	for (unsigned i=0;i<pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries;i++)
4840	Assert(hmR0VmxIsValidWriteField(pVCpu->hwaccm.s.vmx.VMCSCache.Write.aField[i]));
4841
4842	for (unsigned i=0;i<pVCpu->hwaccm.s.vmx.VMCSCache.Read.cValidEntries;i++)
4843	Assert(hmR0VmxIsValidReadField(pVCpu->hwaccm.s.vmx.VMCSCache.Read.aField[i]));
4844	#endif
4845
4846	/* Disable interrupts. */
4847	uOldEFlags = ASMIntDisableFlags();
4848
4849	pCpu = HWACCMR0GetCurrentCpu();
4850	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4851
4852	/* Clear VM Control Structure. Marking it inactive, clearing implementation specific data and writing back VMCS data to memory. */
4853	VMXClearVMCS(pVCpu->hwaccm.s.vmx.HCPhysVMCS);
4854
4855	/* Leave VMX Root Mode. */
4856	VMXDisable();
4857
4858	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4859
4860	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
4861	CPUMSetHyperEIP(pVCpu, pfnHandler);
4862	for (int i=(int)cbParam-1;i>=0;i--)
4863	CPUMPushHyper(pVCpu, paParam[i]);
4864
4865	STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatWorldSwitch3264, z);
4866	/* Call switcher. */
4867	rc = pVM->hwaccm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
4868	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatWorldSwitch3264, z);
4869
4870	/* Make sure the VMX instructions don't cause #UD faults. */
4871	ASMSetCR4(ASMGetCR4() \| X86_CR4_VMXE);
4872
4873	/* Enter VMX Root Mode */
4874	rc2 = VMXEnable(HCPhysCpuPage);
4875	if (RT_FAILURE(rc2))
4876	{
4877	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4878	ASMSetFlags(uOldEFlags);
4879	return VERR_VMX_VMXON_FAILED;
4880	}
4881
4882	rc2 = VMXActivateVMCS(pVCpu->hwaccm.s.vmx.HCPhysVMCS);
4883	AssertRC(rc2);
4884	Assert(!(ASMGetFlags() & X86_EFL_IF));
4885	ASMSetFlags(uOldEFlags);
4886	return rc;
4887	}
4888
4889	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
4890
4891
4892	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
4893	/**
4894	* Executes VMWRITE
4895	*
4896	* @returns VBox status code
4897	* @param pVCpu The VMCPU to operate on.
4898	* @param idxField VMCS index
4899	* @param u64Val 16, 32 or 64 bits value
4900	*/
4901	VMMR0DECL(int) VMXWriteVMCS64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4902	{
4903	int rc;
4904
4905	switch (idxField)
4906	{
4907	case VMX_VMCS_CTRL_TSC_OFFSET_FULL:
4908	case VMX_VMCS_CTRL_IO_BITMAP_A_FULL:
4909	case VMX_VMCS_CTRL_IO_BITMAP_B_FULL:
4910	case VMX_VMCS_CTRL_MSR_BITMAP_FULL:
4911	case VMX_VMCS_CTRL_VMEXIT_MSR_STORE_FULL:
4912	case VMX_VMCS_CTRL_VMEXIT_MSR_LOAD_FULL:
4913	case VMX_VMCS_CTRL_VMENTRY_MSR_LOAD_FULL:
4914	case VMX_VMCS_CTRL_VAPIC_PAGEADDR_FULL:
4915	case VMX_VMCS_CTRL_APIC_ACCESSADDR_FULL:
4916	case VMX_VMCS_GUEST_LINK_PTR_FULL:
4917	case VMX_VMCS_GUEST_PDPTR0_FULL:
4918	case VMX_VMCS_GUEST_PDPTR1_FULL:
4919	case VMX_VMCS_GUEST_PDPTR2_FULL:
4920	case VMX_VMCS_GUEST_PDPTR3_FULL:
4921	case VMX_VMCS_GUEST_DEBUGCTL_FULL:
4922	case VMX_VMCS_GUEST_EFER_FULL:
4923	case VMX_VMCS_CTRL_EPTP_FULL:
4924	/* These fields consist of two parts, which are both writable in 32 bits mode. */
4925	rc = VMXWriteVMCS32(idxField, u64Val);
4926	rc \|= VMXWriteVMCS32(idxField + 1, (uint32_t)(u64Val >> 32ULL));
4927	AssertRC(rc);
4928	return rc;
4929
4930	case VMX_VMCS64_GUEST_LDTR_BASE:
4931	case VMX_VMCS64_GUEST_TR_BASE:
4932	case VMX_VMCS64_GUEST_GDTR_BASE:
4933	case VMX_VMCS64_GUEST_IDTR_BASE:
4934	case VMX_VMCS64_GUEST_SYSENTER_EIP:
4935	case VMX_VMCS64_GUEST_SYSENTER_ESP:
4936	case VMX_VMCS64_GUEST_CR0:
4937	case VMX_VMCS64_GUEST_CR4:
4938	case VMX_VMCS64_GUEST_CR3:
4939	case VMX_VMCS64_GUEST_DR7:
4940	case VMX_VMCS64_GUEST_RIP:
4941	case VMX_VMCS64_GUEST_RSP:
4942	case VMX_VMCS64_GUEST_CS_BASE:
4943	case VMX_VMCS64_GUEST_DS_BASE:
4944	case VMX_VMCS64_GUEST_ES_BASE:
4945	case VMX_VMCS64_GUEST_FS_BASE:
4946	case VMX_VMCS64_GUEST_GS_BASE:
4947	case VMX_VMCS64_GUEST_SS_BASE:
4948	/* Queue a 64 bits value as we can't set it in 32 bits host mode. */
4949	if (u64Val >> 32ULL)
4950	rc = VMXWriteCachedVMCSEx(pVCpu, idxField, u64Val);
4951	else
4952	rc = VMXWriteVMCS32(idxField, (uint32_t)u64Val);
4953
4954	return rc;
4955
4956	default:
4957	AssertMsgFailed(("Unexpected field %x\n", idxField));
4958	return VERR_INVALID_PARAMETER;
4959	}
4960	}
4961
4962	/**
4963	* Cache VMCS writes for performance reasons (Darwin) and for running 64 bits guests on 32 bits hosts.
4964	*
4965	* @param pVCpu The VMCPU to operate on.
4966	* @param idxField VMCS field
4967	* @param u64Val Value
4968	*/
4969	VMMR0DECL(int) VMXWriteCachedVMCSEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4970	{
4971	PVMCSCACHE pCache = &pVCpu->hwaccm.s.vmx.VMCSCache;
4972
4973	AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1, ("entries=%x\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
4974
4975	/* Make sure there are no duplicates. */
4976	for (unsigned i=0;i<pCache->Write.cValidEntries;i++)
4977	{
4978	if (pCache->Write.aField[i] == idxField)
4979	{
4980	pCache->Write.aFieldVal[i] = u64Val;
4981	return VINF_SUCCESS;
4982	}
4983	}
4984
4985	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
4986	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
4987	pCache->Write.cValidEntries++;
4988	return VINF_SUCCESS;
4989	}
4990
4991	#endif /* HC_ARCH_BITS == 32 && !VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0 */
4992

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HWVMXR0.cpp@ 41138

Download in other formats: