HWVMXR0.cpp@ 19861

Last change on this file since 19861 was 19829, checked in by vboxsync, 16 years ago
updates
Property svn:eol-style set to `native` Property svn:keywords set to `Id`
File size: 168.0 KB

Line
1	/* $Id: HWVMXR0.cpp 19829 2009-05-19 14:56:03Z vboxsync $ */
2	/** @file
3	* HWACCM VMX - Host Context Ring 0.
4	*/
5
6	/*
7	* Copyright (C) 2006-2007 Sun Microsystems, Inc.
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*
17	* Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa
18	* Clara, CA 95054 USA or visit http://www.sun.com if you need
19	* additional information or have any questions.
20	*/
21
22
23	/*******************************************************************************
24	* Header Files *
25	*******************************************************************************/
26	#define LOG_GROUP LOG_GROUP_HWACCM
27	#include <VBox/hwaccm.h>
28	#include "HWACCMInternal.h"
29	#include <VBox/vm.h>
30	#include <VBox/x86.h>
31	#include <VBox/pgm.h>
32	#include <VBox/pdm.h>
33	#include <VBox/err.h>
34	#include <VBox/log.h>
35	#include <VBox/selm.h>
36	#include <VBox/iom.h>
37	#include <VBox/rem.h>
38	#include <iprt/param.h>
39	#include <iprt/assert.h>
40	#include <iprt/asm.h>
41	#include <iprt/string.h>
42	#include "HWVMXR0.h"
43
44	/*******************************************************************************
45	* Defined Constants And Macros *
46	*******************************************************************************/
47	#if defined(RT_ARCH_AMD64)
48	# define VMX_IS_64BIT_HOST_MODE() (true)
49	#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
50	# define VMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
51	#else
52	# define VMX_IS_64BIT_HOST_MODE() (false)
53	#endif
54
55	/*******************************************************************************
56	* Global Variables *
57	*******************************************************************************/
58	/* IO operation lookup arrays. */
59	static uint32_t const g_aIOSize[4] = {1, 2, 0, 4};
60	static uint32_t const g_aIOOpAnd[4] = {0xff, 0xffff, 0, 0xffffffff};
61
62	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
63	/** See HWACCMR0A.asm. */
64	extern "C" uint32_t g_fVMXIs64bitHost;
65	#endif
66
67	/*******************************************************************************
68	* Local Functions *
69	*******************************************************************************/
70	static void VMXR0ReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rc, PCPUMCTX pCtx);
71	static void vmxR0SetupTLBEPT(PVM pVM, PVMCPU pVCpu);
72	static void vmxR0SetupTLBVPID(PVM pVM, PVMCPU pVCpu);
73	static void vmxR0SetupTLBDummy(PVM pVM, PVMCPU pVCpu);
74	static void vmxR0FlushEPT(PVM pVM, PVMCPU pVCpu, VMX_FLUSH enmFlush, RTGCPHYS GCPhys);
75	static void vmxR0FlushVPID(PVM pVM, PVMCPU pVCpu, VMX_FLUSH enmFlush, RTGCPTR GCPtr);
76	static void vmxR0UpdateExceptionBitmap(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
77	#ifdef VBOX_STRICT
78	static bool vmxR0IsValidReadField(uint32_t idxField);
79	static bool vmxR0IsValidWriteField(uint32_t idxField);
80	#endif
81
82	static void VMXR0CheckError(PVM pVM, PVMCPU pVCpu, int rc)
83	{
84	if (rc == VERR_VMX_GENERIC)
85	{
86	RTCCUINTREG instrError;
87
88	VMXReadVMCS(VMX_VMCS32_RO_VM_INSTR_ERROR, &instrError);
89	pVCpu->hwaccm.s.vmx.lasterror.ulInstrError = instrError;
90	}
91	pVM->hwaccm.s.lLastError = rc;
92	}
93
94	/**
95	* Sets up and activates VT-x on the current CPU
96	*
97	* @returns VBox status code.
98	* @param pCpu CPU info struct
99	* @param pVM The VM to operate on. (can be NULL after a resume!!)
100	* @param pvPageCpu Pointer to the global cpu page
101	* @param pPageCpuPhys Physical address of the global cpu page
102	*/
103	VMMR0DECL(int) VMXR0EnableCpu(PHWACCM_CPUINFO pCpu, PVM pVM, void *pvPageCpu, RTHCPHYS pPageCpuPhys)
104	{
105	AssertReturn(pPageCpuPhys, VERR_INVALID_PARAMETER);
106	AssertReturn(pvPageCpu, VERR_INVALID_PARAMETER);
107
108	#if defined(LOG_ENABLED) && !defined(DEBUG_bird)
109	SUPR0Printf("VMXR0EnableCpu cpu %d page (%x) %x\n", pCpu->idCpu, pvPageCpu, (uint32_t)pPageCpuPhys);
110	#endif
111	if (pVM)
112	{
113	/* Set revision dword at the beginning of the VMXON structure. */
114	(uint32_t )pvPageCpu = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hwaccm.s.vmx.msr.vmx_basic_info);
115	}
116
117	/** @todo we should unmap the two pages from the virtual address space in order to prevent accidental corruption.
118	* (which can have very bad consequences!!!)
119	*/
120
121	/* Make sure the VMX instructions don't cause #UD faults. */
122	ASMSetCR4(ASMGetCR4() \| X86_CR4_VMXE);
123
124	/* Enter VMX Root Mode */
125	int rc = VMXEnable(pPageCpuPhys);
126	if (RT_FAILURE(rc))
127	{
128	if (pVM)
129	VMXR0CheckError(pVM, &pVM->aCpus[0], rc);
130	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
131	return VERR_VMX_VMXON_FAILED;
132	}
133	return VINF_SUCCESS;
134	}
135
136	/**
137	* Deactivates VT-x on the current CPU
138	*
139	* @returns VBox status code.
140	* @param pCpu CPU info struct
141	* @param pvPageCpu Pointer to the global cpu page
142	* @param pPageCpuPhys Physical address of the global cpu page
143	*/
144	VMMR0DECL(int) VMXR0DisableCpu(PHWACCM_CPUINFO pCpu, void *pvPageCpu, RTHCPHYS pPageCpuPhys)
145	{
146	AssertReturn(pPageCpuPhys, VERR_INVALID_PARAMETER);
147	AssertReturn(pvPageCpu, VERR_INVALID_PARAMETER);
148
149	/* Leave VMX Root Mode. */
150	VMXDisable();
151
152	/* And clear the X86_CR4_VMXE bit */
153	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
154
155	#if defined(LOG_ENABLED) && !defined(DEBUG_bird)
156	SUPR0Printf("VMXR0DisableCpu cpu %d\n", pCpu->idCpu);
157	#endif
158	return VINF_SUCCESS;
159	}
160
161	/**
162	* Does Ring-0 per VM VT-x init.
163	*
164	* @returns VBox status code.
165	* @param pVM The VM to operate on.
166	*/
167	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
168	{
169	int rc;
170
171	#ifdef LOG_ENABLED
172	SUPR0Printf("VMXR0InitVM %x\n", pVM);
173	#endif
174
175	pVM->hwaccm.s.vmx.pMemObjAPIC = NIL_RTR0MEMOBJ;
176
177	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
178	{
179	/* Allocate one page for the virtual APIC mmio cache. */
180	rc = RTR0MemObjAllocCont(&pVM->hwaccm.s.vmx.pMemObjAPIC, 1 << PAGE_SHIFT, true /* executable R0 mapping */);
181	AssertRC(rc);
182	if (RT_FAILURE(rc))
183	return rc;
184
185	pVM->hwaccm.s.vmx.pAPIC = (uint8_t *)RTR0MemObjAddress(pVM->hwaccm.s.vmx.pMemObjAPIC);
186	pVM->hwaccm.s.vmx.pAPICPhys = RTR0MemObjGetPagePhysAddr(pVM->hwaccm.s.vmx.pMemObjAPIC, 0);
187	ASMMemZero32(pVM->hwaccm.s.vmx.pAPIC, PAGE_SIZE);
188	}
189	else
190	{
191	pVM->hwaccm.s.vmx.pMemObjAPIC = 0;
192	pVM->hwaccm.s.vmx.pAPIC = 0;
193	pVM->hwaccm.s.vmx.pAPICPhys = 0;
194	}
195
196	/* Allocate the MSR bitmap if this feature is supported. */
197	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
198	{
199	rc = RTR0MemObjAllocCont(&pVM->hwaccm.s.vmx.pMemObjMSRBitmap, 1 << PAGE_SHIFT, true /* executable R0 mapping */);
200	AssertRC(rc);
201	if (RT_FAILURE(rc))
202	return rc;
203
204	pVM->hwaccm.s.vmx.pMSRBitmap = (uint8_t *)RTR0MemObjAddress(pVM->hwaccm.s.vmx.pMemObjMSRBitmap);
205	pVM->hwaccm.s.vmx.pMSRBitmapPhys = RTR0MemObjGetPagePhysAddr(pVM->hwaccm.s.vmx.pMemObjMSRBitmap, 0);
206	memset(pVM->hwaccm.s.vmx.pMSRBitmap, 0xff, PAGE_SIZE);
207	}
208
209	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
210	{
211	rc = RTR0MemObjAllocCont(&pVM->hwaccm.s.vmx.pMemObjScratch, 1 << PAGE_SHIFT, true /* executable R0 mapping */);
212	AssertRC(rc);
213	if (RT_FAILURE(rc))
214	return rc;
215
216	pVM->hwaccm.s.vmx.pScratch = (uint8_t *)RTR0MemObjAddress(pVM->hwaccm.s.vmx.pMemObjScratch);
217	pVM->hwaccm.s.vmx.pScratchPhys = RTR0MemObjGetPagePhysAddr(pVM->hwaccm.s.vmx.pMemObjScratch, 0);
218
219	ASMMemZero32(pVM->hwaccm.s.vmx.pScratch, PAGE_SIZE);
220	strcpy((char *)pVM->hwaccm.s.vmx.pScratch, "SCRATCH Magic");
221	(uint64_t )(pVM->hwaccm.s.vmx.pScratch + 16) = UINT64_C(0xDEADBEEFDEADBEEF);
222	}
223	#endif
224
225	/* Allocate VMCBs for all guest CPUs. */
226	for (unsigned i=0;i<pVM->cCPUs;i++)
227	{
228	PVMCPU pVCpu = &pVM->aCpus[i];
229
230	pVCpu->hwaccm.s.vmx.pMemObjVMCS = NIL_RTR0MEMOBJ;
231
232	/* Allocate one page for the VM control structure (VMCS). */
233	rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.pMemObjVMCS, 1 << PAGE_SHIFT, true /* executable R0 mapping */);
234	AssertRC(rc);
235	if (RT_FAILURE(rc))
236	return rc;
237
238	pVCpu->hwaccm.s.vmx.pVMCS = RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.pMemObjVMCS);
239	pVCpu->hwaccm.s.vmx.pVMCSPhys = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.pMemObjVMCS, 0);
240	ASMMemZero32(pVCpu->hwaccm.s.vmx.pVMCS, PAGE_SIZE);
241
242	pVCpu->hwaccm.s.vmx.cr0_mask = 0;
243	pVCpu->hwaccm.s.vmx.cr4_mask = 0;
244
245	/* Current guest paging mode. */
246	pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode = PGMMODE_REAL;
247
248	#ifdef LOG_ENABLED
249	SUPR0Printf("VMXR0InitVM %x VMCS=%x (%x)\n", pVM, pVCpu->hwaccm.s.vmx.pVMCS, (uint32_t)pVCpu->hwaccm.s.vmx.pVMCSPhys);
250	#endif
251	}
252
253	return VINF_SUCCESS;
254	}
255
256	/**
257	* Does Ring-0 per VM VT-x termination.
258	*
259	* @returns VBox status code.
260	* @param pVM The VM to operate on.
261	*/
262	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
263	{
264	for (unsigned i=0;i<pVM->cCPUs;i++)
265	{
266	if (pVM->aCpus[i].hwaccm.s.vmx.pMemObjVMCS != NIL_RTR0MEMOBJ)
267	{
268	RTR0MemObjFree(pVM->aCpus[i].hwaccm.s.vmx.pMemObjVMCS, false);
269	pVM->aCpus[i].hwaccm.s.vmx.pMemObjVMCS = NIL_RTR0MEMOBJ;
270	pVM->aCpus[i].hwaccm.s.vmx.pVMCS = 0;
271	pVM->aCpus[i].hwaccm.s.vmx.pVMCSPhys = 0;
272	}
273	}
274	if (pVM->hwaccm.s.vmx.pMemObjAPIC != NIL_RTR0MEMOBJ)
275	{
276	RTR0MemObjFree(pVM->hwaccm.s.vmx.pMemObjAPIC, false);
277	pVM->hwaccm.s.vmx.pMemObjAPIC = NIL_RTR0MEMOBJ;
278	pVM->hwaccm.s.vmx.pAPIC = 0;
279	pVM->hwaccm.s.vmx.pAPICPhys = 0;
280	}
281	if (pVM->hwaccm.s.vmx.pMemObjMSRBitmap != NIL_RTR0MEMOBJ)
282	{
283	RTR0MemObjFree(pVM->hwaccm.s.vmx.pMemObjMSRBitmap, false);
284	pVM->hwaccm.s.vmx.pMemObjMSRBitmap = NIL_RTR0MEMOBJ;
285	pVM->hwaccm.s.vmx.pMSRBitmap = 0;
286	pVM->hwaccm.s.vmx.pMSRBitmapPhys = 0;
287	}
288	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
289	if (pVM->hwaccm.s.vmx.pMemObjScratch != NIL_RTR0MEMOBJ)
290	{
291	ASMMemZero32(pVM->hwaccm.s.vmx.pScratch, PAGE_SIZE);
292	RTR0MemObjFree(pVM->hwaccm.s.vmx.pMemObjScratch, false);
293	pVM->hwaccm.s.vmx.pMemObjScratch = NIL_RTR0MEMOBJ;
294	pVM->hwaccm.s.vmx.pScratch = 0;
295	pVM->hwaccm.s.vmx.pScratchPhys = 0;
296	}
297	#endif
298	return VINF_SUCCESS;
299	}
300
301	/**
302	* Sets up VT-x for the specified VM
303	*
304	* @returns VBox status code.
305	* @param pVM The VM to operate on.
306	*/
307	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
308	{
309	int rc = VINF_SUCCESS;
310	uint32_t val;
311
312	AssertReturn(pVM, VERR_INVALID_PARAMETER);
313
314	for (unsigned i=0;i<pVM->cCPUs;i++)
315	{
316	PVMCPU pVCpu = &pVM->aCpus[i];
317
318	Assert(pVCpu->hwaccm.s.vmx.pVMCS);
319
320	/* Set revision dword at the beginning of the VMCS structure. */
321	(uint32_t )pVCpu->hwaccm.s.vmx.pVMCS = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hwaccm.s.vmx.msr.vmx_basic_info);
322
323	/* Clear VM Control Structure. */
324	Log(("pVMCSPhys = %RHp\n", pVCpu->hwaccm.s.vmx.pVMCSPhys));
325	rc = VMXClearVMCS(pVCpu->hwaccm.s.vmx.pVMCSPhys);
326	if (RT_FAILURE(rc))
327	goto vmx_end;
328
329	/* Activate the VM Control Structure. */
330	rc = VMXActivateVMCS(pVCpu->hwaccm.s.vmx.pVMCSPhys);
331	if (RT_FAILURE(rc))
332	goto vmx_end;
333
334	/* VMX_VMCS_CTRL_PIN_EXEC_CONTROLS
335	* Set required bits to one and zero according to the MSR capabilities.
336	*/
337	val = pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.disallowed0;
338	/* External and non-maskable interrupts cause VM-exits. */
339	val = val \| VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_EXT_INT_EXIT \| VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_NMI_EXIT;
340	val &= pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.allowed1;
341
342	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PIN_EXEC_CONTROLS, val);
343	AssertRC(rc);
344
345	/* VMX_VMCS_CTRL_PROC_EXEC_CONTROLS
346	* Set required bits to one and zero according to the MSR capabilities.
347	*/
348	val = pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.disallowed0;
349	/* Program which event cause VM-exits and which features we want to use. */
350	val = val \| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_HLT_EXIT
351	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_TSC_OFFSET
352	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT
353	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_UNCOND_IO_EXIT
354	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDPMC_EXIT
355	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MWAIT_EXIT; /* don't execute mwait or else we'll idle inside the guest (host thinks the cpu load is high) */
356
357	/* Without nested paging we should intercept invlpg and cr3 mov instructions. */
358	if (!pVM->hwaccm.s.fNestedPaging)
359	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT
360	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
361	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
362
363	/* Note: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MWAIT_EXIT might cause a vmlaunch failure with an invalid control fields error. (combined with some other exit reasons) */
364	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
365	{
366	/* CR8 reads from the APIC shadow page; writes cause an exit is they lower the TPR below the threshold */
367	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW;
368	Assert(pVM->hwaccm.s.vmx.pAPIC);
369	}
370	else
371	/* Exit on CR8 reads & writes in case the TPR shadow feature isn't present. */
372	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_STORE_EXIT \| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_LOAD_EXIT;
373
374	#ifdef VBOX_WITH_VTX_MSR_BITMAPS
375	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
376	{
377	Assert(pVM->hwaccm.s.vmx.pMSRBitmapPhys);
378	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS;
379	}
380	#endif
381
382	/* We will use the secondary control if it's present. */
383	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
384
385	/* Mask away the bits that the CPU doesn't support */
386	/** @todo make sure they don't conflict with the above requirements. */
387	val &= pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1;
388	pVCpu->hwaccm.s.vmx.proc_ctls = val;
389
390	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, val);
391	AssertRC(rc);
392
393	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
394	{
395	/* VMX_VMCS_CTRL_PROC_EXEC_CONTROLS2
396	* Set required bits to one and zero according to the MSR capabilities.
397	*/
398	val = pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0;
399	val \|= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT;
400
401	#ifdef HWACCM_VTX_WITH_EPT
402	if (pVM->hwaccm.s.fNestedPaging)
403	val \|= VMX_VMCS_CTRL_PROC_EXEC2_EPT;
404	#endif /* HWACCM_VTX_WITH_EPT */
405	#ifdef HWACCM_VTX_WITH_VPID
406	else
407	if (pVM->hwaccm.s.vmx.fVPID)
408	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VPID;
409	#endif /* HWACCM_VTX_WITH_VPID */
410
411	/* Mask away the bits that the CPU doesn't support */
412	/** @todo make sure they don't conflict with the above requirements. */
413	val &= pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1;
414
415	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS2, val);
416	AssertRC(rc);
417	}
418
419	/* VMX_VMCS_CTRL_CR3_TARGET_COUNT
420	* Set required bits to one and zero according to the MSR capabilities.
421	*/
422	rc = VMXWriteVMCS(VMX_VMCS_CTRL_CR3_TARGET_COUNT, 0);
423	AssertRC(rc);
424
425	/* Forward all exception except #NM & #PF to the guest.
426	* We always need to check pagefaults since our shadow page table can be out of sync.
427	* And we always lazily sync the FPU & XMM state.
428	*/
429
430	/** @todo Possible optimization:
431	* Keep the FPU and XMM state current in the EM thread. That way there's no need to
432	* lazily sync anything, but the downside is that we can't use the FPU stack or XMM
433	* registers ourselves of course.
434	*
435	* Note: only possible if the current state is actually ours (X86_CR0_TS flag)
436	*/
437
438	/* Don't filter page faults; all of them should cause a switch. */
439	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PAGEFAULT_ERROR_MASK, 0);
440	rc \|= VMXWriteVMCS(VMX_VMCS_CTRL_PAGEFAULT_ERROR_MATCH, 0);
441	AssertRC(rc);
442
443	/* Init TSC offset to zero. */
444	rc = VMXWriteVMCS64(VMX_VMCS_CTRL_TSC_OFFSET_FULL, 0);
445	AssertRC(rc);
446
447	rc = VMXWriteVMCS64(VMX_VMCS_CTRL_IO_BITMAP_A_FULL, 0);
448	AssertRC(rc);
449
450	rc = VMXWriteVMCS64(VMX_VMCS_CTRL_IO_BITMAP_B_FULL, 0);
451	AssertRC(rc);
452
453	/* Set the MSR bitmap address. */
454	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
455	{
456	/* Optional */
457	rc = VMXWriteVMCS64(VMX_VMCS_CTRL_MSR_BITMAP_FULL, pVM->hwaccm.s.vmx.pMSRBitmapPhys);
458	AssertRC(rc);
459	}
460
461	/* Clear MSR controls. */
462	rc = VMXWriteVMCS64(VMX_VMCS_CTRL_VMEXIT_MSR_STORE_FULL, 0);
463	rc \|= VMXWriteVMCS64(VMX_VMCS_CTRL_VMEXIT_MSR_LOAD_FULL, 0);
464	rc \|= VMXWriteVMCS64(VMX_VMCS_CTRL_VMENTRY_MSR_LOAD_FULL, 0);
465	rc \|= VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_MSR_STORE_COUNT, 0);
466	rc \|= VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_MSR_LOAD_COUNT, 0);
467	AssertRC(rc);
468
469	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
470	{
471	Assert(pVM->hwaccm.s.vmx.pMemObjAPIC);
472	/* Optional */
473	rc = VMXWriteVMCS(VMX_VMCS_CTRL_TPR_THRESHOLD, 0);
474	rc \|= VMXWriteVMCS64(VMX_VMCS_CTRL_VAPIC_PAGEADDR_FULL, pVM->hwaccm.s.vmx.pAPICPhys);
475	AssertRC(rc);
476	}
477
478	/* Set link pointer to -1. Not currently used. */
479	rc = VMXWriteVMCS64(VMX_VMCS_GUEST_LINK_PTR_FULL, 0xFFFFFFFFFFFFFFFFULL);
480	AssertRC(rc);
481
482	/* Clear VM Control Structure. Marking it inactive, clearing implementation specific data and writing back VMCS data to memory. */
483	rc = VMXClearVMCS(pVCpu->hwaccm.s.vmx.pVMCSPhys);
484	AssertRC(rc);
485
486	/* Configure the VMCS read cache. */
487	PVMCSCACHE pCache = &pVCpu->hwaccm.s.vmx.VMCSCache;
488
489	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_RIP);
490	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_RSP);
491	VMXSetupCachedReadVMCS(pCache, VMX_VMCS_GUEST_RFLAGS);
492	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE);
493	VMXSetupCachedReadVMCS(pCache, VMX_VMCS_CTRL_CR0_READ_SHADOW);
494	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_CR0);
495	VMXSetupCachedReadVMCS(pCache, VMX_VMCS_CTRL_CR4_READ_SHADOW);
496	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_CR4);
497	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_DR7);
498	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_SYSENTER_CS);
499	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_SYSENTER_EIP);
500	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_SYSENTER_ESP);
501	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_GDTR_LIMIT);
502	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_GDTR_BASE);
503	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_IDTR_LIMIT);
504	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_IDTR_BASE);
505
506	VMX_SETUP_SELREG(ES, pCache);
507	VMX_SETUP_SELREG(SS, pCache);
508	VMX_SETUP_SELREG(CS, pCache);
509	VMX_SETUP_SELREG(DS, pCache);
510	VMX_SETUP_SELREG(FS, pCache);
511	VMX_SETUP_SELREG(GS, pCache);
512	VMX_SETUP_SELREG(LDTR, pCache);
513	VMX_SETUP_SELREG(TR, pCache);
514
515	/* Status code VMCS reads. */
516	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_REASON);
517	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_VM_INSTR_ERROR);
518	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INSTR_LENGTH);
519	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INTERRUPTION_ERRCODE);
520	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO);
521	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INSTR_INFO);
522	VMXSetupCachedReadVMCS(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
523	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_IDT_INFO);
524	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_IDT_ERRCODE);
525
526	if (pVM->hwaccm.s.fNestedPaging)
527	{
528	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_CR3);
529	VMXSetupCachedReadVMCS(pCache, VMX_VMCS_EXIT_PHYS_ADDR_FULL);
530	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
531	}
532	else
533	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
534	} /* for each VMCPU */
535
536	/* Choose the right TLB setup function. */
537	if (pVM->hwaccm.s.fNestedPaging)
538	{
539	pVM->hwaccm.s.vmx.pfnSetupTaggedTLB = vmxR0SetupTLBEPT;
540
541	/* Default values for flushing. */
542	pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_ALL_CONTEXTS;
543	pVM->hwaccm.s.vmx.enmFlushContext = VMX_FLUSH_ALL_CONTEXTS;
544
545	/* If the capabilities specify we can do more, then make use of it. */
546	if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVEPT_CAPS_INDIV)
547	pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_PAGE;
548	else
549	if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVEPT_CAPS_CONTEXT)
550	pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_SINGLE_CONTEXT;
551
552	if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVEPT_CAPS_CONTEXT)
553	pVM->hwaccm.s.vmx.enmFlushContext = VMX_FLUSH_SINGLE_CONTEXT;
554	}
555	#ifdef HWACCM_VTX_WITH_VPID
556	else
557	if (pVM->hwaccm.s.vmx.fVPID)
558	{
559	pVM->hwaccm.s.vmx.pfnSetupTaggedTLB = vmxR0SetupTLBVPID;
560
561	/* Default values for flushing. */
562	pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_ALL_CONTEXTS;
563	pVM->hwaccm.s.vmx.enmFlushContext = VMX_FLUSH_ALL_CONTEXTS;
564
565	/* If the capabilities specify we can do more, then make use of it. */
566	if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_INDIV)
567	pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_PAGE;
568	else
569	if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_CONTEXT)
570	pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_SINGLE_CONTEXT;
571
572	if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_CONTEXT)
573	pVM->hwaccm.s.vmx.enmFlushContext = VMX_FLUSH_SINGLE_CONTEXT;
574	}
575	#endif /* HWACCM_VTX_WITH_VPID */
576	else
577	pVM->hwaccm.s.vmx.pfnSetupTaggedTLB = vmxR0SetupTLBDummy;
578
579	vmx_end:
580	VMXR0CheckError(pVM, &pVM->aCpus[0], rc);
581	return rc;
582	}
583
584
585	/**
586	* Injects an event (trap or external interrupt)
587	*
588	* @returns VBox status code.
589	* @param pVM The VM to operate on.
590	* @param pVCpu The VMCPU to operate on.
591	* @param pCtx CPU Context
592	* @param intInfo VMX interrupt info
593	* @param cbInstr Opcode length of faulting instruction
594	* @param errCode Error code (optional)
595	*/
596	static int VMXR0InjectEvent(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, uint32_t intInfo, uint32_t cbInstr, uint32_t errCode)
597	{
598	int rc;
599	uint32_t iGate = VMX_EXIT_INTERRUPTION_INFO_VECTOR(intInfo);
600
601	#ifdef VBOX_STRICT
602	if (iGate == 0xE)
603	LogFlow(("VMXR0InjectEvent: Injecting interrupt %d at %RGv error code=%08x CR2=%RGv intInfo=%08x\n", iGate, (RTGCPTR)pCtx->rip, errCode, pCtx->cr2, intInfo));
604	else
605	if (iGate < 0x20)
606	LogFlow(("VMXR0InjectEvent: Injecting interrupt %d at %RGv error code=%08x\n", iGate, (RTGCPTR)pCtx->rip, errCode));
607	else
608	{
609	LogFlow(("INJ-EI: %x at %RGv\n", iGate, (RTGCPTR)pCtx->rip));
610	Assert(VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW \|\| !VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
611	Assert(VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW \|\| pCtx->eflags.u32 & X86_EFL_IF);
612	}
613	#endif
614
615	#ifdef HWACCM_VMX_EMULATE_REALMODE
616	if (CPUMIsGuestInRealModeEx(pCtx))
617	{
618	RTGCPHYS GCPhysHandler;
619	uint16_t offset, ip;
620	RTSEL sel;
621
622	/* Injecting events doesn't work right with real mode emulation.
623	* (#GP if we try to inject external hardware interrupts)
624	* Inject the interrupt or trap directly instead.
625	*
626	* ASSUMES no access handlers for the bits we read or write below (should be safe).
627	*/
628	Log(("Manual interrupt/trap '%x' inject (real mode)\n", iGate));
629
630	/* Check if the interrupt handler is present. */
631	if (iGate * 4 + 3 > pCtx->idtr.cbIdt)
632	{
633	Log(("IDT cbIdt violation\n"));
634	if (iGate != X86_XCPT_DF)
635	{
636	RTGCUINTPTR intInfo;
637
638	intInfo = (iGate == X86_XCPT_GP) ? (uint32_t)X86_XCPT_DF : iGate;
639	intInfo \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
640	intInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
641	intInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
642
643	return VMXR0InjectEvent(pVM, pVCpu, pCtx, intInfo, 0, 0 /* no error code according to the Intel docs */);
644	}
645	Log(("Triple fault -> reset the VM!\n"));
646	return VINF_EM_RESET;
647	}
648	if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW
649	\|\| iGate == 3 /* Both #BP and #OF point to the instruction after. */
650	\|\| iGate == 4)
651	{
652	ip = pCtx->ip + cbInstr;
653	}
654	else
655	ip = pCtx->ip;
656
657	/* Read the selector:offset pair of the interrupt handler. */
658	GCPhysHandler = (RTGCPHYS)pCtx->idtr.pIdt + iGate * 4;
659	rc = PGMPhysSimpleReadGCPhys(pVM, &offset, GCPhysHandler, sizeof(offset)); AssertRC(rc);
660	rc = PGMPhysSimpleReadGCPhys(pVM, &sel, GCPhysHandler + 2, sizeof(sel)); AssertRC(rc);
661
662	LogFlow(("IDT handler %04X:%04X\n", sel, offset));
663
664	/* Construct the stack frame. */
665	/** @todo should check stack limit. */
666	pCtx->sp -= 2;
667	LogFlow(("ss:sp %04X:%04X eflags=%x\n", pCtx->ss, pCtx->sp, pCtx->eflags.u));
668	rc = PGMPhysSimpleWriteGCPhys(pVM, pCtx->ssHid.u64Base + pCtx->sp, &pCtx->eflags, sizeof(uint16_t)); AssertRC(rc);
669	pCtx->sp -= 2;
670	LogFlow(("ss:sp %04X:%04X cs=%x\n", pCtx->ss, pCtx->sp, pCtx->cs));
671	rc = PGMPhysSimpleWriteGCPhys(pVM, pCtx->ssHid.u64Base + pCtx->sp, &pCtx->cs, sizeof(uint16_t)); AssertRC(rc);
672	pCtx->sp -= 2;
673	LogFlow(("ss:sp %04X:%04X ip=%x\n", pCtx->ss, pCtx->sp, ip));
674	rc = PGMPhysSimpleWriteGCPhys(pVM, pCtx->ssHid.u64Base + pCtx->sp, &ip, sizeof(ip)); AssertRC(rc);
675
676	/* Update the CPU state for executing the handler. */
677	pCtx->rip = offset;
678	pCtx->cs = sel;
679	pCtx->csHid.u64Base = sel << 4;
680	pCtx->eflags.u &= ~(X86_EFL_IF\|X86_EFL_TF\|X86_EFL_RF\|X86_EFL_AC);
681
682	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_SEGMENT_REGS;
683	return VINF_SUCCESS;
684	}
685	#endif /* HWACCM_VMX_EMULATE_REALMODE */
686
687	/* Set event injection state. */
688	rc = VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_IRQ_INFO, intInfo \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT));
689
690	rc \|= VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
691	rc \|= VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_EXCEPTION_ERRCODE, errCode);
692
693	AssertRC(rc);
694	return rc;
695	}
696
697
698	/**
699	* Checks for pending guest interrupts and injects them
700	*
701	* @returns VBox status code.
702	* @param pVM The VM to operate on.
703	* @param pVCpu The VMCPU to operate on.
704	* @param pCtx CPU Context
705	*/
706	static int VMXR0CheckPendingInterrupt(PVM pVM, PVMCPU pVCpu, CPUMCTX *pCtx)
707	{
708	int rc;
709
710	/* Dispatch any pending interrupts. (injected before, but a VM exit occurred prematurely) */
711	if (pVCpu->hwaccm.s.Event.fPending)
712	{
713	Log(("CPU%d: Reinjecting event %RX64 %08x at %RGv cr2=%RX64\n", pVCpu->idCpu, pVCpu->hwaccm.s.Event.intInfo, pVCpu->hwaccm.s.Event.errCode, (RTGCPTR)pCtx->rip, pCtx->cr2));
714	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatIntReinject);
715	rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, pVCpu->hwaccm.s.Event.intInfo, 0, pVCpu->hwaccm.s.Event.errCode);
716	AssertRC(rc);
717
718	pVCpu->hwaccm.s.Event.fPending = false;
719	return VINF_SUCCESS;
720	}
721
722	if (pVM->hwaccm.s.fInjectNMI)
723	{
724	RTGCUINTPTR intInfo;
725
726	intInfo = X86_XCPT_NMI;
727	intInfo \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
728	intInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
729
730	rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, intInfo, 0, 0);
731	AssertRC(rc);
732
733	pVM->hwaccm.s.fInjectNMI = false;
734	return VINF_SUCCESS;
735	}
736
737	/* When external interrupts are pending, we should exit the VM when IF is set. */
738	if ( !TRPMHasTrap(pVCpu)
739	&& VMCPU_FF_ISPENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC\|VMCPU_FF_INTERRUPT_PIC)))
740	{
741	if (!(pCtx->eflags.u32 & X86_EFL_IF))
742	{
743	if (!(pVCpu->hwaccm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_IRQ_WINDOW_EXIT))
744	{
745	LogFlow(("Enable irq window exit!\n"));
746	pVCpu->hwaccm.s.vmx.proc_ctls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_IRQ_WINDOW_EXIT;
747	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
748	AssertRC(rc);
749	}
750	/* else nothing to do but wait */
751	}
752	else
753	if (!VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
754	{
755	uint8_t u8Interrupt;
756
757	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
758	Log(("CPU%d: Dispatch interrupt: u8Interrupt=%x (%d) rc=%Rrc cs:rip=%04X:%RGv\n", pVCpu->idCpu, u8Interrupt, u8Interrupt, rc, pCtx->cs, (RTGCPTR)pCtx->rip));
759	if (RT_SUCCESS(rc))
760	{
761	rc = TRPMAssertTrap(pVCpu, u8Interrupt, TRPM_HARDWARE_INT);
762	AssertRC(rc);
763	}
764	else
765	{
766	/* Can only happen in rare cases where a pending interrupt is cleared behind our back */
767	Assert(!VMCPU_FF_ISPENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC\|VMCPU_FF_INTERRUPT_PIC)));
768	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatSwitchGuestIrq);
769	/* Just continue */
770	}
771	}
772	else
773	Log(("Pending interrupt blocked at %RGv by VM_FF_INHIBIT_INTERRUPTS!!\n", (RTGCPTR)pCtx->rip));
774	}
775
776	#ifdef VBOX_STRICT
777	if (TRPMHasTrap(pVCpu))
778	{
779	uint8_t u8Vector;
780	rc = TRPMQueryTrapAll(pVCpu, &u8Vector, 0, 0, 0);
781	AssertRC(rc);
782	}
783	#endif
784
785	if ( (pCtx->eflags.u32 & X86_EFL_IF)
786	&& (!VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
787	&& TRPMHasTrap(pVCpu)
788	)
789	{
790	uint8_t u8Vector;
791	int rc;
792	TRPMEVENT enmType;
793	RTGCUINTPTR intInfo;
794	RTGCUINT errCode;
795
796	/* If a new event is pending, then dispatch it now. */
797	rc = TRPMQueryTrapAll(pVCpu, &u8Vector, &enmType, &errCode, 0);
798	AssertRC(rc);
799	Assert(pCtx->eflags.Bits.u1IF == 1 \|\| enmType == TRPM_TRAP);
800	Assert(enmType != TRPM_SOFTWARE_INT);
801
802	/* Clear the pending trap. */
803	rc = TRPMResetTrap(pVCpu);
804	AssertRC(rc);
805
806	intInfo = u8Vector;
807	intInfo \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
808
809	if (enmType == TRPM_TRAP)
810	{
811	switch (u8Vector) {
812	case 8:
813	case 10:
814	case 11:
815	case 12:
816	case 13:
817	case 14:
818	case 17:
819	/* Valid error codes. */
820	intInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
821	break;
822	default:
823	break;
824	}
825	if (u8Vector == X86_XCPT_BP \|\| u8Vector == X86_XCPT_OF)
826	intInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
827	else
828	intInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
829	}
830	else
831	intInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
832
833	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatIntInject);
834	rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, intInfo, 0, errCode);
835	AssertRC(rc);
836	} /* if (interrupts can be dispatched) */
837
838	return VINF_SUCCESS;
839	}
840
841	/**
842	* Save the host state
843	*
844	* @returns VBox status code.
845	* @param pVM The VM to operate on.
846	* @param pVCpu The VMCPU to operate on.
847	*/
848	VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
849	{
850	int rc = VINF_SUCCESS;
851
852	/*
853	* Host CPU Context
854	*/
855	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_HOST_CONTEXT)
856	{
857	RTIDTR idtr;
858	RTGDTR gdtr;
859	RTSEL SelTR;
860	PX86DESCHC pDesc;
861	uintptr_t trBase;
862	RTSEL cs;
863	RTSEL ss;
864	uint64_t cr3;
865
866	/* Control registers */
867	rc = VMXWriteVMCS(VMX_VMCS_HOST_CR0, ASMGetCR0());
868	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
869	if (VMX_IS_64BIT_HOST_MODE())
870	{
871	cr3 = hwaccmR0Get64bitCR3();
872	rc \|= VMXWriteVMCS64(VMX_VMCS_HOST_CR3, cr3);
873	}
874	else
875	#endif
876	{
877	cr3 = ASMGetCR3();
878	rc \|= VMXWriteVMCS(VMX_VMCS_HOST_CR3, cr3);
879	}
880	rc \|= VMXWriteVMCS(VMX_VMCS_HOST_CR4, ASMGetCR4());
881	AssertRC(rc);
882	Log2(("VMX_VMCS_HOST_CR0 %08x\n", ASMGetCR0()));
883	Log2(("VMX_VMCS_HOST_CR3 %08RX64\n", cr3));
884	Log2(("VMX_VMCS_HOST_CR4 %08x\n", ASMGetCR4()));
885
886	/* Selector registers. */
887	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
888	if (VMX_IS_64BIT_HOST_MODE())
889	{
890	cs = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
891	ss = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
892	}
893	else
894	{
895	/* sysenter loads LDT cs & ss, VMX doesn't like this. Load the GDT ones (safe). */
896	cs = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
897	ss = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
898	}
899	#else
900	cs = ASMGetCS();
901	ss = ASMGetSS();
902	#endif
903	Assert(!(cs & X86_SEL_LDT)); Assert((cs & X86_SEL_RPL) == 0);
904	Assert(!(ss & X86_SEL_LDT)); Assert((ss & X86_SEL_RPL) == 0);
905	rc = VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_CS, cs);
906	/* Note: VMX is (again) very picky about the RPL of the selectors here; we'll restore them manually. */
907	rc \|= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_DS, 0);
908	rc \|= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_ES, 0);
909	#if HC_ARCH_BITS == 32
910	if (!VMX_IS_64BIT_HOST_MODE())
911	{
912	rc \|= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_FS, 0);
913	rc \|= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_GS, 0);
914	}
915	#endif
916	rc \|= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_SS, ss);
917	SelTR = ASMGetTR();
918	rc \|= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_TR, SelTR);
919	AssertRC(rc);
920	Log2(("VMX_VMCS_HOST_FIELD_CS %08x (%08x)\n", cs, ASMGetSS()));
921	Log2(("VMX_VMCS_HOST_FIELD_DS 00000000 (%08x)\n", ASMGetDS()));
922	Log2(("VMX_VMCS_HOST_FIELD_ES 00000000 (%08x)\n", ASMGetES()));
923	Log2(("VMX_VMCS_HOST_FIELD_FS 00000000 (%08x)\n", ASMGetFS()));
924	Log2(("VMX_VMCS_HOST_FIELD_GS 00000000 (%08x)\n", ASMGetGS()));
925	Log2(("VMX_VMCS_HOST_FIELD_SS %08x (%08x)\n", ss, ASMGetSS()));
926	Log2(("VMX_VMCS_HOST_FIELD_TR %08x\n", ASMGetTR()));
927
928	/* GDTR & IDTR */
929	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
930	if (VMX_IS_64BIT_HOST_MODE())
931	{
932	X86XDTR64 gdtr64, idtr64;
933	hwaccmR0Get64bitGDTRandIDTR(&gdtr64, &idtr64);
934	rc = VMXWriteVMCS64(VMX_VMCS_HOST_GDTR_BASE, gdtr64.uAddr);
935	rc \|= VMXWriteVMCS64(VMX_VMCS_HOST_IDTR_BASE, gdtr64.uAddr);
936	AssertRC(rc);
937	Log2(("VMX_VMCS_HOST_GDTR_BASE %RX64\n", gdtr64.uAddr));
938	Log2(("VMX_VMCS_HOST_IDTR_BASE %RX64\n", idtr64.uAddr));
939	gdtr.cbGdt = gdtr64.cb;
940	gdtr.pGdt = (uintptr_t)gdtr64.uAddr;
941	}
942	else
943	#endif
944	{
945	ASMGetGDTR(&gdtr);
946	rc = VMXWriteVMCS(VMX_VMCS_HOST_GDTR_BASE, gdtr.pGdt);
947	ASMGetIDTR(&idtr);
948	rc \|= VMXWriteVMCS(VMX_VMCS_HOST_IDTR_BASE, idtr.pIdt);
949	AssertRC(rc);
950	Log2(("VMX_VMCS_HOST_GDTR_BASE %RHv\n", gdtr.pGdt));
951	Log2(("VMX_VMCS_HOST_IDTR_BASE %RHv\n", idtr.pIdt));
952	}
953
954
955	/* Save the base address of the TR selector. */
956	if (SelTR > gdtr.cbGdt)
957	{
958	AssertMsgFailed(("Invalid TR selector %x. GDTR.cbGdt=%x\n", SelTR, gdtr.cbGdt));
959	return VERR_VMX_INVALID_HOST_STATE;
960	}
961
962	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
963	if (VMX_IS_64BIT_HOST_MODE())
964	{
965	pDesc = &((PX86DESCHC)gdtr.pGdt)[SelTR >> X86_SEL_SHIFT_HC]; /// ????
966	uint64_t trBase64 = X86DESC64_BASE(*(PX86DESC64)pDesc);
967	rc = VMXWriteVMCS64(VMX_VMCS_HOST_TR_BASE, trBase64);
968	Log2(("VMX_VMCS_HOST_TR_BASE %RX64\n", trBase64));
969	AssertRC(rc);
970	}
971	else
972	#endif
973	{
974	pDesc = &((PX86DESCHC)gdtr.pGdt)[SelTR >> X86_SEL_SHIFT_HC];
975	#if HC_ARCH_BITS == 64
976	trBase = X86DESC64_BASE(*pDesc);
977	#else
978	trBase = X86DESC_BASE(*pDesc);
979	#endif
980	rc = VMXWriteVMCS(VMX_VMCS_HOST_TR_BASE, trBase);
981	AssertRC(rc);
982	Log2(("VMX_VMCS_HOST_TR_BASE %RHv\n", trBase));
983	}
984
985	/* FS and GS base. */
986	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
987	if (VMX_IS_64BIT_HOST_MODE())
988	{
989	Log2(("MSR_K8_FS_BASE = %RX64\n", ASMRdMsr(MSR_K8_FS_BASE)));
990	Log2(("MSR_K8_GS_BASE = %RX64\n", ASMRdMsr(MSR_K8_GS_BASE)));
991	rc = VMXWriteVMCS64(VMX_VMCS_HOST_FS_BASE, ASMRdMsr(MSR_K8_FS_BASE));
992	rc \|= VMXWriteVMCS64(VMX_VMCS_HOST_GS_BASE, ASMRdMsr(MSR_K8_GS_BASE));
993	}
994	#endif
995	AssertRC(rc);
996
997	/* Sysenter MSRs. */
998	/** @todo expensive!! */
999	rc = VMXWriteVMCS(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
1000	Log2(("VMX_VMCS_HOST_SYSENTER_CS %08x\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_CS)));
1001	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1002	if (VMX_IS_64BIT_HOST_MODE())
1003	{
1004	Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_EIP)));
1005	Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_ESP)));
1006	rc \|= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
1007	rc \|= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
1008	}
1009	else
1010	{
1011	rc \|= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
1012	rc \|= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
1013	Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP)));
1014	Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP)));
1015	}
1016	#elif HC_ARCH_BITS == 32
1017	rc \|= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
1018	rc \|= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
1019	Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP)));
1020	Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP)));
1021	#else
1022	Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_EIP)));
1023	Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_ESP)));
1024	rc \|= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
1025	rc \|= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
1026	#endif
1027	AssertRC(rc);
1028
1029	#if 0 /* @todo deal with 32/64 */
1030	/* Restore the host EFER - on CPUs that support it. */
1031	if (pVM->hwaccm.s.vmx.msr.vmx_exit.n.allowed1 & VMX_VMCS_CTRL_EXIT_CONTROLS_LOAD_HOST_EFER_MSR)
1032	{
1033	uint64_t msrEFER = ASMRdMsr(MSR_IA32_EFER);
1034	rc = VMXWriteVMCS64(VMX_VMCS_HOST_FIELD_EFER_FULL, msrEFER);
1035	AssertRC(rc);
1036	}
1037	#endif
1038	pVCpu->hwaccm.s.fContextUseFlags &= ~HWACCM_CHANGED_HOST_CONTEXT;
1039	}
1040	return rc;
1041	}
1042
1043	/**
1044	* Prefetch the 4 PDPT pointers (PAE and nested paging only)
1045	*
1046	* @param pVM The VM to operate on.
1047	* @param pVCpu The VMCPU to operate on.
1048	* @param pCtx Guest context
1049	*/
1050	static void vmxR0PrefetchPAEPdptrs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1051	{
1052	if (CPUMIsGuestInPAEModeEx(pCtx))
1053	{
1054	X86PDPE Pdpe;
1055
1056	for (unsigned i=0;i<4;i++)
1057	{
1058	Pdpe = PGMGstGetPaePDPtr(pVCpu, i);
1059	int rc = VMXWriteVMCS64(VMX_VMCS_GUEST_PDPTR0_FULL + i*2, Pdpe.u);
1060	AssertRC(rc);
1061	}
1062	}
1063	}
1064
1065	/**
1066	* Update the exception bitmap according to the current CPU state
1067	*
1068	* @param pVM The VM to operate on.
1069	* @param pVCpu The VMCPU to operate on.
1070	* @param pCtx Guest context
1071	*/
1072	static void vmxR0UpdateExceptionBitmap(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1073	{
1074	uint32_t u32TrapMask;
1075	Assert(pCtx);
1076
1077	u32TrapMask = HWACCM_VMX_TRAP_MASK;
1078	#ifndef DEBUG
1079	if (pVM->hwaccm.s.fNestedPaging)
1080	u32TrapMask &= ~RT_BIT(X86_XCPT_PF); /* no longer need to intercept #PF. */
1081	#endif
1082
1083	/* Also catch floating point exceptions as we need to report them to the guest in a different way. */
1084	if ( CPUMIsGuestFPUStateActive(pVCpu) == true
1085	&& !(pCtx->cr0 & X86_CR0_NE)
1086	&& !pVCpu->hwaccm.s.fFPUOldStyleOverride)
1087	{
1088	u32TrapMask \|= RT_BIT(X86_XCPT_MF);
1089	pVCpu->hwaccm.s.fFPUOldStyleOverride = true;
1090	}
1091
1092	#ifdef DEBUG /* till after branching, enable it by default then. */
1093	/* Intercept X86_XCPT_DB if stepping is enabled */
1094	if (DBGFIsStepping(pVCpu))
1095	u32TrapMask \|= RT_BIT(X86_XCPT_DB);
1096	/** @todo Don't trap it unless the debugger has armed breakpoints. */
1097	u32TrapMask \|= RT_BIT(X86_XCPT_BP);
1098	#endif
1099
1100	#ifdef VBOX_STRICT
1101	Assert(u32TrapMask & RT_BIT(X86_XCPT_GP));
1102	#endif
1103
1104	# ifdef HWACCM_VMX_EMULATE_REALMODE
1105	/* Intercept all exceptions in real mode as none of them can be injected directly (#GP otherwise). */
1106	if (CPUMIsGuestInRealModeEx(pCtx) && pVM->hwaccm.s.vmx.pRealModeTSS)
1107	u32TrapMask \|= HWACCM_VMX_TRAP_MASK_REALMODE;
1108	# endif /* HWACCM_VMX_EMULATE_REALMODE */
1109
1110	int rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXCEPTION_BITMAP, u32TrapMask);
1111	AssertRC(rc);
1112	}
1113
1114	/**
1115	* Loads the guest state
1116	*
1117	* NOTE: Don't do anything here that can cause a jump back to ring 3!!!!!
1118	*
1119	* @returns VBox status code.
1120	* @param pVM The VM to operate on.
1121	* @param pVCpu The VMCPU to operate on.
1122	* @param pCtx Guest context
1123	*/
1124	VMMR0DECL(int) VMXR0LoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1125	{
1126	int rc = VINF_SUCCESS;
1127	RTGCUINTPTR val;
1128	X86EFLAGS eflags;
1129
1130	/* VMX_VMCS_CTRL_ENTRY_CONTROLS
1131	* Set required bits to one and zero according to the MSR capabilities.
1132	*/
1133	val = pVM->hwaccm.s.vmx.msr.vmx_entry.n.disallowed0;
1134	/* Load guest debug controls (dr7 & IA32_DEBUGCTL_MSR) (forced to 1 on the 'first' VT-x capable CPUs; this actually includes the newest Nehalem CPUs) */
1135	val \|= VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_DEBUG;
1136	#if 0 /* @todo deal with 32/64 */
1137	/* Required for the EFER write below, not supported on all CPUs. */
1138	val \|= VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_GUEST_EFER_MSR;
1139	#endif
1140	/* 64 bits guest mode? */
1141	if (pCtx->msrEFER & MSR_K6_EFER_LMA)
1142	val \|= VMX_VMCS_CTRL_ENTRY_CONTROLS_IA64_MODE;
1143	/* else Must be zero when AMD64 is not available. */
1144
1145	/* Mask away the bits that the CPU doesn't support */
1146	val &= pVM->hwaccm.s.vmx.msr.vmx_entry.n.allowed1;
1147	rc = VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_CONTROLS, val);
1148	AssertRC(rc);
1149
1150	/* VMX_VMCS_CTRL_EXIT_CONTROLS
1151	* Set required bits to one and zero according to the MSR capabilities.
1152	*/
1153	val = pVM->hwaccm.s.vmx.msr.vmx_exit.n.disallowed0;
1154
1155	/* Save debug controls (dr7 & IA32_DEBUGCTL_MSR) (forced to 1 on the 'first' VT-x capable CPUs; this actually includes the newest Nehalem CPUs) */
1156	#if 0 /* @todo deal with 32/64 */
1157	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_DEBUG \| VMX_VMCS_CTRL_EXIT_CONTROLS_LOAD_HOST_EFER_MSR;
1158	#else
1159	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_DEBUG;
1160	#endif
1161
1162	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1163	if (VMX_IS_64BIT_HOST_MODE())
1164	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_AMD64;
1165	/* else: Must be zero when AMD64 is not available. */
1166	#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
1167	if (pCtx->msrEFER & MSR_K6_EFER_LMA)
1168	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_AMD64; /* our switcher goes to long mode */
1169	else
1170	Assert(!(val & VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_AMD64));
1171	#endif
1172	val &= pVM->hwaccm.s.vmx.msr.vmx_exit.n.allowed1;
1173	/* Don't acknowledge external interrupts on VM-exit. */
1174	rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_CONTROLS, val);
1175	AssertRC(rc);
1176
1177	/* Guest CPU context: ES, CS, SS, DS, FS, GS. */
1178	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_SEGMENT_REGS)
1179	{
1180	#ifdef HWACCM_VMX_EMULATE_REALMODE
1181	if (pVM->hwaccm.s.vmx.pRealModeTSS)
1182	{
1183	PGMMODE enmGuestMode = PGMGetGuestMode(pVCpu);
1184	if (pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode != enmGuestMode)
1185	{
1186	/* Correct weird requirements for switching to protected mode. */
1187	if ( pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode == PGMMODE_REAL
1188	&& enmGuestMode >= PGMMODE_PROTECTED)
1189	{
1190	/* Flush the recompiler code cache as it's not unlikely
1191	* the guest will rewrite code it will later execute in real
1192	* mode (OpenBSD 4.0 is one such example)
1193	*/
1194	REMFlushTBs(pVM);
1195
1196	/* DPL of all hidden selector registers must match the current CPL (0). */
1197	pCtx->csHid.Attr.n.u2Dpl = 0;
1198	pCtx->csHid.Attr.n.u4Type = X86_SEL_TYPE_CODE \| X86_SEL_TYPE_RW_ACC;
1199
1200	pCtx->dsHid.Attr.n.u2Dpl = 0;
1201	pCtx->esHid.Attr.n.u2Dpl = 0;
1202	pCtx->fsHid.Attr.n.u2Dpl = 0;
1203	pCtx->gsHid.Attr.n.u2Dpl = 0;
1204	pCtx->ssHid.Attr.n.u2Dpl = 0;
1205
1206	/* The limit must correspond to the granularity bit. */
1207	if (!pCtx->csHid.Attr.n.u1Granularity)
1208	pCtx->csHid.u32Limit &= 0xffff;
1209	if (!pCtx->dsHid.Attr.n.u1Granularity)
1210	pCtx->dsHid.u32Limit &= 0xffff;
1211	if (!pCtx->esHid.Attr.n.u1Granularity)
1212	pCtx->esHid.u32Limit &= 0xffff;
1213	if (!pCtx->fsHid.Attr.n.u1Granularity)
1214	pCtx->fsHid.u32Limit &= 0xffff;
1215	if (!pCtx->gsHid.Attr.n.u1Granularity)
1216	pCtx->gsHid.u32Limit &= 0xffff;
1217	if (!pCtx->ssHid.Attr.n.u1Granularity)
1218	pCtx->ssHid.u32Limit &= 0xffff;
1219	}
1220	else
1221	/* Switching from protected mode to real mode. */
1222	if ( pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode >= PGMMODE_PROTECTED
1223	&& enmGuestMode == PGMMODE_REAL)
1224	{
1225	/* The limit must also be set to 0xffff. */
1226	pCtx->csHid.u32Limit = 0xffff;
1227	pCtx->dsHid.u32Limit = 0xffff;
1228	pCtx->esHid.u32Limit = 0xffff;
1229	pCtx->fsHid.u32Limit = 0xffff;
1230	pCtx->gsHid.u32Limit = 0xffff;
1231	pCtx->ssHid.u32Limit = 0xffff;
1232
1233	Assert(pCtx->csHid.u64Base <= 0xfffff);
1234	Assert(pCtx->dsHid.u64Base <= 0xfffff);
1235	Assert(pCtx->esHid.u64Base <= 0xfffff);
1236	Assert(pCtx->fsHid.u64Base <= 0xfffff);
1237	Assert(pCtx->gsHid.u64Base <= 0xfffff);
1238	}
1239	pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode = enmGuestMode;
1240	}
1241	else
1242	/* VT-x will fail with a guest invalid state otherwise... (CPU state after a reset) */
1243	if ( CPUMIsGuestInRealModeEx(pCtx)
1244	&& pCtx->csHid.u64Base == 0xffff0000)
1245	{
1246	pCtx->csHid.u64Base = 0xf0000;
1247	pCtx->cs = 0xf000;
1248	}
1249	}
1250	#endif /* HWACCM_VMX_EMULATE_REALMODE */
1251
1252	VMX_WRITE_SELREG(ES, es);
1253	AssertRC(rc);
1254
1255	VMX_WRITE_SELREG(CS, cs);
1256	AssertRC(rc);
1257
1258	VMX_WRITE_SELREG(SS, ss);
1259	AssertRC(rc);
1260
1261	VMX_WRITE_SELREG(DS, ds);
1262	AssertRC(rc);
1263
1264	/* The base values in the hidden fs & gs registers are not in sync with the msrs; they are cut to 32 bits. */
1265	VMX_WRITE_SELREG(FS, fs);
1266	AssertRC(rc);
1267
1268	VMX_WRITE_SELREG(GS, gs);
1269	AssertRC(rc);
1270	}
1271
1272	/* Guest CPU context: LDTR. */
1273	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_LDTR)
1274	{
1275	if (pCtx->ldtr == 0)
1276	{
1277	rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_LDTR, 0);
1278	rc \|= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_LIMIT, 0);
1279	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_LDTR_BASE, 0);
1280	/* Note: vmlaunch will fail with 0 or just 0x02. No idea why. */
1281	rc \|= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, 0x82 /* present, LDT */);
1282	}
1283	else
1284	{
1285	rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_LDTR, pCtx->ldtr);
1286	rc \|= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtrHid.u32Limit);
1287	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_LDTR_BASE, pCtx->ldtrHid.u64Base);
1288	rc \|= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, pCtx->ldtrHid.Attr.u);
1289	}
1290	AssertRC(rc);
1291	}
1292	/* Guest CPU context: TR. */
1293	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_TR)
1294	{
1295	#ifdef HWACCM_VMX_EMULATE_REALMODE
1296	/* Real mode emulation using v86 mode with CR4.VME (interrupt redirection using the int bitmap in the TSS) */
1297	if (CPUMIsGuestInRealModeEx(pCtx))
1298	{
1299	RTGCPHYS GCPhys;
1300
1301	/* We convert it here every time as pci regions could be reconfigured. */
1302	rc = PDMVMMDevHeapR3ToGCPhys(pVM, pVM->hwaccm.s.vmx.pRealModeTSS, &GCPhys);
1303	AssertRC(rc);
1304
1305	rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_TR, 0);
1306	rc \|= VMXWriteVMCS(VMX_VMCS32_GUEST_TR_LIMIT, HWACCM_VTX_TSS_SIZE);
1307	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_TR_BASE, GCPhys /* phys = virt in this mode */);
1308
1309	X86DESCATTR attr;
1310
1311	attr.u = 0;
1312	attr.n.u1Present = 1;
1313	attr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
1314	val = attr.u;
1315	}
1316	else
1317	#endif /* HWACCM_VMX_EMULATE_REALMODE */
1318	{
1319	rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_TR, pCtx->tr);
1320	rc \|= VMXWriteVMCS(VMX_VMCS32_GUEST_TR_LIMIT, pCtx->trHid.u32Limit);
1321	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_TR_BASE, pCtx->trHid.u64Base);
1322
1323	val = pCtx->trHid.Attr.u;
1324
1325	/* The TSS selector must be busy. */
1326	if ((val & 0xF) == X86_SEL_TYPE_SYS_286_TSS_AVAIL)
1327	val = (val & ~0xF) \| X86_SEL_TYPE_SYS_286_TSS_BUSY;
1328	else
1329	/* Default even if no TR selector has been set (otherwise vmlaunch will fail!) */
1330	val = (val & ~0xF) \| X86_SEL_TYPE_SYS_386_TSS_BUSY;
1331
1332	}
1333	rc \|= VMXWriteVMCS(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, val);
1334	AssertRC(rc);
1335	}
1336	/* Guest CPU context: GDTR. */
1337	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_GDTR)
1338	{
1339	rc = VMXWriteVMCS(VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt);
1340	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_GDTR_BASE, pCtx->gdtr.pGdt);
1341	AssertRC(rc);
1342	}
1343	/* Guest CPU context: IDTR. */
1344	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_IDTR)
1345	{
1346	rc = VMXWriteVMCS(VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt);
1347	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_IDTR_BASE, pCtx->idtr.pIdt);
1348	AssertRC(rc);
1349	}
1350
1351	/*
1352	* Sysenter MSRs (unconditional)
1353	*/
1354	rc = VMXWriteVMCS(VMX_VMCS32_GUEST_SYSENTER_CS, pCtx->SysEnter.cs);
1355	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_SYSENTER_EIP, pCtx->SysEnter.eip);
1356	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_SYSENTER_ESP, pCtx->SysEnter.esp);
1357	AssertRC(rc);
1358
1359	/* Control registers */
1360	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_CR0)
1361	{
1362	val = pCtx->cr0;
1363	rc = VMXWriteVMCS(VMX_VMCS_CTRL_CR0_READ_SHADOW, val);
1364	Log2(("Guest CR0-shadow %08x\n", val));
1365	if (CPUMIsGuestFPUStateActive(pVCpu) == false)
1366	{
1367	/* Always use #NM exceptions to load the FPU/XMM state on demand. */
1368	val \|= X86_CR0_TS \| X86_CR0_ET \| X86_CR0_NE \| X86_CR0_MP;
1369	}
1370	else
1371	{
1372	/** @todo check if we support the old style mess correctly. */
1373	if (!(val & X86_CR0_NE))
1374	Log(("Forcing X86_CR0_NE!!!\n"));
1375
1376	val \|= X86_CR0_NE; /* always turn on the native mechanism to report FPU errors (old style uses interrupts) */
1377	}
1378	/* Note: protected mode & paging are always enabled; we use them for emulating real and protected mode without paging too. */
1379	val \|= X86_CR0_PE \| X86_CR0_PG;
1380	if (pVM->hwaccm.s.fNestedPaging)
1381	{
1382	if (CPUMIsGuestInPagedProtectedModeEx(pCtx))
1383	{
1384	/* Disable cr3 read/write monitoring as we don't need it for EPT. */
1385	pVCpu->hwaccm.s.vmx.proc_ctls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
1386	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT);
1387	}
1388	else
1389	{
1390	/* Reenable cr3 read/write monitoring as our identity mapped page table is active. */
1391	pVCpu->hwaccm.s.vmx.proc_ctls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
1392	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
1393	}
1394	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1395	AssertRC(rc);
1396	}
1397	else
1398	{
1399	/* Note: We must also set this as we rely on protecting various pages for which supervisor writes must be caught. */
1400	val \|= X86_CR0_WP;
1401	}
1402
1403	/* Always enable caching. */
1404	val &= ~(X86_CR0_CD\|X86_CR0_NW);
1405
1406	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_CR0, val);
1407	Log2(("Guest CR0 %08x\n", val));
1408	/* CR0 flags owned by the host; if the guests attempts to change them, then
1409	* the VM will exit.
1410	*/
1411	val = X86_CR0_PE /* Must monitor this bit (assumptions are made for real mode emulation) */
1412	\| X86_CR0_WP /* Must monitor this bit (it must always be enabled). */
1413	\| X86_CR0_PG /* Must monitor this bit (assumptions are made for real mode & protected mode without paging emulation) */
1414	\| X86_CR0_TS
1415	\| X86_CR0_ET /* Bit not restored during VM-exit! */
1416	\| X86_CR0_CD /* Bit not restored during VM-exit! */
1417	\| X86_CR0_NW /* Bit not restored during VM-exit! */
1418	\| X86_CR0_NE
1419	\| X86_CR0_MP;
1420	pVCpu->hwaccm.s.vmx.cr0_mask = val;
1421
1422	rc \|= VMXWriteVMCS(VMX_VMCS_CTRL_CR0_MASK, val);
1423	Log2(("Guest CR0-mask %08x\n", val));
1424	AssertRC(rc);
1425	}
1426	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_CR4)
1427	{
1428	/* CR4 */
1429	rc = VMXWriteVMCS(VMX_VMCS_CTRL_CR4_READ_SHADOW, pCtx->cr4);
1430	Log2(("Guest CR4-shadow %08x\n", pCtx->cr4));
1431	/* Set the required bits in cr4 too (currently X86_CR4_VMXE). */
1432	val = pCtx->cr4 \| (uint32_t)pVM->hwaccm.s.vmx.msr.vmx_cr4_fixed0;
1433
1434	if (!pVM->hwaccm.s.fNestedPaging)
1435	{
1436	switch(pVCpu->hwaccm.s.enmShadowMode)
1437	{
1438	case PGMMODE_REAL: /* Real mode -> emulated using v86 mode */
1439	case PGMMODE_PROTECTED: /* Protected mode, no paging -> emulated using identity mapping. */
1440	case PGMMODE_32_BIT: /* 32-bit paging. */
1441	val &= ~X86_CR4_PAE;
1442	break;
1443
1444	case PGMMODE_PAE: /* PAE paging. */
1445	case PGMMODE_PAE_NX: /* PAE paging with NX enabled. */
1446	/** @todo use normal 32 bits paging */
1447	val \|= X86_CR4_PAE;
1448	break;
1449
1450	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
1451	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
1452	#ifdef VBOX_ENABLE_64_BITS_GUESTS
1453	break;
1454	#else
1455	AssertFailed();
1456	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1457	#endif
1458	default: /* shut up gcc */
1459	AssertFailed();
1460	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1461	}
1462	}
1463	else
1464	if (!CPUMIsGuestInPagedProtectedModeEx(pCtx))
1465	{
1466	/* We use 4 MB pages in our identity mapping page table for real and protected mode without paging. */
1467	val \|= X86_CR4_PSE;
1468	/* Our identity mapping is a 32 bits page directory. */
1469	val &= ~X86_CR4_PAE;
1470	}
1471
1472	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_CR4, val);
1473	Log2(("Guest CR4 %08x\n", val));
1474	/* CR4 flags owned by the host; if the guests attempts to change them, then
1475	* the VM will exit.
1476	*/
1477	val = 0
1478	\| X86_CR4_PAE
1479	\| X86_CR4_PGE
1480	\| X86_CR4_PSE
1481	\| X86_CR4_VMXE;
1482	pVCpu->hwaccm.s.vmx.cr4_mask = val;
1483
1484	rc \|= VMXWriteVMCS(VMX_VMCS_CTRL_CR4_MASK, val);
1485	Log2(("Guest CR4-mask %08x\n", val));
1486	AssertRC(rc);
1487	}
1488
1489	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_CR3)
1490	{
1491	if (pVM->hwaccm.s.fNestedPaging)
1492	{
1493	Assert(PGMGetHyperCR3(pVCpu));
1494	pVCpu->hwaccm.s.vmx.GCPhysEPTP = PGMGetHyperCR3(pVCpu);
1495
1496	Assert(!(pVCpu->hwaccm.s.vmx.GCPhysEPTP & 0xfff));
1497	/** @todo Check the IA32_VMX_EPT_VPID_CAP MSR for other supported memory types. */
1498	pVCpu->hwaccm.s.vmx.GCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
1499	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
1500
1501	rc = VMXWriteVMCS64(VMX_VMCS_CTRL_EPTP_FULL, pVCpu->hwaccm.s.vmx.GCPhysEPTP);
1502	AssertRC(rc);
1503
1504	if (!CPUMIsGuestInPagedProtectedModeEx(pCtx))
1505	{
1506	RTGCPHYS GCPhys;
1507
1508	/* We convert it here every time as pci regions could be reconfigured. */
1509	rc = PDMVMMDevHeapR3ToGCPhys(pVM, pVM->hwaccm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
1510	AssertRC(rc);
1511
1512	/* We use our identity mapping page table here as we need to map guest virtual to guest physical addresses; EPT will
1513	* take care of the translation to host physical addresses.
1514	*/
1515	val = GCPhys;
1516	}
1517	else
1518	{
1519	/* Save the real guest CR3 in VMX_VMCS_GUEST_CR3 */
1520	val = pCtx->cr3;
1521	/* Prefetch the four PDPT entries in PAE mode. */
1522	vmxR0PrefetchPAEPdptrs(pVM, pVCpu, pCtx);
1523	}
1524	}
1525	else
1526	{
1527	val = PGMGetHyperCR3(pVCpu);
1528	Assert(val \|\| VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL));
1529	}
1530
1531	/* Save our shadow CR3 register. */
1532	rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_CR3, val);
1533	AssertRC(rc);
1534	}
1535
1536	/* Debug registers. */
1537	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_DEBUG)
1538	{
1539	pCtx->dr[6] \|= X86_DR6_INIT_VAL; /* set all reserved bits to 1. */
1540	pCtx->dr[6] &= ~RT_BIT(12); /* must be zero. */
1541
1542	pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
1543	pCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* must be zero */
1544	pCtx->dr[7] \|= 0x400; /* must be one */
1545
1546	/* Resync DR7 */
1547	rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_DR7, pCtx->dr[7]);
1548	AssertRC(rc);
1549
1550	/* Sync the debug state now if any breakpoint is armed. */
1551	if ( (pCtx->dr[7] & (X86_DR7_ENABLED_MASK\|X86_DR7_GD))
1552	&& !CPUMIsGuestDebugStateActive(pVCpu)
1553	&& !DBGFIsStepping(pVCpu))
1554	{
1555	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatDRxArmed);
1556
1557	/* Disable drx move intercepts. */
1558	pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
1559	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1560	AssertRC(rc);
1561
1562	/* Save the host and load the guest debug state. */
1563	rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
1564	AssertRC(rc);
1565	}
1566
1567	/* IA32_DEBUGCTL MSR. */
1568	rc = VMXWriteVMCS64(VMX_VMCS_GUEST_DEBUGCTL_FULL, 0);
1569	AssertRC(rc);
1570
1571	/** @todo do we really ever need this? */
1572	rc \|= VMXWriteVMCS(VMX_VMCS_GUEST_DEBUG_EXCEPTIONS, 0);
1573	AssertRC(rc);
1574	}
1575
1576	/* EIP, ESP and EFLAGS */
1577	rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_RIP, pCtx->rip);
1578	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_RSP, pCtx->rsp);
1579	AssertRC(rc);
1580
1581	/* Bits 22-31, 15, 5 & 3 must be zero. Bit 1 must be 1. */
1582	eflags = pCtx->eflags;
1583	eflags.u32 &= VMX_EFLAGS_RESERVED_0;
1584	eflags.u32 \|= VMX_EFLAGS_RESERVED_1;
1585
1586	#ifdef HWACCM_VMX_EMULATE_REALMODE
1587	/* Real mode emulation using v86 mode. */
1588	if (CPUMIsGuestInRealModeEx(pCtx))
1589	{
1590	pVCpu->hwaccm.s.vmx.RealMode.eflags = eflags;
1591
1592	eflags.Bits.u1VM = 1;
1593	eflags.Bits.u2IOPL = 0; /* must always be 0 or else certain instructions won't cause faults. */
1594	}
1595	#endif /* HWACCM_VMX_EMULATE_REALMODE */
1596	rc = VMXWriteVMCS(VMX_VMCS_GUEST_RFLAGS, eflags.u32);
1597	AssertRC(rc);
1598
1599	/* TSC offset. */
1600	uint64_t u64TSCOffset;
1601
1602	if (TMCpuTickCanUseRealTSC(pVCpu, &u64TSCOffset))
1603	{
1604	/* Note: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT takes precedence over TSC_OFFSET */
1605	rc = VMXWriteVMCS64(VMX_VMCS_CTRL_TSC_OFFSET_FULL, u64TSCOffset);
1606	AssertRC(rc);
1607
1608	pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
1609	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1610	AssertRC(rc);
1611	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTSCOffset);
1612	}
1613	else
1614	{
1615	pVCpu->hwaccm.s.vmx.proc_ctls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
1616	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1617	AssertRC(rc);
1618	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTSCIntercept);
1619	}
1620
1621	/* 64 bits guest mode? */
1622	if (pCtx->msrEFER & MSR_K6_EFER_LMA)
1623	{
1624	#if !defined(VBOX_ENABLE_64_BITS_GUESTS)
1625	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1626	#elif HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1627	pVCpu->hwaccm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
1628	#else
1629	# ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1630	if (!pVM->hwaccm.s.fAllow64BitGuests)
1631	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1632	# endif
1633	pVCpu->hwaccm.s.vmx.pfnStartVM = VMXR0StartVM64;
1634	#endif
1635	/* Unconditionally update these as wrmsr might have changed them. */
1636	rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_FS_BASE, pCtx->fsHid.u64Base);
1637	AssertRC(rc);
1638	rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_GS_BASE, pCtx->gsHid.u64Base);
1639	AssertRC(rc);
1640	}
1641	else
1642	{
1643	pVCpu->hwaccm.s.vmx.pfnStartVM = VMXR0StartVM32;
1644	}
1645
1646	#if 0 /* @todo deal with 32/64 */
1647	/* Unconditionally update the guest EFER - on CPUs that supports it. */
1648	if (pVM->hwaccm.s.vmx.msr.vmx_entry.n.allowed1 & VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_GUEST_EFER_MSR)
1649	{
1650	rc = VMXWriteVMCS64(VMX_VMCS_GUEST_EFER_FULL, pCtx->msrEFER);
1651	AssertRC(rc);
1652	}
1653	#endif
1654
1655	vmxR0UpdateExceptionBitmap(pVM, pVCpu, pCtx);
1656
1657	/* Done. */
1658	pVCpu->hwaccm.s.fContextUseFlags &= ~HWACCM_CHANGED_ALL_GUEST;
1659
1660	return rc;
1661	}
1662
1663	/**
1664	* Syncs back the guest state
1665	*
1666	* @returns VBox status code.
1667	* @param pVM The VM to operate on.
1668	* @param pVCpu The VMCPU to operate on.
1669	* @param pCtx Guest context
1670	*/
1671	DECLINLINE(int) VMXR0SaveGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1672	{
1673	RTGCUINTREG val, valShadow;
1674	RTGCUINTPTR uInterruptState;
1675	int rc;
1676
1677	/* Let's first sync back eip, esp, and eflags. */
1678	rc = VMXReadCachedVMCS(VMX_VMCS64_GUEST_RIP, &val);
1679	AssertRC(rc);
1680	pCtx->rip = val;
1681	rc = VMXReadCachedVMCS(VMX_VMCS64_GUEST_RSP, &val);
1682	AssertRC(rc);
1683	pCtx->rsp = val;
1684	rc = VMXReadCachedVMCS(VMX_VMCS_GUEST_RFLAGS, &val);
1685	AssertRC(rc);
1686	pCtx->eflags.u32 = val;
1687
1688	/* Take care of instruction fusing (sti, mov ss) */
1689	rc \|= VMXReadCachedVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &val);
1690	uInterruptState = val;
1691	if (uInterruptState != 0)
1692	{
1693	Assert(uInterruptState <= 2); /* only sti & mov ss */
1694	Log(("uInterruptState %x eip=%RGv\n", uInterruptState, pCtx->rip));
1695	EMSetInhibitInterruptsPC(pVCpu, pCtx->rip);
1696	}
1697	else
1698	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
1699
1700	/* Control registers. */
1701	VMXReadCachedVMCS(VMX_VMCS_CTRL_CR0_READ_SHADOW, &valShadow);
1702	VMXReadCachedVMCS(VMX_VMCS64_GUEST_CR0, &val);
1703	val = (valShadow & pVCpu->hwaccm.s.vmx.cr0_mask) \| (val & ~pVCpu->hwaccm.s.vmx.cr0_mask);
1704	CPUMSetGuestCR0(pVCpu, val);
1705
1706	VMXReadCachedVMCS(VMX_VMCS_CTRL_CR4_READ_SHADOW, &valShadow);
1707	VMXReadCachedVMCS(VMX_VMCS64_GUEST_CR4, &val);
1708	val = (valShadow & pVCpu->hwaccm.s.vmx.cr4_mask) \| (val & ~pVCpu->hwaccm.s.vmx.cr4_mask);
1709	CPUMSetGuestCR4(pVCpu, val);
1710
1711	/* Note: no reason to sync back the CRx registers. They can't be changed by the guest. */
1712	/* Note: only in the nested paging case can CR3 & CR4 be changed by the guest. */
1713	if ( pVM->hwaccm.s.fNestedPaging
1714	&& CPUMIsGuestInPagedProtectedModeEx(pCtx))
1715	{
1716	PVMCSCACHE pCache = &pVCpu->hwaccm.s.vmx.VMCSCache;
1717
1718	/* Can be updated behind our back in the nested paging case. */
1719	CPUMSetGuestCR2(pVCpu, pCache->cr2);
1720
1721	VMXReadCachedVMCS(VMX_VMCS64_GUEST_CR3, &val);
1722
1723	if (val != pCtx->cr3)
1724	{
1725	CPUMSetGuestCR3(pVCpu, val);
1726	PGMUpdateCR3(pVCpu, val);
1727	}
1728	/* Prefetch the four PDPT entries in PAE mode. */
1729	vmxR0PrefetchPAEPdptrs(pVM, pVCpu, pCtx);
1730	}
1731
1732	/* Sync back DR7 here. */
1733	VMXReadCachedVMCS(VMX_VMCS64_GUEST_DR7, &val);
1734	pCtx->dr[7] = val;
1735
1736	/* Guest CPU context: ES, CS, SS, DS, FS, GS. */
1737	VMX_READ_SELREG(ES, es);
1738	VMX_READ_SELREG(SS, ss);
1739	VMX_READ_SELREG(CS, cs);
1740	VMX_READ_SELREG(DS, ds);
1741	VMX_READ_SELREG(FS, fs);
1742	VMX_READ_SELREG(GS, gs);
1743
1744	/*
1745	* System MSRs
1746	*/
1747	VMXReadCachedVMCS(VMX_VMCS32_GUEST_SYSENTER_CS, &val);
1748	pCtx->SysEnter.cs = val;
1749	VMXReadCachedVMCS(VMX_VMCS64_GUEST_SYSENTER_EIP, &val);
1750	pCtx->SysEnter.eip = val;
1751	VMXReadCachedVMCS(VMX_VMCS64_GUEST_SYSENTER_ESP, &val);
1752	pCtx->SysEnter.esp = val;
1753
1754	/* Misc. registers; must sync everything otherwise we can get out of sync when jumping to ring 3. */
1755	VMX_READ_SELREG(LDTR, ldtr);
1756
1757	VMXReadCachedVMCS(VMX_VMCS32_GUEST_GDTR_LIMIT, &val);
1758	pCtx->gdtr.cbGdt = val;
1759	VMXReadCachedVMCS(VMX_VMCS64_GUEST_GDTR_BASE, &val);
1760	pCtx->gdtr.pGdt = val;
1761
1762	VMXReadCachedVMCS(VMX_VMCS32_GUEST_IDTR_LIMIT, &val);
1763	pCtx->idtr.cbIdt = val;
1764	VMXReadCachedVMCS(VMX_VMCS64_GUEST_IDTR_BASE, &val);
1765	pCtx->idtr.pIdt = val;
1766
1767	#ifdef HWACCM_VMX_EMULATE_REALMODE
1768	/* Real mode emulation using v86 mode. */
1769	if (CPUMIsGuestInRealModeEx(pCtx))
1770	{
1771	/* Hide our emulation flags */
1772	pCtx->eflags.Bits.u1VM = 0;
1773
1774	/* Restore original IOPL setting as we always use 0. */
1775	pCtx->eflags.Bits.u2IOPL = pVCpu->hwaccm.s.vmx.RealMode.eflags.Bits.u2IOPL;
1776
1777	/* Force a TR resync every time in case we switch modes. */
1778	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_TR;
1779	}
1780	else
1781	#endif /* HWACCM_VMX_EMULATE_REALMODE */
1782	{
1783	/* In real mode we have a fake TSS, so only sync it back when it's supposed to be valid. */
1784	VMX_READ_SELREG(TR, tr);
1785	}
1786	return VINF_SUCCESS;
1787	}
1788
1789	/**
1790	* Dummy placeholder
1791	*
1792	* @param pVM The VM to operate on.
1793	* @param pVCpu The VMCPU to operate on.
1794	*/
1795	static void vmxR0SetupTLBDummy(PVM pVM, PVMCPU pVCpu)
1796	{
1797	NOREF(pVM);
1798	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1799	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1800	pVCpu->hwaccm.s.cTlbShootdownPages = 0;
1801	return;
1802	}
1803
1804	/**
1805	* Setup the tagged TLB for EPT
1806	*
1807	* @returns VBox status code.
1808	* @param pVM The VM to operate on.
1809	* @param pVCpu The VMCPU to operate on.
1810	*/
1811	static void vmxR0SetupTLBEPT(PVM pVM, PVMCPU pVCpu)
1812	{
1813	PHWACCM_CPUINFO pCpu;
1814
1815	Assert(pVM->hwaccm.s.fNestedPaging);
1816	Assert(!pVM->hwaccm.s.vmx.fVPID);
1817
1818	/* Deal with tagged TLBs if VPID or EPT is supported. */
1819	pCpu = HWACCMR0GetCurrentCpu();
1820	/* Force a TLB flush for the first world switch if the current cpu differs from the one we ran on last. */
1821	/* Note that this can happen both for start and resume due to long jumps back to ring 3. */
1822	if ( pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu
1823	/* if the tlb flush count has changed, another VM has flushed the TLB of this cpu, so we can't use our current ASID anymore. */
1824	\|\| pVCpu->hwaccm.s.cTLBFlushes != pCpu->cTLBFlushes)
1825	{
1826	/* Force a TLB flush on VM entry. */
1827	pVCpu->hwaccm.s.fForceTLBFlush = true;
1828	}
1829	else
1830	Assert(!pCpu->fFlushTLB);
1831
1832	/* Check for tlb shootdown flushes. */
1833	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH_BIT))
1834	pVCpu->hwaccm.s.fForceTLBFlush = true;
1835
1836	pVCpu->hwaccm.s.idLastCpu = pCpu->idCpu;
1837	pCpu->fFlushTLB = false;
1838
1839	if (pVCpu->hwaccm.s.fForceTLBFlush)
1840	{
1841	vmxR0FlushEPT(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushContext, 0);
1842	}
1843	else
1844	if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1845	{
1846	/* Deal with pending TLB shootdown actions which were queued when we were not executing code. */
1847	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTlbShootdown);
1848	for (unsigned i=0;i<pVCpu->hwaccm.s.cTlbShootdownPages;i++)
1849	{
1850	/* aTlbShootdownPages contains physical addresses in this case. */
1851	vmxR0FlushEPT(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushContext, pVCpu->hwaccm.s.aTlbShootdownPages[i]);
1852	}
1853	}
1854	pVCpu->hwaccm.s.cTlbShootdownPages = 0;
1855	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1856
1857	#ifdef VBOX_WITH_STATISTICS
1858	if (pVCpu->hwaccm.s.fForceTLBFlush)
1859	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushTLBWorldSwitch);
1860	else
1861	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatNoFlushTLBWorldSwitch);
1862	#endif
1863	}
1864
1865	#ifdef HWACCM_VTX_WITH_VPID
1866	/**
1867	* Setup the tagged TLB for VPID
1868	*
1869	* @returns VBox status code.
1870	* @param pVM The VM to operate on.
1871	* @param pVCpu The VMCPU to operate on.
1872	*/
1873	static void vmxR0SetupTLBVPID(PVM pVM, PVMCPU pVCpu)
1874	{
1875	PHWACCM_CPUINFO pCpu;
1876
1877	Assert(pVM->hwaccm.s.vmx.fVPID);
1878	Assert(!pVM->hwaccm.s.fNestedPaging);
1879
1880	/* Deal with tagged TLBs if VPID or EPT is supported. */
1881	pCpu = HWACCMR0GetCurrentCpu();
1882	/* Force a TLB flush for the first world switch if the current cpu differs from the one we ran on last. */
1883	/* Note that this can happen both for start and resume due to long jumps back to ring 3. */
1884	if ( pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu
1885	/* if the tlb flush count has changed, another VM has flushed the TLB of this cpu, so we can't use our current ASID anymore. */
1886	\|\| pVCpu->hwaccm.s.cTLBFlushes != pCpu->cTLBFlushes)
1887	{
1888	/* Force a TLB flush on VM entry. */
1889	pVCpu->hwaccm.s.fForceTLBFlush = true;
1890	}
1891	else
1892	Assert(!pCpu->fFlushTLB);
1893
1894	pVCpu->hwaccm.s.idLastCpu = pCpu->idCpu;
1895
1896	/* Check for tlb shootdown flushes. */
1897	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH_BIT))
1898	pVCpu->hwaccm.s.fForceTLBFlush = true;
1899
1900	/* Make sure we flush the TLB when required. Switch ASID to achieve the same thing, but without actually flushing the whole TLB (which is expensive). */
1901	if (pVCpu->hwaccm.s.fForceTLBFlush)
1902	{
1903	if ( ++pCpu->uCurrentASID >= pVM->hwaccm.s.uMaxASID
1904	\|\| pCpu->fFlushTLB)
1905	{
1906	pCpu->fFlushTLB = false;
1907	pCpu->uCurrentASID = 1; /* start at 1; host uses 0 */
1908	pCpu->cTLBFlushes++;
1909	}
1910	else
1911	{
1912	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushASID);
1913	pVCpu->hwaccm.s.fForceTLBFlush = false;
1914	}
1915
1916	pVCpu->hwaccm.s.cTLBFlushes = pCpu->cTLBFlushes;
1917	pVCpu->hwaccm.s.uCurrentASID = pCpu->uCurrentASID;
1918	}
1919	else
1920	{
1921	Assert(!pCpu->fFlushTLB);
1922	Assert(pVCpu->hwaccm.s.uCurrentASID && pCpu->uCurrentASID);
1923
1924	if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1925	{
1926	/* Deal with pending TLB shootdown actions which were queued when we were not executing code. */
1927	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTlbShootdown);
1928	for (unsigned i=0;i<pVCpu->hwaccm.s.cTlbShootdownPages;i++)
1929	vmxR0FlushVPID(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushContext, pVCpu->hwaccm.s.aTlbShootdownPages[i]);
1930	}
1931	}
1932	pVCpu->hwaccm.s.cTlbShootdownPages = 0;
1933	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1934
1935	AssertMsg(pVCpu->hwaccm.s.cTLBFlushes == pCpu->cTLBFlushes, ("Flush count mismatch for cpu %d (%x vs %x)\n", pCpu->idCpu, pVCpu->hwaccm.s.cTLBFlushes, pCpu->cTLBFlushes));
1936	AssertMsg(pCpu->uCurrentASID >= 1 && pCpu->uCurrentASID < pVM->hwaccm.s.uMaxASID, ("cpu%d uCurrentASID = %x\n", pCpu->idCpu, pCpu->uCurrentASID));
1937	AssertMsg(pVCpu->hwaccm.s.uCurrentASID >= 1 && pVCpu->hwaccm.s.uCurrentASID < pVM->hwaccm.s.uMaxASID, ("cpu%d VM uCurrentASID = %x\n", pCpu->idCpu, pVCpu->hwaccm.s.uCurrentASID));
1938
1939	int rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hwaccm.s.uCurrentASID);
1940	AssertRC(rc);
1941
1942	if (pVCpu->hwaccm.s.fForceTLBFlush)
1943	vmxR0FlushVPID(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushContext, 0);
1944
1945	#ifdef VBOX_WITH_STATISTICS
1946	if (pVCpu->hwaccm.s.fForceTLBFlush)
1947	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushTLBWorldSwitch);
1948	else
1949	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatNoFlushTLBWorldSwitch);
1950	#endif
1951	}
1952	#endif /* HWACCM_VTX_WITH_VPID */
1953
1954	/**
1955	* Runs guest code in a VT-x VM.
1956	*
1957	* @returns VBox status code.
1958	* @param pVM The VM to operate on.
1959	* @param pVCpu The VMCPU to operate on.
1960	* @param pCtx Guest context
1961	*/
1962	VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1963	{
1964	int rc = VINF_SUCCESS;
1965	RTGCUINTREG val;
1966	RTGCUINTREG exitReason = VMX_EXIT_INVALID;
1967	RTGCUINTREG instrError, cbInstr;
1968	RTGCUINTPTR exitQualification;
1969	RTGCUINTPTR intInfo = 0; /* shut up buggy gcc 4 */
1970	RTGCUINTPTR errCode, instrInfo;
1971	bool fSyncTPR = false;
1972	PHWACCM_CPUINFO pCpu = 0;
1973	RTCCUINTREG uOldEFlags;
1974	unsigned cResume = 0;
1975	#ifdef VBOX_STRICT
1976	RTCPUID idCpuCheck;
1977	#endif
1978	#ifdef VBOX_WITH_STATISTICS
1979	bool fStatEntryStarted = true;
1980	bool fStatExit2Started = false;
1981	#endif
1982
1983	Log2(("\nE"));
1984
1985	STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatEntry, x);
1986
1987	#ifdef VBOX_STRICT
1988	{
1989	RTCCUINTREG val;
1990
1991	rc = VMXReadVMCS(VMX_VMCS_CTRL_PIN_EXEC_CONTROLS, &val);
1992	AssertRC(rc);
1993	Log2(("VMX_VMCS_CTRL_PIN_EXEC_CONTROLS = %08x\n", val));
1994
1995	/* allowed zero */
1996	if ((val & pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.disallowed0)
1997	Log(("Invalid VMX_VMCS_CTRL_PIN_EXEC_CONTROLS: zero\n"));
1998
1999	/* allowed one */
2000	if ((val & ~pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.allowed1) != 0)
2001	Log(("Invalid VMX_VMCS_CTRL_PIN_EXEC_CONTROLS: one\n"));
2002
2003	rc = VMXReadVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, &val);
2004	AssertRC(rc);
2005	Log2(("VMX_VMCS_CTRL_PROC_EXEC_CONTROLS = %08x\n", val));
2006
2007	/* Must be set according to the MSR, but can be cleared in case of EPT. */
2008	if (pVM->hwaccm.s.fNestedPaging)
2009	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT
2010	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
2011	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
2012
2013	/* allowed zero */
2014	if ((val & pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.disallowed0)
2015	Log(("Invalid VMX_VMCS_CTRL_PROC_EXEC_CONTROLS: zero\n"));
2016
2017	/* allowed one */
2018	if ((val & ~pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1) != 0)
2019	Log(("Invalid VMX_VMCS_CTRL_PROC_EXEC_CONTROLS: one\n"));
2020
2021	rc = VMXReadVMCS(VMX_VMCS_CTRL_ENTRY_CONTROLS, &val);
2022	AssertRC(rc);
2023	Log2(("VMX_VMCS_CTRL_ENTRY_CONTROLS = %08x\n", val));
2024
2025	/* allowed zero */
2026	if ((val & pVM->hwaccm.s.vmx.msr.vmx_entry.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_entry.n.disallowed0)
2027	Log(("Invalid VMX_VMCS_CTRL_ENTRY_CONTROLS: zero\n"));
2028
2029	/* allowed one */
2030	if ((val & ~pVM->hwaccm.s.vmx.msr.vmx_entry.n.allowed1) != 0)
2031	Log(("Invalid VMX_VMCS_CTRL_ENTRY_CONTROLS: one\n"));
2032
2033	rc = VMXReadVMCS(VMX_VMCS_CTRL_EXIT_CONTROLS, &val);
2034	AssertRC(rc);
2035	Log2(("VMX_VMCS_CTRL_EXIT_CONTROLS = %08x\n", val));
2036
2037	/* allowed zero */
2038	if ((val & pVM->hwaccm.s.vmx.msr.vmx_exit.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_exit.n.disallowed0)
2039	Log(("Invalid VMX_VMCS_CTRL_EXIT_CONTROLS: zero\n"));
2040
2041	/* allowed one */
2042	if ((val & ~pVM->hwaccm.s.vmx.msr.vmx_exit.n.allowed1) != 0)
2043	Log(("Invalid VMX_VMCS_CTRL_EXIT_CONTROLS: one\n"));
2044	}
2045	#endif
2046
2047	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2048	pVCpu->hwaccm.s.vmx.VMCSCache.u64TimeEntry = RTTimeNanoTS();
2049	#endif
2050
2051	/* We can jump to this point to resume execution after determining that a VM-exit is innocent.
2052	*/
2053	ResumeExecution:
2054	STAM_STATS({
2055	if (fStatExit2Started) { STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2, y); fStatExit2Started = false; }
2056	if (!fStatEntryStarted) { STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatEntry, x); fStatEntryStarted = true; }
2057	});
2058	AssertMsg(pVCpu->hwaccm.s.idEnteredCpu == RTMpCpuId(),
2059	("Expected %d, I'm %d; cResume=%d exitReason=%RGv exitQualification=%RGv\n",
2060	(int)pVCpu->hwaccm.s.idEnteredCpu, (int)RTMpCpuId(), cResume, exitReason, exitQualification));
2061	Assert(!HWACCMR0SuspendPending());
2062
2063	/* Safety precaution; looping for too long here can have a very bad effect on the host */
2064	if (++cResume > HWACCM_MAX_RESUME_LOOPS)
2065	{
2066	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitMaxResume);
2067	rc = VINF_EM_RAW_INTERRUPT;
2068	goto end;
2069	}
2070
2071	/* Check for irq inhibition due to instruction fusing (sti, mov ss). */
2072	if (VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
2073	{
2074	Log(("VM_FF_INHIBIT_INTERRUPTS at %RGv successor %RGv\n", (RTGCPTR)pCtx->rip, EMGetInhibitInterruptsPC(pVCpu)));
2075	if (pCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
2076	{
2077	/* Note: we intentionally don't clear VM_FF_INHIBIT_INTERRUPTS here.
2078	* Before we are able to execute this instruction in raw mode (iret to guest code) an external interrupt might
2079	* force a world switch again. Possibly allowing a guest interrupt to be dispatched in the process. This could
2080	* break the guest. Sounds very unlikely, but such timing sensitive problems are not as rare as you might think.
2081	*/
2082	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2083	/* Irq inhibition is no longer active; clear the corresponding VMX state. */
2084	rc = VMXWriteVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, 0);
2085	AssertRC(rc);
2086	}
2087	}
2088	else
2089	{
2090	/* Irq inhibition is no longer active; clear the corresponding VMX state. */
2091	rc = VMXWriteVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, 0);
2092	AssertRC(rc);
2093	}
2094
2095	/* Check for pending actions that force us to go back to ring 3. */
2096	if ( VM_FF_ISPENDING(pVM, VM_FF_HWACCM_TO_R3_MASK)
2097	\|\| VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_HWACCM_TO_R3_MASK))
2098	{
2099	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
2100	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatSwitchToR3);
2101	rc = RT_UNLIKELY(VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
2102	goto end;
2103	}
2104	/* Pending request packets might contain actions that need immediate attention, such as pending hardware interrupts. */
2105	if ( VM_FF_ISPENDING(pVM, VM_FF_REQUEST)
2106	\|\| VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_REQUEST))
2107	{
2108	rc = VINF_EM_PENDING_REQUEST;
2109	goto end;
2110	}
2111
2112	/* When external interrupts are pending, we should exit the VM when IF is set. */
2113	/* Note! After VM_FF_INHIBIT_INTERRUPTS check!!! */
2114	rc = VMXR0CheckPendingInterrupt(pVM, pVCpu, pCtx);
2115	if (RT_FAILURE(rc))
2116	goto end;
2117
2118	/** @todo check timers?? */
2119
2120	/* TPR caching using CR8 is only available in 64 bits mode */
2121	/* Note the 32 bits exception for AMD (X86_CPUID_AMD_FEATURE_ECX_CR8L), but that appears missing in Intel CPUs */
2122	/* Note: we can't do this in LoadGuestState as PDMApicGetTPR can jump back to ring 3 (lock)!!!!! */
2123	/**
2124	* @todo reduce overhead
2125	*/
2126	if ( (pCtx->msrEFER & MSR_K6_EFER_LMA)
2127	&& pVM->hwaccm.s.vmx.pAPIC)
2128	{
2129	/* TPR caching in CR8 */
2130	uint8_t u8TPR;
2131	bool fPending;
2132
2133	int rc = PDMApicGetTPR(pVM, &u8TPR, &fPending);
2134	AssertRC(rc);
2135	/* The TPR can be found at offset 0x80 in the APIC mmio page. */
2136	pVM->hwaccm.s.vmx.pAPIC[0x80] = u8TPR << 4; /* bits 7-4 contain the task priority */
2137
2138	/* Two options here:
2139	* - external interrupt pending, but masked by the TPR value.
2140	* -> a CR8 update that lower the current TPR value should cause an exit
2141	* - no pending interrupts
2142	* -> We don't need to be explicitely notified. There are enough world switches for detecting pending interrupts.
2143	*/
2144	rc = VMXWriteVMCS(VMX_VMCS_CTRL_TPR_THRESHOLD, (fPending) ? u8TPR : 0);
2145	AssertRC(rc);
2146
2147	/* Always sync back the TPR; we should optimize this though / /* @todo optimize TPR sync. */
2148	fSyncTPR = true;
2149	}
2150
2151	#if defined(HWACCM_VTX_WITH_EPT) && defined(LOG_ENABLED)
2152	if ( pVM->hwaccm.s.fNestedPaging
2153	# ifdef HWACCM_VTX_WITH_VPID
2154	\|\| pVM->hwaccm.s.vmx.fVPID
2155	# endif /* HWACCM_VTX_WITH_VPID */
2156	)
2157	{
2158	pCpu = HWACCMR0GetCurrentCpu();
2159	if ( pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu
2160	\|\| pVCpu->hwaccm.s.cTLBFlushes != pCpu->cTLBFlushes)
2161	{
2162	if (pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu)
2163	Log(("Force TLB flush due to rescheduling to a different cpu (%d vs %d)\n", pVCpu->hwaccm.s.idLastCpu, pCpu->idCpu));
2164	else
2165	Log(("Force TLB flush due to changed TLB flush count (%x vs %x)\n", pVCpu->hwaccm.s.cTLBFlushes, pCpu->cTLBFlushes));
2166	}
2167	if (pCpu->fFlushTLB)
2168	Log(("Force TLB flush: first time cpu %d is used -> flush\n", pCpu->idCpu));
2169	else
2170	if (pVCpu->hwaccm.s.fForceTLBFlush)
2171	LogFlow(("Manual TLB flush\n"));
2172	}
2173	#endif
2174	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
2175	PGMDynMapFlushAutoSet(pVCpu);
2176	#endif
2177
2178	/*
2179	* NOTE: DO NOT DO ANYTHING AFTER THIS POINT THAT MIGHT JUMP BACK TO RING 3!
2180	* (until the actual world switch)
2181	*/
2182	#ifdef VBOX_STRICT
2183	idCpuCheck = RTMpCpuId();
2184	#endif
2185	#ifdef LOG_LOGGING
2186	VMMR0LogFlushDisable(pVCpu);
2187	#endif
2188	/* Save the host state first. */
2189	rc = VMXR0SaveHostState(pVM, pVCpu);
2190	if (rc != VINF_SUCCESS)
2191	goto end;
2192	/* Load the guest state */
2193	rc = VMXR0LoadGuestState(pVM, pVCpu, pCtx);
2194	if (rc != VINF_SUCCESS)
2195	goto end;
2196
2197	/* Disable interrupts to make sure a poke will interrupt execution.
2198	* This must be done before we check for TLB flushes; TLB shootdowns rely on this.
2199	*/
2200	uOldEFlags = ASMIntDisableFlags();
2201	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
2202
2203	/* Deal with tagged TLB setup and invalidation. */
2204	pVM->hwaccm.s.vmx.pfnSetupTaggedTLB(pVM, pVCpu);
2205
2206	/* Non-register state Guest Context */
2207	/** @todo change me according to cpu state */
2208	rc = VMXWriteVMCS(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_CMS_GUEST_ACTIVITY_ACTIVE);
2209	AssertRC(rc);
2210
2211	STAM_STATS({ STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatEntry, x); fStatEntryStarted = false; });
2212
2213	/* Manual save and restore:
2214	* - General purpose registers except RIP, RSP
2215	*
2216	* Trashed:
2217	* - CR2 (we don't care)
2218	* - LDTR (reset to 0)
2219	* - DRx (presumably not changed at all)
2220	* - DR7 (reset to 0x400)
2221	* - EFLAGS (reset to RT_BIT(1); not relevant)
2222	*
2223	*/
2224
2225
2226	/* All done! Let's start VM execution. */
2227	STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatInGC, z);
2228	#ifdef VBOX_STRICT
2229	Assert(idCpuCheck == RTMpCpuId());
2230	#endif
2231
2232	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2233	pVCpu->hwaccm.s.vmx.VMCSCache.cResume = cResume;
2234	pVCpu->hwaccm.s.vmx.VMCSCache.u64TimeSwitch = RTTimeNanoTS();
2235	#endif
2236
2237	TMNotifyStartOfExecution(pVCpu);
2238	rc = pVCpu->hwaccm.s.vmx.pfnStartVM(pVCpu->hwaccm.s.fResumeVM, pCtx, &pVCpu->hwaccm.s.vmx.VMCSCache, pVM, pVCpu);
2239	TMNotifyEndOfExecution(pVCpu);
2240	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED);
2241	ASMSetFlags(uOldEFlags);
2242
2243	AssertMsg(!pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries, ("pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries=%d\n", pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries));
2244
2245	/* In case we execute a goto ResumeExecution later on. */
2246	pVCpu->hwaccm.s.fResumeVM = true;
2247	pVCpu->hwaccm.s.fForceTLBFlush = false;
2248
2249	/*
2250	* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
2251	* IMPORTANT: WE CAN'T DO ANY LOGGING OR OPERATIONS THAT CAN DO A LONGJMP BACK TO RING 3 BEFORE WE'VE SYNCED BACK (MOST OF) THE GUEST STATE
2252	* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
2253	*/
2254	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatInGC, z);
2255	STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatExit1, v);
2256
2257	if (rc != VINF_SUCCESS)
2258	{
2259	VMXR0ReportWorldSwitchError(pVM, pVCpu, rc, pCtx);
2260	goto end;
2261	}
2262
2263	/* Success. Query the guest state and figure out what has happened. */
2264
2265	/* Investigate why there was a VM-exit. */
2266	rc = VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_REASON, &exitReason);
2267	STAM_COUNTER_INC(&pVCpu->hwaccm.s.paStatExitReasonR0[exitReason & MASK_EXITREASON_STAT]);
2268
2269	exitReason &= 0xffff; /* bit 0-15 contain the exit code. */
2270	rc \|= VMXReadCachedVMCS(VMX_VMCS32_RO_VM_INSTR_ERROR, &instrError);
2271	rc \|= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &cbInstr);
2272	rc \|= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &intInfo);
2273	/* might not be valid; depends on VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID. */
2274	rc \|= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERRCODE, &errCode);
2275	rc \|= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INSTR_INFO, &instrInfo);
2276	rc \|= VMXReadCachedVMCS(VMX_VMCS_RO_EXIT_QUALIFICATION, &exitQualification);
2277	AssertRC(rc);
2278
2279	/* Sync back the guest state */
2280	rc = VMXR0SaveGuestState(pVM, pVCpu, pCtx);
2281	AssertRC(rc);
2282
2283	/* Note! NOW IT'S SAFE FOR LOGGING! */
2284	#ifdef LOG_LOGGING
2285	VMMR0LogFlushEnable(pVCpu);
2286	#endif
2287	Log2(("Raw exit reason %08x\n", exitReason));
2288
2289	/* Check if an injected event was interrupted prematurely. */
2290	rc = VMXReadCachedVMCS(VMX_VMCS32_RO_IDT_INFO, &val);
2291	AssertRC(rc);
2292	pVCpu->hwaccm.s.Event.intInfo = VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(val);
2293	if ( VMX_EXIT_INTERRUPTION_INFO_VALID(pVCpu->hwaccm.s.Event.intInfo)
2294	/* Ignore 'int xx' as they'll be restarted anyway. */
2295	&& VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hwaccm.s.Event.intInfo) != VMX_EXIT_INTERRUPTION_INFO_TYPE_SW
2296	/* Ignore software exceptions (such as int3) as they're reoccur when we restart the instruction anyway. */
2297	&& VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hwaccm.s.Event.intInfo) != VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT)
2298	{
2299	Assert(!pVCpu->hwaccm.s.Event.fPending);
2300	pVCpu->hwaccm.s.Event.fPending = true;
2301	/* Error code present? */
2302	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVCpu->hwaccm.s.Event.intInfo))
2303	{
2304	rc = VMXReadCachedVMCS(VMX_VMCS32_RO_IDT_ERRCODE, &val);
2305	AssertRC(rc);
2306	pVCpu->hwaccm.s.Event.errCode = val;
2307	Log(("Pending inject %RX64 at %RGv exit=%08x intInfo=%08x exitQualification=%RGv pending error=%RX64\n", pVCpu->hwaccm.s.Event.intInfo, (RTGCPTR)pCtx->rip, exitReason, intInfo, exitQualification, val));
2308	}
2309	else
2310	{
2311	Log(("Pending inject %RX64 at %RGv exit=%08x intInfo=%08x exitQualification=%RGv\n", pVCpu->hwaccm.s.Event.intInfo, (RTGCPTR)pCtx->rip, exitReason, intInfo, exitQualification));
2312	pVCpu->hwaccm.s.Event.errCode = 0;
2313	}
2314	}
2315	#ifdef VBOX_STRICT
2316	else
2317	if ( VMX_EXIT_INTERRUPTION_INFO_VALID(pVCpu->hwaccm.s.Event.intInfo)
2318	/* Ignore software exceptions (such as int3) as they're reoccur when we restart the instruction anyway. */
2319	&& VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hwaccm.s.Event.intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT)
2320	{
2321	Log(("Ignore pending inject %RX64 at %RGv exit=%08x intInfo=%08x exitQualification=%RGv\n", pVCpu->hwaccm.s.Event.intInfo, (RTGCPTR)pCtx->rip, exitReason, intInfo, exitQualification));
2322	}
2323
2324	if (exitReason == VMX_EXIT_ERR_INVALID_GUEST_STATE)
2325	HWACCMDumpRegs(pVM, pVCpu, pCtx);
2326	#endif
2327
2328	Log2(("E%d: New EIP=%RGv\n", exitReason, (RTGCPTR)pCtx->rip));
2329	Log2(("Exit reason %d, exitQualification %RGv\n", (uint32_t)exitReason, exitQualification));
2330	Log2(("instrInfo=%d instrError=%d instr length=%d\n", (uint32_t)instrInfo, (uint32_t)instrError, (uint32_t)cbInstr));
2331	Log2(("Interruption error code %d\n", (uint32_t)errCode));
2332	Log2(("IntInfo = %08x\n", (uint32_t)intInfo));
2333
2334	if (fSyncTPR)
2335	{
2336	rc = PDMApicSetTPR(pVM, pVM->hwaccm.s.vmx.pAPIC[0x80] >> 4);
2337	AssertRC(rc);
2338	}
2339
2340	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit1, v);
2341	STAM_STATS({ STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatExit2, y); fStatExit2Started = true; });
2342
2343	/* Some cases don't need a complete resync of the guest CPU state; handle them here. */
2344	switch (exitReason)
2345	{
2346	case VMX_EXIT_EXCEPTION: /* 0 Exception or non-maskable interrupt (NMI). */
2347	case VMX_EXIT_EXTERNAL_IRQ: /* 1 External interrupt. */
2348	{
2349	uint32_t vector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(intInfo);
2350
2351	if (!VMX_EXIT_INTERRUPTION_INFO_VALID(intInfo))
2352	{
2353	Assert(exitReason == VMX_EXIT_EXTERNAL_IRQ);
2354	/* External interrupt; leave to allow it to be dispatched again. */
2355	rc = VINF_EM_RAW_INTERRUPT;
2356	break;
2357	}
2358	STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2359	switch (VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo))
2360	{
2361	case VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI: /* Non-maskable interrupt. */
2362	/* External interrupt; leave to allow it to be dispatched again. */
2363	rc = VINF_EM_RAW_INTERRUPT;
2364	break;
2365
2366	case VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT: /* External hardware interrupt. */
2367	AssertFailed(); /* can't come here; fails the first check. */
2368	break;
2369
2370	case VMX_EXIT_INTERRUPTION_INFO_TYPE_DBEXCPT: /* Unknown why we get this type for #DB */
2371	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT: /* Software exception. (#BP or #OF) */
2372	Assert(vector == 1 \|\| vector == 3 \|\| vector == 4);
2373	/* no break */
2374	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT: /* Hardware exception. */
2375	Log2(("Hardware/software interrupt %d\n", vector));
2376	switch (vector)
2377	{
2378	case X86_XCPT_NM:
2379	{
2380	Log(("#NM fault at %RGv error code %x\n", (RTGCPTR)pCtx->rip, errCode));
2381
2382	/** @todo don't intercept #NM exceptions anymore when we've activated the guest FPU state. */
2383	/* If we sync the FPU/XMM state on-demand, then we can continue execution as if nothing has happened. */
2384	rc = CPUMR0LoadGuestFPU(pVM, pVCpu, pCtx);
2385	if (rc == VINF_SUCCESS)
2386	{
2387	Assert(CPUMIsGuestFPUStateActive(pVCpu));
2388
2389	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitShadowNM);
2390
2391	/* Continue execution. */
2392	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR0;
2393
2394	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2395	goto ResumeExecution;
2396	}
2397
2398	Log(("Forward #NM fault to the guest\n"));
2399	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestNM);
2400	rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, 0);
2401	AssertRC(rc);
2402	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2403	goto ResumeExecution;
2404	}
2405
2406	case X86_XCPT_PF: /* Page fault */
2407	{
2408	#ifdef DEBUG
2409	if (pVM->hwaccm.s.fNestedPaging)
2410	{ /* A genuine pagefault.
2411	* Forward the trap to the guest by injecting the exception and resuming execution.
2412	*/
2413	Log(("Guest page fault at %RGv cr2=%RGv error code %x rsp=%RGv\n", (RTGCPTR)pCtx->rip, exitQualification, errCode, (RTGCPTR)pCtx->rsp));
2414
2415	Assert(CPUMIsGuestInPagedProtectedModeEx(pCtx));
2416
2417	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestPF);
2418
2419	/* Now we must update CR2. */
2420	pCtx->cr2 = exitQualification;
2421	rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
2422	AssertRC(rc);
2423
2424	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2425	goto ResumeExecution;
2426	}
2427	#endif
2428	Assert(!pVM->hwaccm.s.fNestedPaging);
2429
2430	Log2(("Page fault at %RGv error code %x\n", exitQualification, errCode));
2431	/* Exit qualification contains the linear address of the page fault. */
2432	TRPMAssertTrap(pVCpu, X86_XCPT_PF, TRPM_TRAP);
2433	TRPMSetErrorCode(pVCpu, errCode);
2434	TRPMSetFaultAddress(pVCpu, exitQualification);
2435
2436	/* Forward it to our trap handler first, in case our shadow pages are out of sync. */
2437	rc = PGMTrap0eHandler(pVCpu, errCode, CPUMCTX2CORE(pCtx), (RTGCPTR)exitQualification);
2438	Log2(("PGMTrap0eHandler %RGv returned %Rrc\n", (RTGCPTR)pCtx->rip, rc));
2439	if (rc == VINF_SUCCESS)
2440	{ /* We've successfully synced our shadow pages, so let's just continue execution. */
2441	Log2(("Shadow page fault at %RGv cr2=%RGv error code %x\n", (RTGCPTR)pCtx->rip, exitQualification ,errCode));
2442	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitShadowPF);
2443
2444	TRPMResetTrap(pVCpu);
2445
2446	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2447	goto ResumeExecution;
2448	}
2449	else
2450	if (rc == VINF_EM_RAW_GUEST_TRAP)
2451	{ /* A genuine pagefault.
2452	* Forward the trap to the guest by injecting the exception and resuming execution.
2453	*/
2454	Log2(("Forward page fault to the guest\n"));
2455
2456	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestPF);
2457	/* The error code might have been changed. */
2458	errCode = TRPMGetErrorCode(pVCpu);
2459
2460	TRPMResetTrap(pVCpu);
2461
2462	/* Now we must update CR2. */
2463	pCtx->cr2 = exitQualification;
2464	rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
2465	AssertRC(rc);
2466
2467	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2468	goto ResumeExecution;
2469	}
2470	#ifdef VBOX_STRICT
2471	if (rc != VINF_EM_RAW_EMULATE_INSTR && rc != VINF_EM_RAW_EMULATE_IO_BLOCK)
2472	Log2(("PGMTrap0eHandler failed with %d\n", rc));
2473	#endif
2474	/* Need to go back to the recompiler to emulate the instruction. */
2475	TRPMResetTrap(pVCpu);
2476	break;
2477	}
2478
2479	case X86_XCPT_MF: /* Floating point exception. */
2480	{
2481	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestMF);
2482	if (!(pCtx->cr0 & X86_CR0_NE))
2483	{
2484	/* old style FPU error reporting needs some extra work. */
2485	/** @todo don't fall back to the recompiler, but do it manually. */
2486	rc = VINF_EM_RAW_EMULATE_INSTR;
2487	break;
2488	}
2489	Log(("Trap %x at %04X:%RGv\n", vector, pCtx->cs, (RTGCPTR)pCtx->rip));
2490	rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
2491	AssertRC(rc);
2492
2493	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2494	goto ResumeExecution;
2495	}
2496
2497	case X86_XCPT_DB: /* Debug exception. */
2498	{
2499	uint64_t uDR6;
2500
2501	/* DR6, DR7.GD and IA32_DEBUGCTL.LBR are not updated yet.
2502	*
2503	* Exit qualification bits:
2504	* 3:0 B0-B3 which breakpoint condition was met
2505	* 12:4 Reserved (0)
2506	* 13 BD - debug register access detected
2507	* 14 BS - single step execution or branch taken
2508	* 63:15 Reserved (0)
2509	*/
2510	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestDB);
2511
2512	/* Note that we don't support guest and host-initiated debugging at the same time. */
2513	Assert(DBGFIsStepping(pVCpu) \|\| CPUMIsGuestInRealModeEx(pCtx));
2514
2515	uDR6 = X86_DR6_INIT_VAL;
2516	uDR6 \|= (exitQualification & (X86_DR6_B0\|X86_DR6_B1\|X86_DR6_B2\|X86_DR6_B3\|X86_DR6_BD\|X86_DR6_BS));
2517	rc = DBGFRZTrap01Handler(pVM, pVCpu, CPUMCTX2CORE(pCtx), uDR6);
2518	if (rc == VINF_EM_RAW_GUEST_TRAP)
2519	{
2520	/** @todo this isn't working, but we'll never get here normally. */
2521
2522	/* Update DR6 here. */
2523	pCtx->dr[6] = uDR6;
2524
2525	/* X86_DR7_GD will be cleared if drx accesses should be trapped inside the guest. */
2526	pCtx->dr[7] &= ~X86_DR7_GD;
2527
2528	/* Paranoia. */
2529	pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
2530	pCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* must be zero */
2531	pCtx->dr[7] \|= 0x400; /* must be one */
2532
2533	/* Resync DR7 */
2534	rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_DR7, pCtx->dr[7]);
2535	AssertRC(rc);
2536
2537	Log(("Trap %x (debug) at %RGv exit qualification %RX64\n", vector, (RTGCPTR)pCtx->rip, exitQualification));
2538	rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
2539	AssertRC(rc);
2540
2541	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2542	goto ResumeExecution;
2543	}
2544	/* Return to ring 3 to deal with the debug exit code. */
2545	break;
2546	}
2547
2548	case X86_XCPT_BP: /* Breakpoint. */
2549	{
2550	rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pCtx));
2551	if (rc == VINF_EM_RAW_GUEST_TRAP)
2552	{
2553	Log(("Guest #BP at %04x:%RGv\n", pCtx->cs, pCtx->rip));
2554	rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
2555	AssertRC(rc);
2556	goto ResumeExecution;
2557	}
2558	if (rc == VINF_SUCCESS)
2559	goto ResumeExecution;
2560	Log(("Debugger BP at %04x:%RGv (rc=%Rrc)\n", pCtx->cs, pCtx->rip, rc));
2561	break;
2562	}
2563
2564	case X86_XCPT_GP: /* General protection failure exception.*/
2565	{
2566	uint32_t cbOp;
2567	uint32_t cbSize;
2568	DISCPUSTATE Cpu;
2569
2570	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestGP);
2571	#ifdef VBOX_STRICT
2572	if (!CPUMIsGuestInRealModeEx(pCtx))
2573	{
2574	Log(("Trap %x at %04X:%RGv errorCode=%x\n", vector, pCtx->cs, (RTGCPTR)pCtx->rip, errCode));
2575	rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
2576	AssertRC(rc);
2577	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2578	goto ResumeExecution;
2579	}
2580	#endif
2581	Assert(CPUMIsGuestInRealModeEx(pCtx));
2582
2583	LogFlow(("Real mode X86_XCPT_GP instruction emulation at %RGv\n", (RTGCPTR)pCtx->rip));
2584
2585	rc = EMInterpretDisasOne(pVM, pVCpu, CPUMCTX2CORE(pCtx), &Cpu, &cbOp);
2586	if (RT_SUCCESS(rc))
2587	{
2588	bool fUpdateRIP = true;
2589
2590	Assert(cbOp == Cpu.opsize);
2591	switch (Cpu.pCurInstr->opcode)
2592	{
2593	case OP_CLI:
2594	pCtx->eflags.Bits.u1IF = 0;
2595	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCli);
2596	break;
2597
2598	case OP_STI:
2599	pCtx->eflags.Bits.u1IF = 1;
2600	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitSti);
2601	break;
2602
2603	case OP_HLT:
2604	fUpdateRIP = false;
2605	rc = VINF_EM_HALT;
2606	pCtx->rip += Cpu.opsize;
2607	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitHlt);
2608	break;
2609
2610	case OP_POPF:
2611	{
2612	RTGCPTR GCPtrStack;
2613	uint32_t cbParm;
2614	uint32_t uMask;
2615	X86EFLAGS eflags;
2616
2617	if (Cpu.prefix & PREFIX_OPSIZE)
2618	{
2619	cbParm = 4;
2620	uMask = 0xffffffff;
2621	}
2622	else
2623	{
2624	cbParm = 2;
2625	uMask = 0xffff;
2626	}
2627
2628	rc = SELMToFlatEx(pVM, DIS_SELREG_SS, CPUMCTX2CORE(pCtx), pCtx->esp & uMask, 0, &GCPtrStack);
2629	if (RT_FAILURE(rc))
2630	{
2631	rc = VERR_EM_INTERPRETER;
2632	break;
2633	}
2634	eflags.u = 0;
2635	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &eflags.u, cbParm);
2636	if (RT_FAILURE(rc))
2637	{
2638	rc = VERR_EM_INTERPRETER;
2639	break;
2640	}
2641	LogFlow(("POPF %x -> %RGv mask=%x\n", eflags.u, pCtx->rsp, uMask));
2642	pCtx->eflags.u = (pCtx->eflags.u & ~(X86_EFL_POPF_BITS & uMask)) \| (eflags.u & X86_EFL_POPF_BITS & uMask);
2643	/* RF cleared when popped in real mode; see pushf description in AMD manual. */
2644	pCtx->eflags.Bits.u1RF = 0;
2645	pCtx->esp += cbParm;
2646	pCtx->esp &= uMask;
2647
2648	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitPopf);
2649	break;
2650	}
2651
2652	case OP_PUSHF:
2653	{
2654	RTGCPTR GCPtrStack;
2655	uint32_t cbParm;
2656	uint32_t uMask;
2657	X86EFLAGS eflags;
2658
2659	if (Cpu.prefix & PREFIX_OPSIZE)
2660	{
2661	cbParm = 4;
2662	uMask = 0xffffffff;
2663	}
2664	else
2665	{
2666	cbParm = 2;
2667	uMask = 0xffff;
2668	}
2669
2670	rc = SELMToFlatEx(pVM, DIS_SELREG_SS, CPUMCTX2CORE(pCtx), (pCtx->esp - cbParm) & uMask, 0, &GCPtrStack);
2671	if (RT_FAILURE(rc))
2672	{
2673	rc = VERR_EM_INTERPRETER;
2674	break;
2675	}
2676	eflags = pCtx->eflags;
2677	/* RF & VM cleared when pushed in real mode; see pushf description in AMD manual. */
2678	eflags.Bits.u1RF = 0;
2679	eflags.Bits.u1VM = 0;
2680
2681	rc = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &eflags.u, cbParm);
2682	if (RT_FAILURE(rc))
2683	{
2684	rc = VERR_EM_INTERPRETER;
2685	break;
2686	}
2687	LogFlow(("PUSHF %x -> %RGv\n", eflags.u, GCPtrStack));
2688	pCtx->esp -= cbParm;
2689	pCtx->esp &= uMask;
2690	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitPushf);
2691	break;
2692	}
2693
2694	case OP_IRET:
2695	{
2696	RTGCPTR GCPtrStack;
2697	uint32_t uMask = 0xffff;
2698	uint16_t aIretFrame[3];
2699
2700	if (Cpu.prefix & (PREFIX_OPSIZE \| PREFIX_ADDRSIZE))
2701	{
2702	rc = VERR_EM_INTERPRETER;
2703	break;
2704	}
2705
2706	rc = SELMToFlatEx(pVM, DIS_SELREG_SS, CPUMCTX2CORE(pCtx), pCtx->esp & uMask, 0, &GCPtrStack);
2707	if (RT_FAILURE(rc))
2708	{
2709	rc = VERR_EM_INTERPRETER;
2710	break;
2711	}
2712	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
2713	if (RT_FAILURE(rc))
2714	{
2715	rc = VERR_EM_INTERPRETER;
2716	break;
2717	}
2718	pCtx->ip = aIretFrame[0];
2719	pCtx->cs = aIretFrame[1];
2720	pCtx->csHid.u64Base = pCtx->cs << 4;
2721	pCtx->eflags.u = (pCtx->eflags.u & ~(X86_EFL_POPF_BITS & uMask)) \| (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
2722	pCtx->sp += sizeof(aIretFrame);
2723
2724	LogFlow(("iret to %04x:%x\n", pCtx->cs, pCtx->ip));
2725	fUpdateRIP = false;
2726	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIret);
2727	break;
2728	}
2729
2730	case OP_INT:
2731	{
2732	RTGCUINTPTR intInfo;
2733
2734	LogFlow(("Realmode: INT %x\n", Cpu.param1.parval & 0xff));
2735	intInfo = Cpu.param1.parval & 0xff;
2736	intInfo \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
2737	intInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
2738
2739	rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, intInfo, cbOp, 0);
2740	AssertRC(rc);
2741	fUpdateRIP = false;
2742	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInt);
2743	break;
2744	}
2745
2746	case OP_INTO:
2747	{
2748	if (pCtx->eflags.Bits.u1OF)
2749	{
2750	RTGCUINTPTR intInfo;
2751
2752	LogFlow(("Realmode: INTO\n"));
2753	intInfo = X86_XCPT_OF;
2754	intInfo \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
2755	intInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
2756
2757	rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, intInfo, cbOp, 0);
2758	AssertRC(rc);
2759	fUpdateRIP = false;
2760	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInt);
2761	}
2762	break;
2763	}
2764
2765	case OP_INT3:
2766	{
2767	RTGCUINTPTR intInfo;
2768
2769	LogFlow(("Realmode: INT 3\n"));
2770	intInfo = 3;
2771	intInfo \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
2772	intInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
2773
2774	rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, intInfo, cbOp, 0);
2775	AssertRC(rc);
2776	fUpdateRIP = false;
2777	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInt);
2778	break;
2779	}
2780
2781	default:
2782	rc = EMInterpretInstructionCPU(pVM, pVCpu, &Cpu, CPUMCTX2CORE(pCtx), 0, &cbSize);
2783	break;
2784	}
2785
2786	if (rc == VINF_SUCCESS)
2787	{
2788	if (fUpdateRIP)
2789	pCtx->rip += cbOp; /* Move on to the next instruction. */
2790
2791	/* lidt, lgdt can end up here. In the future crx changes as well. Just reload the whole context to be done with it. */
2792	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_ALL;
2793
2794	/* Only resume if successful. */
2795	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2796	goto ResumeExecution;
2797	}
2798	}
2799	else
2800	rc = VERR_EM_INTERPRETER;
2801
2802	AssertMsg(rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_EM_HALT, ("Unexpected rc=%Rrc\n", rc));
2803	break;
2804	}
2805
2806	#ifdef VBOX_STRICT
2807	case X86_XCPT_DE: /* Divide error. */
2808	case X86_XCPT_UD: /* Unknown opcode exception. */
2809	case X86_XCPT_SS: /* Stack segment exception. */
2810	case X86_XCPT_NP: /* Segment not present exception. */
2811	{
2812	switch(vector)
2813	{
2814	case X86_XCPT_DE:
2815	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestDE);
2816	break;
2817	case X86_XCPT_UD:
2818	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestUD);
2819	break;
2820	case X86_XCPT_SS:
2821	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestSS);
2822	break;
2823	case X86_XCPT_NP:
2824	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestNP);
2825	break;
2826	}
2827
2828	Log(("Trap %x at %04X:%RGv\n", vector, pCtx->cs, (RTGCPTR)pCtx->rip));
2829	rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
2830	AssertRC(rc);
2831
2832	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2833	goto ResumeExecution;
2834	}
2835	#endif
2836	default:
2837	#ifdef HWACCM_VMX_EMULATE_REALMODE
2838	if (CPUMIsGuestInRealModeEx(pCtx))
2839	{
2840	Log(("Real Mode Trap %x at %04x:%04X error code %x\n", vector, pCtx->cs, pCtx->eip, errCode));
2841	rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
2842	AssertRC(rc);
2843
2844	/* Go back to ring 3 in case of a triple fault. */
2845	if ( vector == X86_XCPT_DF
2846	&& rc == VINF_EM_RESET)
2847	break;
2848
2849	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2850	goto ResumeExecution;
2851	}
2852	#endif
2853	AssertMsgFailed(("Unexpected vm-exit caused by exception %x\n", vector));
2854	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
2855	break;
2856	} /* switch (vector) */
2857
2858	break;
2859
2860	default:
2861	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_CODE;
2862	AssertMsgFailed(("Unexpected interuption code %x\n", intInfo));
2863	break;
2864	}
2865
2866	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2867	break;
2868	}
2869
2870	case VMX_EXIT_EPT_VIOLATION: /* 48 EPT violation. An attempt to access memory with a guest-physical address was disallowed by the configuration of the EPT paging structures. */
2871	{
2872	RTGCPHYS GCPhys;
2873
2874	Assert(pVM->hwaccm.s.fNestedPaging);
2875
2876	rc = VMXReadVMCS64(VMX_VMCS_EXIT_PHYS_ADDR_FULL, &GCPhys);
2877	AssertRC(rc);
2878	Assert(((exitQualification >> 7) & 3) != 2);
2879
2880	/* Determine the kind of violation. */
2881	errCode = 0;
2882	if (exitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
2883	errCode \|= X86_TRAP_PF_ID;
2884
2885	if (exitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
2886	errCode \|= X86_TRAP_PF_RW;
2887
2888	/* If the page is present, then it's a page level protection fault. */
2889	if (exitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
2890	errCode \|= X86_TRAP_PF_P;
2891
2892	Log(("EPT Page fault %x at %RGp error code %x\n", (uint32_t)exitQualification, GCPhys, errCode));
2893
2894	/* GCPhys contains the guest physical address of the page fault. */
2895	TRPMAssertTrap(pVCpu, X86_XCPT_PF, TRPM_TRAP);
2896	TRPMSetErrorCode(pVCpu, errCode);
2897	TRPMSetFaultAddress(pVCpu, GCPhys);
2898
2899	/* Handle the pagefault trap for the nested shadow table. */
2900	rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, errCode, CPUMCTX2CORE(pCtx), GCPhys);
2901	Log2(("PGMR0Trap0eHandlerNestedPaging %RGv returned %Rrc\n", (RTGCPTR)pCtx->rip, rc));
2902	if (rc == VINF_SUCCESS)
2903	{ /* We've successfully synced our shadow pages, so let's just continue execution. */
2904	Log2(("Shadow page fault at %RGv cr2=%RGp error code %x\n", (RTGCPTR)pCtx->rip, exitQualification , errCode));
2905	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitReasonNPF);
2906
2907	TRPMResetTrap(pVCpu);
2908
2909	goto ResumeExecution;
2910	}
2911
2912	#ifdef VBOX_STRICT
2913	if (rc != VINF_EM_RAW_EMULATE_INSTR)
2914	LogFlow(("PGMTrap0eHandlerNestedPaging failed with %d\n", rc));
2915	#endif
2916	/* Need to go back to the recompiler to emulate the instruction. */
2917	TRPMResetTrap(pVCpu);
2918	break;
2919	}
2920
2921	case VMX_EXIT_EPT_MISCONFIG:
2922	{
2923	RTGCPHYS GCPhys;
2924
2925	Assert(pVM->hwaccm.s.fNestedPaging);
2926
2927	rc = VMXReadVMCS64(VMX_VMCS_EXIT_PHYS_ADDR_FULL, &GCPhys);
2928	AssertRC(rc);
2929
2930	Log(("VMX_EXIT_EPT_MISCONFIG for %VGp\n", GCPhys));
2931	break;
2932	}
2933
2934	case VMX_EXIT_IRQ_WINDOW: /* 7 Interrupt window. */
2935	/* Clear VM-exit on IF=1 change. */
2936	LogFlow(("VMX_EXIT_IRQ_WINDOW %RGv pending=%d IF=%d\n", (RTGCPTR)pCtx->rip, VMCPU_FF_ISPENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC\|VMCPU_FF_INTERRUPT_PIC)), pCtx->eflags.Bits.u1IF));
2937	pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_IRQ_WINDOW_EXIT;
2938	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
2939	AssertRC(rc);
2940	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIrqWindow);
2941	goto ResumeExecution; /* we check for pending guest interrupts there */
2942
2943	case VMX_EXIT_WBINVD: /* 54 Guest software attempted to execute WBINVD. (conditional) */
2944	case VMX_EXIT_INVD: /* 13 Guest software attempted to execute INVD. (unconditional) */
2945	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInvd);
2946	/* Skip instruction and continue directly. */
2947	pCtx->rip += cbInstr;
2948	/* Continue execution.*/
2949	goto ResumeExecution;
2950
2951	case VMX_EXIT_CPUID: /* 10 Guest software attempted to execute CPUID. */
2952	{
2953	Log2(("VMX: Cpuid %x\n", pCtx->eax));
2954	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCpuid);
2955	rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pCtx));
2956	if (rc == VINF_SUCCESS)
2957	{
2958	/* Update EIP and continue execution. */
2959	Assert(cbInstr == 2);
2960	pCtx->rip += cbInstr;
2961	goto ResumeExecution;
2962	}
2963	AssertMsgFailed(("EMU: cpuid failed with %Rrc\n", rc));
2964	rc = VINF_EM_RAW_EMULATE_INSTR;
2965	break;
2966	}
2967
2968	case VMX_EXIT_RDPMC: /* 15 Guest software attempted to execute RDPMC. */
2969	{
2970	Log2(("VMX: Rdpmc %x\n", pCtx->ecx));
2971	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitRdpmc);
2972	rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pCtx));
2973	if (rc == VINF_SUCCESS)
2974	{
2975	/* Update EIP and continue execution. */
2976	Assert(cbInstr == 2);
2977	pCtx->rip += cbInstr;
2978	goto ResumeExecution;
2979	}
2980	rc = VINF_EM_RAW_EMULATE_INSTR;
2981	break;
2982	}
2983
2984	case VMX_EXIT_RDTSC: /* 16 Guest software attempted to execute RDTSC. */
2985	{
2986	Log2(("VMX: Rdtsc\n"));
2987	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitRdtsc);
2988	rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pCtx));
2989	if (rc == VINF_SUCCESS)
2990	{
2991	/* Update EIP and continue execution. */
2992	Assert(cbInstr == 2);
2993	pCtx->rip += cbInstr;
2994	goto ResumeExecution;
2995	}
2996	rc = VINF_EM_RAW_EMULATE_INSTR;
2997	break;
2998	}
2999
3000	case VMX_EXIT_INVPG: /* 14 Guest software attempted to execute INVPG. */
3001	{
3002	Log2(("VMX: invlpg\n"));
3003	Assert(!pVM->hwaccm.s.fNestedPaging);
3004
3005	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInvpg);
3006	rc = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pCtx), exitQualification);
3007	if (rc == VINF_SUCCESS)
3008	{
3009	/* Update EIP and continue execution. */
3010	pCtx->rip += cbInstr;
3011	goto ResumeExecution;
3012	}
3013	AssertMsg(rc == VERR_EM_INTERPRETER, ("EMU: invlpg %RGv failed with %Rrc\n", exitQualification, rc));
3014	break;
3015	}
3016
3017	case VMX_EXIT_RDMSR: /* 31 RDMSR. Guest software attempted to execute RDMSR. */
3018	case VMX_EXIT_WRMSR: /* 32 WRMSR. Guest software attempted to execute WRMSR. */
3019	{
3020	uint32_t cbSize;
3021
3022	STAM_COUNTER_INC((exitReason == VMX_EXIT_RDMSR) ? &pVCpu->hwaccm.s.StatExitRdmsr : &pVCpu->hwaccm.s.StatExitWrmsr);
3023
3024	/* Note: the intel manual claims there's a REX version of RDMSR that's slightly different, so we play safe by completely disassembling the instruction. */
3025	Log2(("VMX: %s\n", (exitReason == VMX_EXIT_RDMSR) ? "rdmsr" : "wrmsr"));
3026	rc = EMInterpretInstruction(pVM, pVCpu, CPUMCTX2CORE(pCtx), 0, &cbSize);
3027	if (rc == VINF_SUCCESS)
3028	{
3029	/* EIP has been updated already. */
3030
3031	/* Only resume if successful. */
3032	goto ResumeExecution;
3033	}
3034	AssertMsg(rc == VERR_EM_INTERPRETER, ("EMU: %s failed with %Rrc\n", (exitReason == VMX_EXIT_RDMSR) ? "rdmsr" : "wrmsr", rc));
3035	break;
3036	}
3037
3038	case VMX_EXIT_CRX_MOVE: /* 28 Control-register accesses. */
3039	{
3040	STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatExit2Sub2, y2);
3041
3042	switch (VMX_EXIT_QUALIFICATION_CRX_ACCESS(exitQualification))
3043	{
3044	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE:
3045	Log2(("VMX: %RGv mov cr%d, x\n", (RTGCPTR)pCtx->rip, VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification)));
3046	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification)]);
3047	rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
3048	VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification),
3049	VMX_EXIT_QUALIFICATION_CRX_GENREG(exitQualification));
3050
3051	switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification))
3052	{
3053	case 0:
3054	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR0 \| HWACCM_CHANGED_GUEST_CR3;
3055	break;
3056	case 2:
3057	break;
3058	case 3:
3059	Assert(!pVM->hwaccm.s.fNestedPaging \|\| !CPUMIsGuestInPagedProtectedModeEx(pCtx));
3060	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR3;
3061	break;
3062	case 4:
3063	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR4;
3064	break;
3065	case 8:
3066	/* CR8 contains the APIC TPR */
3067	Assert(!(pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
3068	break;
3069
3070	default:
3071	AssertFailed();
3072	break;
3073	}
3074	/* Check if a sync operation is pending. */
3075	if ( rc == VINF_SUCCESS /* don't bother if we are going to ring 3 anyway */
3076	&& VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
3077	{
3078	rc = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4, VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
3079	AssertRC(rc);
3080	}
3081	break;
3082
3083	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ:
3084	Log2(("VMX: mov x, crx\n"));
3085	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification)]);
3086
3087	Assert(!pVM->hwaccm.s.fNestedPaging \|\| !CPUMIsGuestInPagedProtectedModeEx(pCtx) \|\| VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification) != USE_REG_CR3);
3088
3089	/* CR8 reads only cause an exit when the TPR shadow feature isn't present. */
3090	Assert(VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification) != 8 \|\| !(pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
3091
3092	rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
3093	VMX_EXIT_QUALIFICATION_CRX_GENREG(exitQualification),
3094	VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification));
3095	break;
3096
3097	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS:
3098	Log2(("VMX: clts\n"));
3099	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCLTS);
3100	rc = EMInterpretCLTS(pVM, pVCpu);
3101	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR0;
3102	break;
3103
3104	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW:
3105	Log2(("VMX: lmsw %x\n", VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(exitQualification)));
3106	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitLMSW);
3107	rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(exitQualification));
3108	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR0;
3109	break;
3110	}
3111
3112	/* Update EIP if no error occurred. */
3113	if (RT_SUCCESS(rc))
3114	pCtx->rip += cbInstr;
3115
3116	if (rc == VINF_SUCCESS)
3117	{
3118	/* Only resume if successful. */
3119	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub2, y2);
3120	goto ResumeExecution;
3121	}
3122	Assert(rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3);
3123	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub2, y2);
3124	break;
3125	}
3126
3127	case VMX_EXIT_DRX_MOVE: /* 29 Debug-register accesses. */
3128	{
3129	if (!DBGFIsStepping(pVCpu))
3130	{
3131	/* Disable drx move intercepts. */
3132	pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
3133	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
3134	AssertRC(rc);
3135
3136	/* Save the host and load the guest debug state. */
3137	rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
3138	AssertRC(rc);
3139
3140	#ifdef VBOX_WITH_STATISTICS
3141	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatDRxContextSwitch);
3142	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
3143	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxWrite);
3144	else
3145	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxRead);
3146	#endif
3147
3148	goto ResumeExecution;
3149	}
3150
3151	/** @todo clear VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT after the first time and restore drx registers afterwards */
3152	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
3153	{
3154	Log2(("VMX: mov drx%d, genreg%d\n", VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification), VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification)));
3155	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxWrite);
3156	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
3157	VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification),
3158	VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification));
3159	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_DEBUG;
3160	Log2(("DR7=%08x\n", pCtx->dr[7]));
3161	}
3162	else
3163	{
3164	Log2(("VMX: mov x, drx\n"));
3165	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxRead);
3166	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
3167	VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification),
3168	VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification));
3169	}
3170	/* Update EIP if no error occurred. */
3171	if (RT_SUCCESS(rc))
3172	pCtx->rip += cbInstr;
3173
3174	if (rc == VINF_SUCCESS)
3175	{
3176	/* Only resume if successful. */
3177	goto ResumeExecution;
3178	}
3179	Assert(rc == VERR_EM_INTERPRETER);
3180	break;
3181	}
3182
3183	/* Note: We'll get a #GP if the IO instruction isn't allowed (IOPL or TSS bitmap); no need to double check. */
3184	case VMX_EXIT_PORT_IO: /* 30 I/O instruction. */
3185	{
3186	STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3187	uint32_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(exitQualification);
3188	uint32_t uPort;
3189	bool fIOWrite = (VMX_EXIT_QUALIFICATION_IO_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
3190
3191	/** @todo necessary to make the distinction? */
3192	if (VMX_EXIT_QUALIFICATION_IO_ENCODING(exitQualification) == VMX_EXIT_QUALIFICATION_IO_ENCODING_DX)
3193	{
3194	uPort = pCtx->edx & 0xffff;
3195	}
3196	else
3197	uPort = VMX_EXIT_QUALIFICATION_IO_PORT(exitQualification); /* Immediate encoding. */
3198
3199	/* paranoia */
3200	if (RT_UNLIKELY(uIOWidth == 2 \|\| uIOWidth >= 4))
3201	{
3202	rc = fIOWrite ? VINF_IOM_HC_IOPORT_WRITE : VINF_IOM_HC_IOPORT_READ;
3203	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3204	break;
3205	}
3206
3207	uint32_t cbSize = g_aIOSize[uIOWidth];
3208
3209	if (VMX_EXIT_QUALIFICATION_IO_STRING(exitQualification))
3210	{
3211	/* ins/outs */
3212	DISCPUSTATE Cpu;
3213
3214	/* Disassemble manually to deal with segment prefixes. */
3215	/** @todo VMX_VMCS_EXIT_GUEST_LINEAR_ADDR contains the flat pointer operand of the instruction. */
3216	/** @todo VMX_VMCS32_RO_EXIT_INSTR_INFO also contains segment prefix info. */
3217	rc = EMInterpretDisasOne(pVM, pVCpu, CPUMCTX2CORE(pCtx), &Cpu, NULL);
3218	if (rc == VINF_SUCCESS)
3219	{
3220	if (fIOWrite)
3221	{
3222	Log2(("IOMInterpretOUTSEx %RGv %x size=%d\n", (RTGCPTR)pCtx->rip, uPort, cbSize));
3223	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIOStringWrite);
3224	rc = IOMInterpretOUTSEx(pVM, CPUMCTX2CORE(pCtx), uPort, Cpu.prefix, cbSize);
3225	}
3226	else
3227	{
3228	Log2(("IOMInterpretINSEx %RGv %x size=%d\n", (RTGCPTR)pCtx->rip, uPort, cbSize));
3229	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIOStringRead);
3230	rc = IOMInterpretINSEx(pVM, CPUMCTX2CORE(pCtx), uPort, Cpu.prefix, cbSize);
3231	}
3232	}
3233	else
3234	rc = VINF_EM_RAW_EMULATE_INSTR;
3235	}
3236	else
3237	{
3238	/* normal in/out */
3239	uint32_t uAndVal = g_aIOOpAnd[uIOWidth];
3240
3241	Assert(!VMX_EXIT_QUALIFICATION_IO_REP(exitQualification));
3242
3243	if (fIOWrite)
3244	{
3245	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIOWrite);
3246	rc = IOMIOPortWrite(pVM, uPort, pCtx->eax & uAndVal, cbSize);
3247	}
3248	else
3249	{
3250	uint32_t u32Val = 0;
3251
3252	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIORead);
3253	rc = IOMIOPortRead(pVM, uPort, &u32Val, cbSize);
3254	if (IOM_SUCCESS(rc))
3255	{
3256	/* Write back to the EAX register. */
3257	pCtx->eax = (pCtx->eax & ~uAndVal) \| (u32Val & uAndVal);
3258	}
3259	}
3260	}
3261	/*
3262	* Handled the I/O return codes.
3263	* (The unhandled cases end up with rc == VINF_EM_RAW_EMULATE_INSTR.)
3264	*/
3265	if (IOM_SUCCESS(rc))
3266	{
3267	/* Update EIP and continue execution. */
3268	pCtx->rip += cbInstr;
3269	if (RT_LIKELY(rc == VINF_SUCCESS))
3270	{
3271	/* If any IO breakpoints are armed, then we should check if a debug trap needs to be generated. */
3272	if (pCtx->dr[7] & X86_DR7_ENABLED_MASK)
3273	{
3274	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatDRxIOCheck);
3275	for (unsigned i=0;i<4;i++)
3276	{
3277	unsigned uBPLen = g_aIOSize[X86_DR7_GET_LEN(pCtx->dr[7], i)];
3278
3279	if ( (uPort >= pCtx->dr[i] && uPort < pCtx->dr[i] + uBPLen)
3280	&& (pCtx->dr[7] & (X86_DR7_L(i) \| X86_DR7_G(i)))
3281	&& (pCtx->dr[7] & X86_DR7_RW(i, X86_DR7_RW_IO)) == X86_DR7_RW(i, X86_DR7_RW_IO))
3282	{
3283	uint64_t uDR6;
3284
3285	Assert(CPUMIsGuestDebugStateActive(pVCpu));
3286
3287	uDR6 = ASMGetDR6();
3288
3289	/* Clear all breakpoint status flags and set the one we just hit. */
3290	uDR6 &= ~(X86_DR6_B0\|X86_DR6_B1\|X86_DR6_B2\|X86_DR6_B3);
3291	uDR6 \|= (uint64_t)RT_BIT(i);
3292
3293	/* Note: AMD64 Architecture Programmer's Manual 13.1:
3294	* Bits 15:13 of the DR6 register is never cleared by the processor and must be cleared by software after
3295	* the contents have been read.
3296	*/
3297	ASMSetDR6(uDR6);
3298
3299	/* X86_DR7_GD will be cleared if drx accesses should be trapped inside the guest. */
3300	pCtx->dr[7] &= ~X86_DR7_GD;
3301
3302	/* Paranoia. */
3303	pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
3304	pCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* must be zero */
3305	pCtx->dr[7] \|= 0x400; /* must be one */
3306
3307	/* Resync DR7 */
3308	rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_DR7, pCtx->dr[7]);
3309	AssertRC(rc);
3310
3311	/* Construct inject info. */
3312	intInfo = X86_XCPT_DB;
3313	intInfo \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
3314	intInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
3315
3316	Log(("Inject IO debug trap at %RGv\n", (RTGCPTR)pCtx->rip));
3317	rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), 0, 0);
3318	AssertRC(rc);
3319
3320	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3321	goto ResumeExecution;
3322	}
3323	}
3324	}
3325
3326	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3327	goto ResumeExecution;
3328	}
3329	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3330	break;
3331	}
3332
3333	#ifdef VBOX_STRICT
3334	if (rc == VINF_IOM_HC_IOPORT_READ)
3335	Assert(!fIOWrite);
3336	else if (rc == VINF_IOM_HC_IOPORT_WRITE)
3337	Assert(fIOWrite);
3338	else
3339	AssertMsg(RT_FAILURE(rc) \|\| rc == VINF_EM_RAW_EMULATE_INSTR \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", rc));
3340	#endif
3341	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3342	break;
3343	}
3344
3345	case VMX_EXIT_TPR: /* 43 TPR below threshold. Guest software executed MOV to CR8. */
3346	LogFlow(("VMX_EXIT_TPR\n"));
3347	/* RIP is already set to the next instruction and the TPR has been synced back. Just resume. */
3348	goto ResumeExecution;
3349
3350	case VMX_EXIT_PREEMPTION_TIMER: /* 52 VMX-preemption timer expired. The preemption timer counted down to zero. */
3351	goto ResumeExecution;
3352
3353	default:
3354	/* The rest is handled after syncing the entire CPU state. */
3355	break;
3356	}
3357
3358	/* Note: the guest state isn't entirely synced back at this stage. */
3359
3360	/* Investigate why there was a VM-exit. (part 2) */
3361	switch (exitReason)
3362	{
3363	case VMX_EXIT_EXCEPTION: /* 0 Exception or non-maskable interrupt (NMI). */
3364	case VMX_EXIT_EXTERNAL_IRQ: /* 1 External interrupt. */
3365	case VMX_EXIT_EPT_VIOLATION:
3366	case VMX_EXIT_PREEMPTION_TIMER: /* 52 VMX-preemption timer expired. The preemption timer counted down to zero. */
3367	/* Already handled above. */
3368	break;
3369
3370	case VMX_EXIT_TRIPLE_FAULT: /* 2 Triple fault. */
3371	rc = VINF_EM_RESET; /* Triple fault equals a reset. */
3372	break;
3373
3374	case VMX_EXIT_INIT_SIGNAL: /* 3 INIT signal. */
3375	case VMX_EXIT_SIPI: /* 4 Start-up IPI (SIPI). */
3376	rc = VINF_EM_RAW_INTERRUPT;
3377	AssertFailed(); /* Can't happen. Yet. */
3378	break;
3379
3380	case VMX_EXIT_IO_SMI_IRQ: /* 5 I/O system-management interrupt (SMI). */
3381	case VMX_EXIT_SMI_IRQ: /* 6 Other SMI. */
3382	rc = VINF_EM_RAW_INTERRUPT;
3383	AssertFailed(); /* Can't happen afaik. */
3384	break;
3385
3386	case VMX_EXIT_TASK_SWITCH: /* 9 Task switch. */
3387	rc = VERR_EM_INTERPRETER;
3388	break;
3389
3390	case VMX_EXIT_HLT: /* 12 Guest software attempted to execute HLT. */
3391	/** Check if external interrupts are pending; if so, don't switch back. */
3392	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitHlt);
3393	pCtx->rip++; /* skip hlt */
3394	if ( pCtx->eflags.Bits.u1IF
3395	&& VMCPU_FF_ISPENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC\|VMCPU_FF_INTERRUPT_PIC)))
3396	goto ResumeExecution;
3397
3398	rc = VINF_EM_HALT;
3399	break;
3400
3401	case VMX_EXIT_MWAIT: /* 36 Guest software executed MWAIT. */
3402	Log2(("VMX: mwait\n"));
3403	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitMwait);
3404	rc = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pCtx));
3405	if ( rc == VINF_EM_HALT
3406	\|\| rc == VINF_SUCCESS)
3407	{
3408	/* Update EIP and continue execution. */
3409	pCtx->rip += cbInstr;
3410
3411	/** Check if external interrupts are pending; if so, don't switch back. */
3412	if ( rc == VINF_SUCCESS
3413	\|\| ( rc == VINF_EM_HALT
3414	&& pCtx->eflags.Bits.u1IF
3415	&& VMCPU_FF_ISPENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC\|VMCPU_FF_INTERRUPT_PIC)))
3416	)
3417	goto ResumeExecution;
3418	}
3419	AssertMsg(rc == VERR_EM_INTERPRETER \|\| rc == VINF_EM_HALT, ("EMU: mwait failed with %Rrc\n", rc));
3420	break;
3421
3422	case VMX_EXIT_RSM: /* 17 Guest software attempted to execute RSM in SMM. */
3423	AssertFailed(); /* can't happen. */
3424	rc = VINF_EM_RAW_EXCEPTION_PRIVILEGED;
3425	break;
3426
3427	case VMX_EXIT_VMCALL: /* 18 Guest software executed VMCALL. */
3428	case VMX_EXIT_VMCLEAR: /* 19 Guest software executed VMCLEAR. */
3429	case VMX_EXIT_VMLAUNCH: /* 20 Guest software executed VMLAUNCH. */
3430	case VMX_EXIT_VMPTRLD: /* 21 Guest software executed VMPTRLD. */
3431	case VMX_EXIT_VMPTRST: /* 22 Guest software executed VMPTRST. */
3432	case VMX_EXIT_VMREAD: /* 23 Guest software executed VMREAD. */
3433	case VMX_EXIT_VMRESUME: /* 24 Guest software executed VMRESUME. */
3434	case VMX_EXIT_VMWRITE: /* 25 Guest software executed VMWRITE. */
3435	case VMX_EXIT_VMXOFF: /* 26 Guest software executed VMXOFF. */
3436	case VMX_EXIT_VMXON: /* 27 Guest software executed VMXON. */
3437	/** @todo inject #UD immediately */
3438	rc = VINF_EM_RAW_EXCEPTION_PRIVILEGED;
3439	break;
3440
3441	case VMX_EXIT_CPUID: /* 10 Guest software attempted to execute CPUID. */
3442	case VMX_EXIT_RDTSC: /* 16 Guest software attempted to execute RDTSC. */
3443	case VMX_EXIT_INVPG: /* 14 Guest software attempted to execute INVPG. */
3444	case VMX_EXIT_CRX_MOVE: /* 28 Control-register accesses. */
3445	case VMX_EXIT_DRX_MOVE: /* 29 Debug-register accesses. */
3446	case VMX_EXIT_PORT_IO: /* 30 I/O instruction. */
3447	case VMX_EXIT_RDPMC: /* 15 Guest software attempted to execute RDPMC. */
3448	/* already handled above */
3449	AssertMsg( rc == VINF_PGM_CHANGE_MODE
3450	\|\| rc == VINF_EM_RAW_INTERRUPT
3451	\|\| rc == VERR_EM_INTERPRETER
3452	\|\| rc == VINF_EM_RAW_EMULATE_INSTR
3453	\|\| rc == VINF_PGM_SYNC_CR3
3454	\|\| rc == VINF_IOM_HC_IOPORT_READ
3455	\|\| rc == VINF_IOM_HC_IOPORT_WRITE
3456	\|\| rc == VINF_EM_RAW_GUEST_TRAP
3457	\|\| rc == VINF_TRPM_XCPT_DISPATCHED
3458	\|\| rc == VINF_EM_RESCHEDULE_REM,
3459	("rc = %d\n", rc));
3460	break;
3461
3462	case VMX_EXIT_TPR: /* 43 TPR below threshold. Guest software executed MOV to CR8. */
3463	case VMX_EXIT_RDMSR: /* 31 RDMSR. Guest software attempted to execute RDMSR. */
3464	case VMX_EXIT_WRMSR: /* 32 WRMSR. Guest software attempted to execute WRMSR. */
3465	/* Note: If we decide to emulate them here, then we must sync the MSRs that could have been changed (sysenter, fs/gs base)!!! */
3466	rc = VERR_EM_INTERPRETER;
3467	break;
3468
3469	case VMX_EXIT_MONITOR: /* 39 Guest software attempted to execute MONITOR. */
3470	case VMX_EXIT_PAUSE: /* 40 Guest software attempted to execute PAUSE. */
3471	rc = VINF_EM_RAW_EXCEPTION_PRIVILEGED;
3472	break;
3473
3474	case VMX_EXIT_IRQ_WINDOW: /* 7 Interrupt window. */
3475	Assert(rc == VINF_EM_RAW_INTERRUPT);
3476	break;
3477
3478	case VMX_EXIT_ERR_INVALID_GUEST_STATE: /* 33 VM-entry failure due to invalid guest state. */
3479	{
3480	#ifdef VBOX_STRICT
3481	RTCCUINTREG val = 0;
3482
3483	Log(("VMX_EXIT_ERR_INVALID_GUEST_STATE\n"));
3484
3485	VMXReadVMCS(VMX_VMCS64_GUEST_RIP, &val);
3486	Log(("Old eip %RGv new %RGv\n", (RTGCPTR)pCtx->rip, (RTGCPTR)val));
3487
3488	VMXReadVMCS(VMX_VMCS64_GUEST_CR0, &val);
3489	Log(("VMX_VMCS_GUEST_CR0 %RX64\n", (uint64_t)val));
3490
3491	VMXReadVMCS(VMX_VMCS64_GUEST_CR3, &val);
3492	Log(("VMX_VMCS_GUEST_CR3 %RX64\n", (uint64_t)val));
3493
3494	VMXReadVMCS(VMX_VMCS64_GUEST_CR4, &val);
3495	Log(("VMX_VMCS_GUEST_CR4 %RX64\n", (uint64_t)val));
3496
3497	VMXReadVMCS(VMX_VMCS_GUEST_RFLAGS, &val);
3498	Log(("VMX_VMCS_GUEST_RFLAGS %08x\n", val));
3499
3500	VMX_LOG_SELREG(CS, "CS");
3501	VMX_LOG_SELREG(DS, "DS");
3502	VMX_LOG_SELREG(ES, "ES");
3503	VMX_LOG_SELREG(FS, "FS");
3504	VMX_LOG_SELREG(GS, "GS");
3505	VMX_LOG_SELREG(SS, "SS");
3506	VMX_LOG_SELREG(TR, "TR");
3507	VMX_LOG_SELREG(LDTR, "LDTR");
3508
3509	VMXReadVMCS(VMX_VMCS64_GUEST_GDTR_BASE, &val);
3510	Log(("VMX_VMCS_GUEST_GDTR_BASE %RX64\n", (uint64_t)val));
3511	VMXReadVMCS(VMX_VMCS64_GUEST_IDTR_BASE, &val);
3512	Log(("VMX_VMCS_GUEST_IDTR_BASE %RX64\n", (uint64_t)val));
3513	#endif /* VBOX_STRICT */
3514	rc = VERR_VMX_INVALID_GUEST_STATE;
3515	break;
3516	}
3517
3518	case VMX_EXIT_ERR_MSR_LOAD: /* 34 VM-entry failure due to MSR loading. */
3519	case VMX_EXIT_ERR_MACHINE_CHECK: /* 41 VM-entry failure due to machine-check. */
3520	default:
3521	rc = VERR_VMX_UNEXPECTED_EXIT_CODE;
3522	AssertMsgFailed(("Unexpected exit code %d\n", exitReason)); /* Can't happen. */
3523	break;
3524
3525	}
3526	end:
3527
3528	/* Signal changes for the recompiler. */
3529	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR \| CPUM_CHANGED_LDTR \| CPUM_CHANGED_GDTR \| CPUM_CHANGED_IDTR \| CPUM_CHANGED_TR \| CPUM_CHANGED_HIDDEN_SEL_REGS);
3530
3531	/* If we executed vmlaunch/vmresume and an external irq was pending, then we don't have to do a full sync the next time. */
3532	if ( exitReason == VMX_EXIT_EXTERNAL_IRQ
3533	&& !VMX_EXIT_INTERRUPTION_INFO_VALID(intInfo))
3534	{
3535	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatPendingHostIrq);
3536	/* On the next entry we'll only sync the host context. */
3537	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_HOST_CONTEXT;
3538	}
3539	else
3540	{
3541	/* On the next entry we'll sync everything. */
3542	/** @todo we can do better than this */
3543	/* Not in the VINF_PGM_CHANGE_MODE though! */
3544	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_ALL;
3545	}
3546
3547	/* translate into a less severe return code */
3548	if (rc == VERR_EM_INTERPRETER)
3549	rc = VINF_EM_RAW_EMULATE_INSTR;
3550	else
3551	/* Try to extract more information about what might have gone wrong here. */
3552	if (rc == VERR_VMX_INVALID_VMCS_PTR)
3553	{
3554	VMXGetActivateVMCS(&pVCpu->hwaccm.s.vmx.lasterror.u64VMCSPhys);
3555	pVCpu->hwaccm.s.vmx.lasterror.ulVMCSRevision = (uint32_t )pVCpu->hwaccm.s.vmx.pVMCS;
3556	pVCpu->hwaccm.s.vmx.lasterror.idEnteredCpu = pVCpu->hwaccm.s.idEnteredCpu;
3557	pVCpu->hwaccm.s.vmx.lasterror.idCurrentCpu = RTMpCpuId();
3558	}
3559
3560	/* Just set the correct state here instead of trying to catch every goto above. */
3561	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED, VMCPUSTATE_STARTED_EXEC);
3562
3563	STAM_STATS({
3564	if (fStatExit2Started) STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2, y);
3565	else if (fStatEntryStarted) STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatEntry, x);
3566	});
3567	Log2(("X"));
3568	return rc;
3569	}
3570
3571
3572	/**
3573	* Enters the VT-x session
3574	*
3575	* @returns VBox status code.
3576	* @param pVM The VM to operate on.
3577	* @param pVCpu The VMCPU to operate on.
3578	* @param pCpu CPU info struct
3579	*/
3580	VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHWACCM_CPUINFO pCpu)
3581	{
3582	Assert(pVM->hwaccm.s.vmx.fSupported);
3583
3584	unsigned cr4 = ASMGetCR4();
3585	if (!(cr4 & X86_CR4_VMXE))
3586	{
3587	AssertMsgFailed(("X86_CR4_VMXE should be set!\n"));
3588	return VERR_VMX_X86_CR4_VMXE_CLEARED;
3589	}
3590
3591	/* Activate the VM Control Structure. */
3592	int rc = VMXActivateVMCS(pVCpu->hwaccm.s.vmx.pVMCSPhys);
3593	if (RT_FAILURE(rc))
3594	return rc;
3595
3596	pVCpu->hwaccm.s.fResumeVM = false;
3597	return VINF_SUCCESS;
3598	}
3599
3600
3601	/**
3602	* Leaves the VT-x session
3603	*
3604	* @returns VBox status code.
3605	* @param pVM The VM to operate on.
3606	* @param pVCpu The VMCPU to operate on.
3607	* @param pCtx CPU context
3608	*/
3609	VMMR0DECL(int) VMXR0Leave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3610	{
3611	Assert(pVM->hwaccm.s.vmx.fSupported);
3612
3613	/* Save the guest debug state if necessary. */
3614	if (CPUMIsGuestDebugStateActive(pVCpu))
3615	{
3616	CPUMR0SaveGuestDebugState(pVM, pVCpu, pCtx, true /* save DR6 */);
3617
3618	/* Enable drx move intercepts again. */
3619	pVCpu->hwaccm.s.vmx.proc_ctls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
3620	int rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
3621	AssertRC(rc);
3622
3623	/* Resync the debug registers the next time. */
3624	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_DEBUG;
3625	}
3626	else
3627	Assert(pVCpu->hwaccm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT);
3628
3629	/* Clear VM Control Structure. Marking it inactive, clearing implementation specific data and writing back VMCS data to memory. */
3630	int rc = VMXClearVMCS(pVCpu->hwaccm.s.vmx.pVMCSPhys);
3631	AssertRC(rc);
3632
3633	return VINF_SUCCESS;
3634	}
3635
3636	/**
3637	* Flush the TLB (EPT)
3638	*
3639	* @returns VBox status code.
3640	* @param pVM The VM to operate on.
3641	* @param pVCpu The VM CPU to operate on.
3642	* @param enmFlush Type of flush
3643	* @param GCPhys Physical address of the page to flush
3644	*/
3645	static void vmxR0FlushEPT(PVM pVM, PVMCPU pVCpu, VMX_FLUSH enmFlush, RTGCPHYS GCPhys)
3646	{
3647	uint64_t descriptor[2];
3648
3649	LogFlow(("vmxR0FlushEPT %d %RGv\n", enmFlush, GCPhys));
3650	Assert(pVM->hwaccm.s.fNestedPaging);
3651	descriptor[0] = pVCpu->hwaccm.s.vmx.GCPhysEPTP;
3652	descriptor[1] = GCPhys;
3653	int rc = VMXR0InvEPT(enmFlush, &descriptor[0]);
3654	AssertRC(rc);
3655	}
3656
3657	#ifdef HWACCM_VTX_WITH_VPID
3658	/**
3659	* Flush the TLB (EPT)
3660	*
3661	* @returns VBox status code.
3662	* @param pVM The VM to operate on.
3663	* @param pVCpu The VM CPU to operate on.
3664	* @param enmFlush Type of flush
3665	* @param GCPtr Virtual address of the page to flush
3666	*/
3667	static void vmxR0FlushVPID(PVM pVM, PVMCPU pVCpu, VMX_FLUSH enmFlush, RTGCPTR GCPtr)
3668	{
3669	#if HC_ARCH_BITS == 32
3670	/* If we get a flush in 64 bits guest mode, then force a full TLB flush. Invvpid probably takes only 32 bits addresses. (@todo) */
3671	if ( CPUMIsGuestInLongMode(pVCpu)
3672	&& !VMX_IS_64BIT_HOST_MODE())
3673	{
3674	pVCpu->hwaccm.s.fForceTLBFlush = true;
3675	}
3676	else
3677	#endif
3678	{
3679	uint64_t descriptor[2];
3680
3681	Assert(pVM->hwaccm.s.vmx.fVPID);
3682	descriptor[0] = pVCpu->hwaccm.s.uCurrentASID;
3683	descriptor[1] = GCPtr;
3684	int rc = VMXR0InvVPID(enmFlush, &descriptor[0]);
3685	AssertRC(rc);
3686	}
3687	}
3688	#endif /* HWACCM_VTX_WITH_VPID */
3689
3690	/**
3691	* Invalidates a guest page
3692	*
3693	* @returns VBox status code.
3694	* @param pVM The VM to operate on.
3695	* @param pVCpu The VM CPU to operate on.
3696	* @param GCVirt Page to invalidate
3697	*/
3698	VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
3699	{
3700	bool fFlushPending = pVCpu->hwaccm.s.fForceTLBFlush;
3701
3702	Log2(("VMXR0InvalidatePage %RGv\n", GCVirt));
3703
3704	/* Only relevant if we want to use VPID.
3705	* In the nested paging case we still see such calls, but
3706	* can safely ignore them. (e.g. after cr3 updates)
3707	*/
3708	#ifdef HWACCM_VTX_WITH_VPID
3709	/* Skip it if a TLB flush is already pending. */
3710	if ( !fFlushPending
3711	&& pVM->hwaccm.s.vmx.fVPID)
3712	vmxR0FlushVPID(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushPage, GCVirt);
3713	#endif /* HWACCM_VTX_WITH_VPID */
3714
3715	return VINF_SUCCESS;
3716	}
3717
3718	/**
3719	* Invalidates a guest page by physical address
3720	*
3721	* NOTE: Assumes the current instruction references this physical page though a virtual address!!
3722	*
3723	* @returns VBox status code.
3724	* @param pVM The VM to operate on.
3725	* @param pVCpu The VM CPU to operate on.
3726	* @param GCPhys Page to invalidate
3727	*/
3728	VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
3729	{
3730	bool fFlushPending = pVCpu->hwaccm.s.fForceTLBFlush;
3731
3732	Assert(pVM->hwaccm.s.fNestedPaging);
3733
3734	LogFlow(("VMXR0InvalidatePhysPage %RGp\n", GCPhys));
3735
3736	/* Skip it if a TLB flush is already pending. */
3737	if (!fFlushPending)
3738	vmxR0FlushEPT(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushPage, GCPhys);
3739
3740	return VINF_SUCCESS;
3741	}
3742
3743	/**
3744	* Report world switch error and dump some useful debug info
3745	*
3746	* @param pVM The VM to operate on.
3747	* @param pVCpu The VMCPU to operate on.
3748	* @param rc Return code
3749	* @param pCtx Current CPU context (not updated)
3750	*/
3751	static void VMXR0ReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rc, PCPUMCTX pCtx)
3752	{
3753	switch (rc)
3754	{
3755	case VERR_VMX_INVALID_VMXON_PTR:
3756	AssertFailed();
3757	break;
3758
3759	case VERR_VMX_UNABLE_TO_START_VM:
3760	case VERR_VMX_UNABLE_TO_RESUME_VM:
3761	{
3762	int rc;
3763	RTCCUINTREG exitReason, instrError;
3764
3765	rc = VMXReadVMCS(VMX_VMCS32_RO_EXIT_REASON, &exitReason);
3766	rc \|= VMXReadVMCS(VMX_VMCS32_RO_VM_INSTR_ERROR, &instrError);
3767	AssertRC(rc);
3768	if (rc == VINF_SUCCESS)
3769	{
3770	Log(("Unable to start/resume VM for reason: %x. Instruction error %x\n", (uint32_t)exitReason, (uint32_t)instrError));
3771	Log(("Current stack %08x\n", &rc));
3772
3773	pVCpu->hwaccm.s.vmx.lasterror.ulInstrError = instrError;
3774	pVCpu->hwaccm.s.vmx.lasterror.ulExitReason = exitReason;
3775
3776	#ifdef VBOX_STRICT
3777	RTGDTR gdtr;
3778	PX86DESCHC pDesc;
3779	RTCCUINTREG val;
3780
3781	ASMGetGDTR(&gdtr);
3782
3783	VMXReadVMCS(VMX_VMCS64_GUEST_RIP, &val);
3784	Log(("Old eip %RGv new %RGv\n", (RTGCPTR)pCtx->rip, (RTGCPTR)val));
3785	VMXReadVMCS(VMX_VMCS_CTRL_PIN_EXEC_CONTROLS, &val);
3786	Log(("VMX_VMCS_CTRL_PIN_EXEC_CONTROLS %08x\n", val));
3787	VMXReadVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, &val);
3788	Log(("VMX_VMCS_CTRL_PROC_EXEC_CONTROLS %08x\n", val));
3789	VMXReadVMCS(VMX_VMCS_CTRL_ENTRY_CONTROLS, &val);
3790	Log(("VMX_VMCS_CTRL_ENTRY_CONTROLS %08x\n", val));
3791	VMXReadVMCS(VMX_VMCS_CTRL_EXIT_CONTROLS, &val);
3792	Log(("VMX_VMCS_CTRL_EXIT_CONTROLS %08x\n", val));
3793
3794	VMXReadVMCS(VMX_VMCS_HOST_CR0, &val);
3795	Log(("VMX_VMCS_HOST_CR0 %08x\n", val));
3796
3797	VMXReadVMCS(VMX_VMCS_HOST_CR3, &val);
3798	Log(("VMX_VMCS_HOST_CR3 %08x\n", val));
3799
3800	VMXReadVMCS(VMX_VMCS_HOST_CR4, &val);
3801	Log(("VMX_VMCS_HOST_CR4 %08x\n", val));
3802
3803	VMXReadVMCS(VMX_VMCS16_HOST_FIELD_CS, &val);
3804	Log(("VMX_VMCS_HOST_FIELD_CS %08x\n", val));
3805
3806	VMXReadVMCS(VMX_VMCS_GUEST_RFLAGS, &val);
3807	Log(("VMX_VMCS_GUEST_RFLAGS %08x\n", val));
3808
3809	if (val < gdtr.cbGdt)
3810	{
3811	pDesc = &((PX86DESCHC)gdtr.pGdt)[val >> X86_SEL_SHIFT_HC];
3812	HWACCMR0DumpDescriptor(pDesc, val, "CS: ");
3813	}
3814
3815	VMXReadVMCS(VMX_VMCS16_HOST_FIELD_DS, &val);
3816	Log(("VMX_VMCS_HOST_FIELD_DS %08x\n", val));
3817	if (val < gdtr.cbGdt)
3818	{
3819	pDesc = &((PX86DESCHC)gdtr.pGdt)[val >> X86_SEL_SHIFT_HC];
3820	HWACCMR0DumpDescriptor(pDesc, val, "DS: ");
3821	}
3822
3823	VMXReadVMCS(VMX_VMCS16_HOST_FIELD_ES, &val);
3824	Log(("VMX_VMCS_HOST_FIELD_ES %08x\n", val));
3825	if (val < gdtr.cbGdt)
3826	{
3827	pDesc = &((PX86DESCHC)gdtr.pGdt)[val >> X86_SEL_SHIFT_HC];
3828	HWACCMR0DumpDescriptor(pDesc, val, "ES: ");
3829	}
3830
3831	VMXReadVMCS(VMX_VMCS16_HOST_FIELD_FS, &val);
3832	Log(("VMX_VMCS16_HOST_FIELD_FS %08x\n", val));
3833	if (val < gdtr.cbGdt)
3834	{
3835	pDesc = &((PX86DESCHC)gdtr.pGdt)[val >> X86_SEL_SHIFT_HC];
3836	HWACCMR0DumpDescriptor(pDesc, val, "FS: ");
3837	}
3838
3839	VMXReadVMCS(VMX_VMCS16_HOST_FIELD_GS, &val);
3840	Log(("VMX_VMCS16_HOST_FIELD_GS %08x\n", val));
3841	if (val < gdtr.cbGdt)
3842	{
3843	pDesc = &((PX86DESCHC)gdtr.pGdt)[val >> X86_SEL_SHIFT_HC];
3844	HWACCMR0DumpDescriptor(pDesc, val, "GS: ");
3845	}
3846
3847	VMXReadVMCS(VMX_VMCS16_HOST_FIELD_SS, &val);
3848	Log(("VMX_VMCS16_HOST_FIELD_SS %08x\n", val));
3849	if (val < gdtr.cbGdt)
3850	{
3851	pDesc = &((PX86DESCHC)gdtr.pGdt)[val >> X86_SEL_SHIFT_HC];
3852	HWACCMR0DumpDescriptor(pDesc, val, "SS: ");
3853	}
3854
3855	VMXReadVMCS(VMX_VMCS16_HOST_FIELD_TR, &val);
3856	Log(("VMX_VMCS16_HOST_FIELD_TR %08x\n", val));
3857	if (val < gdtr.cbGdt)
3858	{
3859	pDesc = &((PX86DESCHC)gdtr.pGdt)[val >> X86_SEL_SHIFT_HC];
3860	HWACCMR0DumpDescriptor(pDesc, val, "TR: ");
3861	}
3862
3863	VMXReadVMCS(VMX_VMCS_HOST_TR_BASE, &val);
3864	Log(("VMX_VMCS_HOST_TR_BASE %RHv\n", val));
3865
3866	VMXReadVMCS(VMX_VMCS_HOST_GDTR_BASE, &val);
3867	Log(("VMX_VMCS_HOST_GDTR_BASE %RHv\n", val));
3868	VMXReadVMCS(VMX_VMCS_HOST_IDTR_BASE, &val);
3869	Log(("VMX_VMCS_HOST_IDTR_BASE %RHv\n", val));
3870
3871	VMXReadVMCS(VMX_VMCS32_HOST_SYSENTER_CS, &val);
3872	Log(("VMX_VMCS_HOST_SYSENTER_CS %08x\n", val));
3873
3874	VMXReadVMCS(VMX_VMCS_HOST_SYSENTER_EIP, &val);
3875	Log(("VMX_VMCS_HOST_SYSENTER_EIP %RHv\n", val));
3876
3877	VMXReadVMCS(VMX_VMCS_HOST_SYSENTER_ESP, &val);
3878	Log(("VMX_VMCS_HOST_SYSENTER_ESP %RHv\n", val));
3879
3880	VMXReadVMCS(VMX_VMCS_HOST_RSP, &val);
3881	Log(("VMX_VMCS_HOST_RSP %RHv\n", val));
3882	VMXReadVMCS(VMX_VMCS_HOST_RIP, &val);
3883	Log(("VMX_VMCS_HOST_RIP %RHv\n", val));
3884
3885	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3886	if (VMX_IS_64BIT_HOST_MODE())
3887	{
3888	Log(("MSR_K6_EFER = %RX64\n", ASMRdMsr(MSR_K6_EFER)));
3889	Log(("MSR_K6_STAR = %RX64\n", ASMRdMsr(MSR_K6_STAR)));
3890	Log(("MSR_K8_LSTAR = %RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
3891	Log(("MSR_K8_CSTAR = %RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
3892	Log(("MSR_K8_SF_MASK = %RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
3893	}
3894	# endif
3895	#endif /* VBOX_STRICT */
3896	}
3897	break;
3898	}
3899
3900	default:
3901	/* impossible */
3902	AssertMsgFailed(("%Rrc (%#x)\n", rc, rc));
3903	break;
3904	}
3905	}
3906
3907	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3908	/**
3909	* Prepares for and executes VMLAUNCH (64 bits guest mode)
3910	*
3911	* @returns VBox status code
3912	* @param fResume vmlauch/vmresume
3913	* @param pCtx Guest context
3914	* @param pCache VMCS cache
3915	* @param pVM The VM to operate on.
3916	* @param pVCpu The VMCPU to operate on.
3917	*/
3918	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
3919	{
3920	uint32_t aParam[6];
3921	PHWACCM_CPUINFO pCpu;
3922	RTHCPHYS pPageCpuPhys;
3923	int rc;
3924
3925	pCpu = HWACCMR0GetCurrentCpu();
3926	pPageCpuPhys = RTR0MemObjGetPagePhysAddr(pCpu->pMemObj, 0);
3927
3928	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
3929	pCache->uPos = 1;
3930	pCache->interPD = PGMGetInterPaeCR3(pVM);
3931	pCache->pSwitcher = (uint64_t)pVM->hwaccm.s.pfnHost32ToGuest64R0;
3932	#endif
3933
3934	#ifdef DEBUG
3935	pCache->TestIn.pPageCpuPhys = 0;
3936	pCache->TestIn.pVMCSPhys = 0;
3937	pCache->TestIn.pCache = 0;
3938	pCache->TestOut.pVMCSPhys = 0;
3939	pCache->TestOut.pCache = 0;
3940	pCache->TestOut.pCtx = 0;
3941	pCache->TestOut.eflags = 0;
3942	#endif
3943
3944	aParam[0] = (uint32_t)(pPageCpuPhys); /* Param 1: VMXON physical address - Lo. */
3945	aParam[1] = (uint32_t)(pPageCpuPhys >> 32); /* Param 1: VMXON physical address - Hi. */
3946	aParam[2] = (uint32_t)(pVCpu->hwaccm.s.vmx.pVMCSPhys); /* Param 2: VMCS physical address - Lo. */
3947	aParam[3] = (uint32_t)(pVCpu->hwaccm.s.vmx.pVMCSPhys >> 32); /* Param 2: VMCS physical address - Hi. */
3948	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hwaccm.s.vmx.VMCSCache);
3949	aParam[5] = 0;
3950
3951	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
3952	pCtx->dr[4] = pVM->hwaccm.s.vmx.pScratchPhys + 16 + 8;
3953	(uint32_t )(pVM->hwaccm.s.vmx.pScratch + 16 + 8) = 1;
3954	#endif
3955	rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, pVM->hwaccm.s.pfnVMXGCStartVM64, 6, &aParam[0]);
3956
3957	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
3958	Assert((uint32_t )(pVM->hwaccm.s.vmx.pScratch + 16 + 8) == 5);
3959	Assert(pCtx->dr[4] == 10);
3960	(uint32_t )(pVM->hwaccm.s.vmx.pScratch + 16 + 8) = 0xff;
3961	#endif
3962
3963	#ifdef DEBUG
3964	AssertMsg(pCache->TestIn.pPageCpuPhys == pPageCpuPhys, ("%RHp vs %RHp\n", pCache->TestIn.pPageCpuPhys, pPageCpuPhys));
3965	AssertMsg(pCache->TestIn.pVMCSPhys == pVCpu->hwaccm.s.vmx.pVMCSPhys, ("%RHp vs %RHp\n", pCache->TestIn.pVMCSPhys, pVCpu->hwaccm.s.vmx.pVMCSPhys));
3966	AssertMsg(pCache->TestIn.pVMCSPhys == pCache->TestOut.pVMCSPhys, ("%RHp vs %RHp\n", pCache->TestIn.pVMCSPhys, pCache->TestOut.pVMCSPhys));
3967	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache, pCache->TestOut.pCache));
3968	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hwaccm.s.vmx.VMCSCache), ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hwaccm.s.vmx.VMCSCache)));
3969	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx, pCache->TestOut.pCtx));
3970	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
3971	#endif
3972	return rc;
3973	}
3974
3975	/**
3976	* Executes the specified handler in 64 mode
3977	*
3978	* @returns VBox status code.
3979	* @param pVM The VM to operate on.
3980	* @param pVCpu The VMCPU to operate on.
3981	* @param pCtx Guest context
3982	* @param pfnHandler RC handler
3983	* @param cbParam Number of parameters
3984	* @param paParam Array of 32 bits parameters
3985	*/
3986	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, RTRCPTR pfnHandler, uint32_t cbParam, uint32_t *paParam)
3987	{
3988	int rc, rc2;
3989	PHWACCM_CPUINFO pCpu;
3990	RTHCPHYS pPageCpuPhys;
3991
3992	/* @todo This code is not guest SMP safe (hyper stack) */
3993	AssertReturn(pVM->cCPUs == 1, VERR_ACCESS_DENIED);
3994	AssertReturn(pVM->hwaccm.s.pfnHost32ToGuest64R0, VERR_INTERNAL_ERROR);
3995	Assert(pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hwaccm.s.vmx.VMCSCache.Write.aField));
3996	Assert(pVCpu->hwaccm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hwaccm.s.vmx.VMCSCache.Read.aField));
3997
3998	#ifdef VBOX_STRICT
3999	for (unsigned i=0;i<pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries;i++)
4000	Assert(vmxR0IsValidWriteField(pVCpu->hwaccm.s.vmx.VMCSCache.Write.aField[i]));
4001
4002	for (unsigned i=0;i<pVCpu->hwaccm.s.vmx.VMCSCache.Read.cValidEntries;i++)
4003	Assert(vmxR0IsValidReadField(pVCpu->hwaccm.s.vmx.VMCSCache.Read.aField[i]));
4004	#endif
4005
4006	pCpu = HWACCMR0GetCurrentCpu();
4007	pPageCpuPhys = RTR0MemObjGetPagePhysAddr(pCpu->pMemObj, 0);
4008
4009	/* Clear VM Control Structure. Marking it inactive, clearing implementation specific data and writing back VMCS data to memory. */
4010	VMXClearVMCS(pVCpu->hwaccm.s.vmx.pVMCSPhys);
4011
4012	/* Leave VMX Root Mode. */
4013	VMXDisable();
4014
4015	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4016
4017	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVM));
4018	CPUMSetHyperEIP(pVCpu, pfnHandler);
4019	for (int i=(int)cbParam-1;i>=0;i--)
4020	CPUMPushHyper(pVCpu, paParam[i]);
4021
4022	STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatWorldSwitch3264, z);
4023	/* Call switcher. */
4024	rc = pVM->hwaccm.s.pfnHost32ToGuest64R0(pVM);
4025	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatWorldSwitch3264, z);
4026
4027	/* Make sure the VMX instructions don't cause #UD faults. */
4028	ASMSetCR4(ASMGetCR4() \| X86_CR4_VMXE);
4029
4030	/* Enter VMX Root Mode */
4031	rc2 = VMXEnable(pPageCpuPhys);
4032	if (RT_FAILURE(rc2))
4033	{
4034	if (pVM)
4035	VMXR0CheckError(pVM, pVCpu, rc2);
4036	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4037	return VERR_VMX_VMXON_FAILED;
4038	}
4039
4040	rc2 = VMXActivateVMCS(pVCpu->hwaccm.s.vmx.pVMCSPhys);
4041	AssertRCReturn(rc2, rc2);
4042	#ifdef RT_OS_WINDOWS
4043	Assert(ASMGetFlags() & X86_EFL_IF);
4044	#else
4045	Assert(!(ASMGetFlags() & X86_EFL_IF));
4046	#endif
4047	return rc;
4048	}
4049
4050	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
4051
4052
4053	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
4054	/**
4055	* Executes VMWRITE
4056	*
4057	* @returns VBox status code
4058	* @param pVCpu The VMCPU to operate on.
4059	* @param idxField VMCS index
4060	* @param u64Val 16, 32 or 64 bits value
4061	*/
4062	VMMR0DECL(int) VMXWriteVMCS64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4063	{
4064	int rc;
4065
4066	switch (idxField)
4067	{
4068	case VMX_VMCS_CTRL_TSC_OFFSET_FULL:
4069	case VMX_VMCS_CTRL_IO_BITMAP_A_FULL:
4070	case VMX_VMCS_CTRL_IO_BITMAP_B_FULL:
4071	case VMX_VMCS_CTRL_MSR_BITMAP_FULL:
4072	case VMX_VMCS_CTRL_VMEXIT_MSR_STORE_FULL:
4073	case VMX_VMCS_CTRL_VMEXIT_MSR_LOAD_FULL:
4074	case VMX_VMCS_CTRL_VMENTRY_MSR_LOAD_FULL:
4075	case VMX_VMCS_CTRL_VAPIC_PAGEADDR_FULL:
4076	case VMX_VMCS_GUEST_LINK_PTR_FULL:
4077	case VMX_VMCS_GUEST_PDPTR0_FULL:
4078	case VMX_VMCS_GUEST_PDPTR1_FULL:
4079	case VMX_VMCS_GUEST_PDPTR2_FULL:
4080	case VMX_VMCS_GUEST_PDPTR3_FULL:
4081	case VMX_VMCS_GUEST_DEBUGCTL_FULL:
4082	case VMX_VMCS_GUEST_EFER_FULL:
4083	case VMX_VMCS_CTRL_EPTP_FULL:
4084	/* These fields consist of two parts, which are both writable in 32 bits mode. */
4085	rc = VMXWriteVMCS32(idxField, u64Val);
4086	rc \|= VMXWriteVMCS32(idxField + 1, (uint32_t)(u64Val >> 32ULL));
4087	AssertRC(rc);
4088	return rc;
4089
4090	case VMX_VMCS64_GUEST_LDTR_BASE:
4091	case VMX_VMCS64_GUEST_TR_BASE:
4092	case VMX_VMCS64_GUEST_GDTR_BASE:
4093	case VMX_VMCS64_GUEST_IDTR_BASE:
4094	case VMX_VMCS64_GUEST_SYSENTER_EIP:
4095	case VMX_VMCS64_GUEST_SYSENTER_ESP:
4096	case VMX_VMCS64_GUEST_CR0:
4097	case VMX_VMCS64_GUEST_CR4:
4098	case VMX_VMCS64_GUEST_CR3:
4099	case VMX_VMCS64_GUEST_DR7:
4100	case VMX_VMCS64_GUEST_RIP:
4101	case VMX_VMCS64_GUEST_RSP:
4102	case VMX_VMCS64_GUEST_CS_BASE:
4103	case VMX_VMCS64_GUEST_DS_BASE:
4104	case VMX_VMCS64_GUEST_ES_BASE:
4105	case VMX_VMCS64_GUEST_FS_BASE:
4106	case VMX_VMCS64_GUEST_GS_BASE:
4107	case VMX_VMCS64_GUEST_SS_BASE:
4108	/* Queue a 64 bits value as we can't set it in 32 bits host mode. */
4109	if (u64Val >> 32ULL)
4110	rc = VMXWriteCachedVMCSEx(pVCpu, idxField, u64Val);
4111	else
4112	rc = VMXWriteVMCS32(idxField, (uint32_t)u64Val);
4113
4114	return rc;
4115
4116	default:
4117	AssertMsgFailed(("Unexpected field %x\n", idxField));
4118	return VERR_INVALID_PARAMETER;
4119	}
4120	}
4121
4122	/**
4123	* Cache VMCS writes for performance reasons (Darwin) and for running 64 bits guests on 32 bits hosts.
4124	*
4125	* @param pVCpu The VMCPU to operate on.
4126	* @param idxField VMCS field
4127	* @param u64Val Value
4128	*/
4129	VMMR0DECL(int) VMXWriteCachedVMCSEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4130	{
4131	PVMCSCACHE pCache = &pVCpu->hwaccm.s.vmx.VMCSCache;
4132
4133	AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1, ("entries=%x\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
4134
4135	/* Make sure there are no duplicates. */
4136	for (unsigned i=0;i<pCache->Write.cValidEntries;i++)
4137	{
4138	if (pCache->Write.aField[i] == idxField)
4139	{
4140	pCache->Write.aFieldVal[i] = u64Val;
4141	return VINF_SUCCESS;
4142	}
4143	}
4144
4145	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
4146	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
4147	pCache->Write.cValidEntries++;
4148	return VINF_SUCCESS;
4149	}
4150
4151	#endif /* HC_ARCH_BITS == 32 && !VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0 */
4152
4153	#ifdef VBOX_STRICT
4154	static bool vmxR0IsValidReadField(uint32_t idxField)
4155	{
4156	switch(idxField)
4157	{
4158	case VMX_VMCS64_GUEST_RIP:
4159	case VMX_VMCS64_GUEST_RSP:
4160	case VMX_VMCS_GUEST_RFLAGS:
4161	case VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE:
4162	case VMX_VMCS_CTRL_CR0_READ_SHADOW:
4163	case VMX_VMCS64_GUEST_CR0:
4164	case VMX_VMCS_CTRL_CR4_READ_SHADOW:
4165	case VMX_VMCS64_GUEST_CR4:
4166	case VMX_VMCS64_GUEST_DR7:
4167	case VMX_VMCS32_GUEST_SYSENTER_CS:
4168	case VMX_VMCS64_GUEST_SYSENTER_EIP:
4169	case VMX_VMCS64_GUEST_SYSENTER_ESP:
4170	case VMX_VMCS32_GUEST_GDTR_LIMIT:
4171	case VMX_VMCS64_GUEST_GDTR_BASE:
4172	case VMX_VMCS32_GUEST_IDTR_LIMIT:
4173	case VMX_VMCS64_GUEST_IDTR_BASE:
4174	case VMX_VMCS16_GUEST_FIELD_CS:
4175	case VMX_VMCS32_GUEST_CS_LIMIT:
4176	case VMX_VMCS64_GUEST_CS_BASE:
4177	case VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS:
4178	case VMX_VMCS16_GUEST_FIELD_DS:
4179	case VMX_VMCS32_GUEST_DS_LIMIT:
4180	case VMX_VMCS64_GUEST_DS_BASE:
4181	case VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS:
4182	case VMX_VMCS16_GUEST_FIELD_ES:
4183	case VMX_VMCS32_GUEST_ES_LIMIT:
4184	case VMX_VMCS64_GUEST_ES_BASE:
4185	case VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS:
4186	case VMX_VMCS16_GUEST_FIELD_FS:
4187	case VMX_VMCS32_GUEST_FS_LIMIT:
4188	case VMX_VMCS64_GUEST_FS_BASE:
4189	case VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS:
4190	case VMX_VMCS16_GUEST_FIELD_GS:
4191	case VMX_VMCS32_GUEST_GS_LIMIT:
4192	case VMX_VMCS64_GUEST_GS_BASE:
4193	case VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS:
4194	case VMX_VMCS16_GUEST_FIELD_SS:
4195	case VMX_VMCS32_GUEST_SS_LIMIT:
4196	case VMX_VMCS64_GUEST_SS_BASE:
4197	case VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS:
4198	case VMX_VMCS16_GUEST_FIELD_LDTR:
4199	case VMX_VMCS32_GUEST_LDTR_LIMIT:
4200	case VMX_VMCS64_GUEST_LDTR_BASE:
4201	case VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS:
4202	case VMX_VMCS16_GUEST_FIELD_TR:
4203	case VMX_VMCS32_GUEST_TR_LIMIT:
4204	case VMX_VMCS64_GUEST_TR_BASE:
4205	case VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS:
4206	case VMX_VMCS32_RO_EXIT_REASON:
4207	case VMX_VMCS32_RO_VM_INSTR_ERROR:
4208	case VMX_VMCS32_RO_EXIT_INSTR_LENGTH:
4209	case VMX_VMCS32_RO_EXIT_INTERRUPTION_ERRCODE:
4210	case VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO:
4211	case VMX_VMCS32_RO_EXIT_INSTR_INFO:
4212	case VMX_VMCS_RO_EXIT_QUALIFICATION:
4213	case VMX_VMCS32_RO_IDT_INFO:
4214	case VMX_VMCS32_RO_IDT_ERRCODE:
4215	case VMX_VMCS64_GUEST_CR3:
4216	case VMX_VMCS_EXIT_PHYS_ADDR_FULL:
4217	return true;
4218	}
4219	return false;
4220	}
4221
4222	static bool vmxR0IsValidWriteField(uint32_t idxField)
4223	{
4224	switch(idxField)
4225	{
4226	case VMX_VMCS64_GUEST_LDTR_BASE:
4227	case VMX_VMCS64_GUEST_TR_BASE:
4228	case VMX_VMCS64_GUEST_GDTR_BASE:
4229	case VMX_VMCS64_GUEST_IDTR_BASE:
4230	case VMX_VMCS64_GUEST_SYSENTER_EIP:
4231	case VMX_VMCS64_GUEST_SYSENTER_ESP:
4232	case VMX_VMCS64_GUEST_CR0:
4233	case VMX_VMCS64_GUEST_CR4:
4234	case VMX_VMCS64_GUEST_CR3:
4235	case VMX_VMCS64_GUEST_DR7:
4236	case VMX_VMCS64_GUEST_RIP:
4237	case VMX_VMCS64_GUEST_RSP:
4238	case VMX_VMCS64_GUEST_CS_BASE:
4239	case VMX_VMCS64_GUEST_DS_BASE:
4240	case VMX_VMCS64_GUEST_ES_BASE:
4241	case VMX_VMCS64_GUEST_FS_BASE:
4242	case VMX_VMCS64_GUEST_GS_BASE:
4243	case VMX_VMCS64_GUEST_SS_BASE:
4244	return true;
4245	}
4246	return false;
4247	}
4248
4249	#endif

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HWVMXR0.cpp@ 19861

Download in other formats: