HWVMXR0.cpp@ 18768

Last change on this file since 18768 was 18666, checked in by vboxsync, 16 years ago
VMM: Clean out the VBOX_WITH_NEW_PHYS_CODE #ifdefs. (part 2)
Property svn:eol-style set to `native` Property svn:keywords set to `Id`
File size: 164.0 KB

Line
1	/* $Id: HWVMXR0.cpp 18666 2009-04-02 23:10:12Z vboxsync $ */
2	/** @file
3	* HWACCM VMX - Host Context Ring 0.
4	*/
5
6	/*
7	* Copyright (C) 2006-2007 Sun Microsystems, Inc.
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*
17	* Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa
18	* Clara, CA 95054 USA or visit http://www.sun.com if you need
19	* additional information or have any questions.
20	*/
21
22
23	/*******************************************************************************
24	* Header Files *
25	*******************************************************************************/
26	#define LOG_GROUP LOG_GROUP_HWACCM
27	#include <VBox/hwaccm.h>
28	#include "HWACCMInternal.h"
29	#include <VBox/vm.h>
30	#include <VBox/x86.h>
31	#include <VBox/pgm.h>
32	#include <VBox/pdm.h>
33	#include <VBox/err.h>
34	#include <VBox/log.h>
35	#include <VBox/selm.h>
36	#include <VBox/iom.h>
37	#include <VBox/rem.h>
38	#include <iprt/param.h>
39	#include <iprt/assert.h>
40	#include <iprt/asm.h>
41	#include <iprt/string.h>
42	#include "HWVMXR0.h"
43
44	/*******************************************************************************
45	* Defined Constants And Macros *
46	*******************************************************************************/
47	#if defined(RT_ARCH_AMD64)
48	# define VMX_IS_64BIT_HOST_MODE() (true)
49	#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
50	# define VMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
51	#else
52	# define VMX_IS_64BIT_HOST_MODE() (false)
53	#endif
54
55	/*******************************************************************************
56	* Global Variables *
57	*******************************************************************************/
58	/* IO operation lookup arrays. */
59	static uint32_t const g_aIOSize[4] = {1, 2, 0, 4};
60	static uint32_t const g_aIOOpAnd[4] = {0xff, 0xffff, 0, 0xffffffff};
61
62	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
63	/** See HWACCMR0A.asm. */
64	extern "C" uint32_t g_fVMXIs64bitHost;
65	#endif
66
67	/*******************************************************************************
68	* Local Functions *
69	*******************************************************************************/
70	static void VMXR0ReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rc, PCPUMCTX pCtx);
71	static void vmxR0SetupTLBEPT(PVM pVM, PVMCPU pVCpu);
72	static void vmxR0SetupTLBVPID(PVM pVM, PVMCPU pVCpu);
73	static void vmxR0SetupTLBDummy(PVM pVM, PVMCPU pVCpu);
74	static void vmxR0FlushEPT(PVM pVM, PVMCPU pVCpu, VMX_FLUSH enmFlush, RTGCPHYS GCPhys);
75	static void vmxR0FlushVPID(PVM pVM, PVMCPU pVCpu, VMX_FLUSH enmFlush, RTGCPTR GCPtr);
76	static void vmxR0UpdateExceptionBitmap(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
77	#ifdef VBOX_STRICT
78	static bool vmxR0IsValidReadField(uint32_t idxField);
79	static bool vmxR0IsValidWriteField(uint32_t idxField);
80	#endif
81
82	static void VMXR0CheckError(PVM pVM, PVMCPU pVCpu, int rc)
83	{
84	if (rc == VERR_VMX_GENERIC)
85	{
86	RTCCUINTREG instrError;
87
88	VMXReadVMCS(VMX_VMCS32_RO_VM_INSTR_ERROR, &instrError);
89	pVCpu->hwaccm.s.vmx.lasterror.ulInstrError = instrError;
90	}
91	pVM->hwaccm.s.lLastError = rc;
92	}
93
94	/**
95	* Sets up and activates VT-x on the current CPU
96	*
97	* @returns VBox status code.
98	* @param pCpu CPU info struct
99	* @param pVM The VM to operate on. (can be NULL after a resume!!)
100	* @param pvPageCpu Pointer to the global cpu page
101	* @param pPageCpuPhys Physical address of the global cpu page
102	*/
103	VMMR0DECL(int) VMXR0EnableCpu(PHWACCM_CPUINFO pCpu, PVM pVM, void *pvPageCpu, RTHCPHYS pPageCpuPhys)
104	{
105	AssertReturn(pPageCpuPhys, VERR_INVALID_PARAMETER);
106	AssertReturn(pvPageCpu, VERR_INVALID_PARAMETER);
107
108	#ifdef LOG_ENABLED
109	SUPR0Printf("VMXR0EnableCpu cpu %d page (%x) %x\n", pCpu->idCpu, pvPageCpu, (uint32_t)pPageCpuPhys);
110	#endif
111	if (pVM)
112	{
113	/* Set revision dword at the beginning of the VMXON structure. */
114	(uint32_t )pvPageCpu = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hwaccm.s.vmx.msr.vmx_basic_info);
115	}
116
117	/** @todo we should unmap the two pages from the virtual address space in order to prevent accidental corruption.
118	* (which can have very bad consequences!!!)
119	*/
120
121	/* Make sure the VMX instructions don't cause #UD faults. */
122	ASMSetCR4(ASMGetCR4() \| X86_CR4_VMXE);
123
124	/* Enter VMX Root Mode */
125	int rc = VMXEnable(pPageCpuPhys);
126	if (RT_FAILURE(rc))
127	{
128	if (pVM)
129	VMXR0CheckError(pVM, &pVM->aCpus[0], rc);
130	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
131	return VERR_VMX_VMXON_FAILED;
132	}
133	return VINF_SUCCESS;
134	}
135
136	/**
137	* Deactivates VT-x on the current CPU
138	*
139	* @returns VBox status code.
140	* @param pCpu CPU info struct
141	* @param pvPageCpu Pointer to the global cpu page
142	* @param pPageCpuPhys Physical address of the global cpu page
143	*/
144	VMMR0DECL(int) VMXR0DisableCpu(PHWACCM_CPUINFO pCpu, void *pvPageCpu, RTHCPHYS pPageCpuPhys)
145	{
146	AssertReturn(pPageCpuPhys, VERR_INVALID_PARAMETER);
147	AssertReturn(pvPageCpu, VERR_INVALID_PARAMETER);
148
149	/* Leave VMX Root Mode. */
150	VMXDisable();
151
152	/* And clear the X86_CR4_VMXE bit */
153	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
154
155	#ifdef LOG_ENABLED
156	SUPR0Printf("VMXR0DisableCpu cpu %d\n", pCpu->idCpu);
157	#endif
158	return VINF_SUCCESS;
159	}
160
161	/**
162	* Does Ring-0 per VM VT-x init.
163	*
164	* @returns VBox status code.
165	* @param pVM The VM to operate on.
166	*/
167	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
168	{
169	int rc;
170
171	#ifdef LOG_ENABLED
172	SUPR0Printf("VMXR0InitVM %x\n", pVM);
173	#endif
174
175	pVM->hwaccm.s.vmx.pMemObjAPIC = NIL_RTR0MEMOBJ;
176
177	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
178	{
179	/* Allocate one page for the virtual APIC mmio cache. */
180	rc = RTR0MemObjAllocCont(&pVM->hwaccm.s.vmx.pMemObjAPIC, 1 << PAGE_SHIFT, true /* executable R0 mapping */);
181	AssertRC(rc);
182	if (RT_FAILURE(rc))
183	return rc;
184
185	pVM->hwaccm.s.vmx.pAPIC = (uint8_t *)RTR0MemObjAddress(pVM->hwaccm.s.vmx.pMemObjAPIC);
186	pVM->hwaccm.s.vmx.pAPICPhys = RTR0MemObjGetPagePhysAddr(pVM->hwaccm.s.vmx.pMemObjAPIC, 0);
187	ASMMemZero32(pVM->hwaccm.s.vmx.pAPIC, PAGE_SIZE);
188	}
189	else
190	{
191	pVM->hwaccm.s.vmx.pMemObjAPIC = 0;
192	pVM->hwaccm.s.vmx.pAPIC = 0;
193	pVM->hwaccm.s.vmx.pAPICPhys = 0;
194	}
195
196	/* Allocate the MSR bitmap if this feature is supported. */
197	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
198	{
199	rc = RTR0MemObjAllocCont(&pVM->hwaccm.s.vmx.pMemObjMSRBitmap, 1 << PAGE_SHIFT, true /* executable R0 mapping */);
200	AssertRC(rc);
201	if (RT_FAILURE(rc))
202	return rc;
203
204	pVM->hwaccm.s.vmx.pMSRBitmap = (uint8_t *)RTR0MemObjAddress(pVM->hwaccm.s.vmx.pMemObjMSRBitmap);
205	pVM->hwaccm.s.vmx.pMSRBitmapPhys = RTR0MemObjGetPagePhysAddr(pVM->hwaccm.s.vmx.pMemObjMSRBitmap, 0);
206	memset(pVM->hwaccm.s.vmx.pMSRBitmap, 0xff, PAGE_SIZE);
207	}
208
209	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
210	{
211	rc = RTR0MemObjAllocCont(&pVM->hwaccm.s.vmx.pMemObjScratch, 1 << PAGE_SHIFT, true /* executable R0 mapping */);
212	AssertRC(rc);
213	if (RT_FAILURE(rc))
214	return rc;
215
216	pVM->hwaccm.s.vmx.pScratch = (uint8_t *)RTR0MemObjAddress(pVM->hwaccm.s.vmx.pMemObjScratch);
217	pVM->hwaccm.s.vmx.pScratchPhys = RTR0MemObjGetPagePhysAddr(pVM->hwaccm.s.vmx.pMemObjScratch, 0);
218
219	ASMMemZero32(pVM->hwaccm.s.vmx.pScratch, PAGE_SIZE);
220	strcpy((char *)pVM->hwaccm.s.vmx.pScratch, "SCRATCH Magic");
221	(uint64_t )(pVM->hwaccm.s.vmx.pScratch + 16) = UINT64_C(0xDEADBEEFDEADBEEF);
222	}
223	#endif
224
225	/* Allocate VMCBs for all guest CPUs. */
226	for (unsigned i=0;i<pVM->cCPUs;i++)
227	{
228	PVMCPU pVCpu = &pVM->aCpus[i];
229
230	pVCpu->hwaccm.s.vmx.pMemObjVMCS = NIL_RTR0MEMOBJ;
231
232	/* Allocate one page for the VM control structure (VMCS). */
233	rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.pMemObjVMCS, 1 << PAGE_SHIFT, true /* executable R0 mapping */);
234	AssertRC(rc);
235	if (RT_FAILURE(rc))
236	return rc;
237
238	pVCpu->hwaccm.s.vmx.pVMCS = RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.pMemObjVMCS);
239	pVCpu->hwaccm.s.vmx.pVMCSPhys = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.pMemObjVMCS, 0);
240	ASMMemZero32(pVCpu->hwaccm.s.vmx.pVMCS, PAGE_SIZE);
241
242	pVCpu->hwaccm.s.vmx.cr0_mask = 0;
243	pVCpu->hwaccm.s.vmx.cr4_mask = 0;
244
245	/* Current guest paging mode. */
246	pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode = PGMMODE_REAL;
247
248	#ifdef LOG_ENABLED
249	SUPR0Printf("VMXR0InitVM %x VMCS=%x (%x)\n", pVM, pVCpu->hwaccm.s.vmx.pVMCS, (uint32_t)pVCpu->hwaccm.s.vmx.pVMCSPhys);
250	#endif
251	}
252
253	return VINF_SUCCESS;
254	}
255
256	/**
257	* Does Ring-0 per VM VT-x termination.
258	*
259	* @returns VBox status code.
260	* @param pVM The VM to operate on.
261	*/
262	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
263	{
264	for (unsigned i=0;i<pVM->cCPUs;i++)
265	{
266	if (pVM->aCpus[i].hwaccm.s.vmx.pMemObjVMCS != NIL_RTR0MEMOBJ)
267	{
268	RTR0MemObjFree(pVM->aCpus[i].hwaccm.s.vmx.pMemObjVMCS, false);
269	pVM->aCpus[i].hwaccm.s.vmx.pMemObjVMCS = NIL_RTR0MEMOBJ;
270	pVM->aCpus[i].hwaccm.s.vmx.pVMCS = 0;
271	pVM->aCpus[i].hwaccm.s.vmx.pVMCSPhys = 0;
272	}
273	}
274	if (pVM->hwaccm.s.vmx.pMemObjAPIC != NIL_RTR0MEMOBJ)
275	{
276	RTR0MemObjFree(pVM->hwaccm.s.vmx.pMemObjAPIC, false);
277	pVM->hwaccm.s.vmx.pMemObjAPIC = NIL_RTR0MEMOBJ;
278	pVM->hwaccm.s.vmx.pAPIC = 0;
279	pVM->hwaccm.s.vmx.pAPICPhys = 0;
280	}
281	if (pVM->hwaccm.s.vmx.pMemObjMSRBitmap != NIL_RTR0MEMOBJ)
282	{
283	RTR0MemObjFree(pVM->hwaccm.s.vmx.pMemObjMSRBitmap, false);
284	pVM->hwaccm.s.vmx.pMemObjMSRBitmap = NIL_RTR0MEMOBJ;
285	pVM->hwaccm.s.vmx.pMSRBitmap = 0;
286	pVM->hwaccm.s.vmx.pMSRBitmapPhys = 0;
287	}
288	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
289	if (pVM->hwaccm.s.vmx.pMemObjScratch != NIL_RTR0MEMOBJ)
290	{
291	ASMMemZero32(pVM->hwaccm.s.vmx.pScratch, PAGE_SIZE);
292	RTR0MemObjFree(pVM->hwaccm.s.vmx.pMemObjScratch, false);
293	pVM->hwaccm.s.vmx.pMemObjScratch = NIL_RTR0MEMOBJ;
294	pVM->hwaccm.s.vmx.pScratch = 0;
295	pVM->hwaccm.s.vmx.pScratchPhys = 0;
296	}
297	#endif
298	return VINF_SUCCESS;
299	}
300
301	/**
302	* Sets up VT-x for the specified VM
303	*
304	* @returns VBox status code.
305	* @param pVM The VM to operate on.
306	*/
307	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
308	{
309	int rc = VINF_SUCCESS;
310	uint32_t val;
311
312	AssertReturn(pVM, VERR_INVALID_PARAMETER);
313
314	for (unsigned i=0;i<pVM->cCPUs;i++)
315	{
316	PVMCPU pVCpu = &pVM->aCpus[i];
317
318	Assert(pVCpu->hwaccm.s.vmx.pVMCS);
319
320	/* Set revision dword at the beginning of the VMCS structure. */
321	(uint32_t )pVCpu->hwaccm.s.vmx.pVMCS = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hwaccm.s.vmx.msr.vmx_basic_info);
322
323	/* Clear VM Control Structure. */
324	Log(("pVMCSPhys = %RHp\n", pVCpu->hwaccm.s.vmx.pVMCSPhys));
325	rc = VMXClearVMCS(pVCpu->hwaccm.s.vmx.pVMCSPhys);
326	if (RT_FAILURE(rc))
327	goto vmx_end;
328
329	/* Activate the VM Control Structure. */
330	rc = VMXActivateVMCS(pVCpu->hwaccm.s.vmx.pVMCSPhys);
331	if (RT_FAILURE(rc))
332	goto vmx_end;
333
334	/* VMX_VMCS_CTRL_PIN_EXEC_CONTROLS
335	* Set required bits to one and zero according to the MSR capabilities.
336	*/
337	val = pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.disallowed0;
338	/* External and non-maskable interrupts cause VM-exits. */
339	val = val \| VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_EXT_INT_EXIT \| VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_NMI_EXIT;
340	val &= pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.allowed1;
341
342	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PIN_EXEC_CONTROLS, val);
343	AssertRC(rc);
344
345	/* VMX_VMCS_CTRL_PROC_EXEC_CONTROLS
346	* Set required bits to one and zero according to the MSR capabilities.
347	*/
348	val = pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.disallowed0;
349	/* Program which event cause VM-exits and which features we want to use. */
350	val = val \| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_HLT_EXIT
351	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_TSC_OFFSET
352	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT
353	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_UNCOND_IO_EXIT
354	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDPMC_EXIT
355	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MWAIT_EXIT; /* don't execute mwait or else we'll idle inside the guest (host thinks the cpu load is high) */
356
357	/* Without nested paging we should intercept invlpg and cr3 mov instructions. */
358	if (!pVM->hwaccm.s.fNestedPaging)
359	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT
360	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
361	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
362
363	/* Note: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MWAIT_EXIT might cause a vmlaunch failure with an invalid control fields error. (combined with some other exit reasons) */
364	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
365	{
366	/* CR8 reads from the APIC shadow page; writes cause an exit is they lower the TPR below the threshold */
367	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW;
368	Assert(pVM->hwaccm.s.vmx.pAPIC);
369	}
370	else
371	/* Exit on CR8 reads & writes in case the TPR shadow feature isn't present. */
372	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_STORE_EXIT \| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_LOAD_EXIT;
373
374	#ifdef VBOX_WITH_VTX_MSR_BITMAPS
375	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
376	{
377	Assert(pVM->hwaccm.s.vmx.pMSRBitmapPhys);
378	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS;
379	}
380	#endif
381
382	/* We will use the secondary control if it's present. */
383	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
384
385	/* Mask away the bits that the CPU doesn't support */
386	/** @todo make sure they don't conflict with the above requirements. */
387	val &= pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1;
388	pVCpu->hwaccm.s.vmx.proc_ctls = val;
389
390	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, val);
391	AssertRC(rc);
392
393	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
394	{
395	/* VMX_VMCS_CTRL_PROC_EXEC_CONTROLS2
396	* Set required bits to one and zero according to the MSR capabilities.
397	*/
398	val = pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0;
399	val \|= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT;
400
401	#ifdef HWACCM_VTX_WITH_EPT
402	if (pVM->hwaccm.s.fNestedPaging)
403	val \|= VMX_VMCS_CTRL_PROC_EXEC2_EPT;
404	#endif /* HWACCM_VTX_WITH_EPT */
405	#ifdef HWACCM_VTX_WITH_VPID
406	else
407	if (pVM->hwaccm.s.vmx.fVPID)
408	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VPID;
409	#endif /* HWACCM_VTX_WITH_VPID */
410
411	/* Mask away the bits that the CPU doesn't support */
412	/** @todo make sure they don't conflict with the above requirements. */
413	val &= pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1;
414
415	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS2, val);
416	AssertRC(rc);
417	}
418
419	/* VMX_VMCS_CTRL_CR3_TARGET_COUNT
420	* Set required bits to one and zero according to the MSR capabilities.
421	*/
422	rc = VMXWriteVMCS(VMX_VMCS_CTRL_CR3_TARGET_COUNT, 0);
423	AssertRC(rc);
424
425	/* Forward all exception except #NM & #PF to the guest.
426	* We always need to check pagefaults since our shadow page table can be out of sync.
427	* And we always lazily sync the FPU & XMM state.
428	*/
429
430	/** @todo Possible optimization:
431	* Keep the FPU and XMM state current in the EM thread. That way there's no need to
432	* lazily sync anything, but the downside is that we can't use the FPU stack or XMM
433	* registers ourselves of course.
434	*
435	* Note: only possible if the current state is actually ours (X86_CR0_TS flag)
436	*/
437
438	/* Don't filter page faults; all of them should cause a switch. */
439	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PAGEFAULT_ERROR_MASK, 0);
440	rc \|= VMXWriteVMCS(VMX_VMCS_CTRL_PAGEFAULT_ERROR_MATCH, 0);
441	AssertRC(rc);
442
443	/* Init TSC offset to zero. */
444	rc = VMXWriteVMCS64(VMX_VMCS_CTRL_TSC_OFFSET_FULL, 0);
445	AssertRC(rc);
446
447	rc = VMXWriteVMCS64(VMX_VMCS_CTRL_IO_BITMAP_A_FULL, 0);
448	AssertRC(rc);
449
450	rc = VMXWriteVMCS64(VMX_VMCS_CTRL_IO_BITMAP_B_FULL, 0);
451	AssertRC(rc);
452
453	/* Set the MSR bitmap address. */
454	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
455	{
456	/* Optional */
457	rc = VMXWriteVMCS64(VMX_VMCS_CTRL_MSR_BITMAP_FULL, pVM->hwaccm.s.vmx.pMSRBitmapPhys);
458	AssertRC(rc);
459	}
460
461	/* Clear MSR controls. */
462	rc = VMXWriteVMCS64(VMX_VMCS_CTRL_VMEXIT_MSR_STORE_FULL, 0);
463	rc \|= VMXWriteVMCS64(VMX_VMCS_CTRL_VMEXIT_MSR_LOAD_FULL, 0);
464	rc \|= VMXWriteVMCS64(VMX_VMCS_CTRL_VMENTRY_MSR_LOAD_FULL, 0);
465	rc \|= VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_MSR_STORE_COUNT, 0);
466	rc \|= VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_MSR_LOAD_COUNT, 0);
467	AssertRC(rc);
468
469	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
470	{
471	Assert(pVM->hwaccm.s.vmx.pMemObjAPIC);
472	/* Optional */
473	rc = VMXWriteVMCS(VMX_VMCS_CTRL_TPR_THRESHOLD, 0);
474	rc \|= VMXWriteVMCS64(VMX_VMCS_CTRL_VAPIC_PAGEADDR_FULL, pVM->hwaccm.s.vmx.pAPICPhys);
475	AssertRC(rc);
476	}
477
478	/* Set link pointer to -1. Not currently used. */
479	rc = VMXWriteVMCS64(VMX_VMCS_GUEST_LINK_PTR_FULL, 0xFFFFFFFFFFFFFFFFULL);
480	AssertRC(rc);
481
482	/* Clear VM Control Structure. Marking it inactive, clearing implementation specific data and writing back VMCS data to memory. */
483	rc = VMXClearVMCS(pVCpu->hwaccm.s.vmx.pVMCSPhys);
484	AssertRC(rc);
485
486	/* Configure the VMCS read cache. */
487	PVMCSCACHE pCache = &pVCpu->hwaccm.s.vmx.VMCSCache;
488
489	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_RIP);
490	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_RSP);
491	VMXSetupCachedReadVMCS(pCache, VMX_VMCS_GUEST_RFLAGS);
492	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE);
493	VMXSetupCachedReadVMCS(pCache, VMX_VMCS_CTRL_CR0_READ_SHADOW);
494	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_CR0);
495	VMXSetupCachedReadVMCS(pCache, VMX_VMCS_CTRL_CR4_READ_SHADOW);
496	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_CR4);
497	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_DR7);
498	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_SYSENTER_CS);
499	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_SYSENTER_EIP);
500	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_SYSENTER_ESP);
501	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_GDTR_LIMIT);
502	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_GDTR_BASE);
503	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_IDTR_LIMIT);
504	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_IDTR_BASE);
505
506	VMX_SETUP_SELREG(ES, pCache);
507	VMX_SETUP_SELREG(SS, pCache);
508	VMX_SETUP_SELREG(CS, pCache);
509	VMX_SETUP_SELREG(DS, pCache);
510	VMX_SETUP_SELREG(FS, pCache);
511	VMX_SETUP_SELREG(GS, pCache);
512	VMX_SETUP_SELREG(LDTR, pCache);
513	VMX_SETUP_SELREG(TR, pCache);
514
515	/* Status code VMCS reads. */
516	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_REASON);
517	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_VM_INSTR_ERROR);
518	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INSTR_LENGTH);
519	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INTERRUPTION_ERRCODE);
520	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO);
521	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INSTR_INFO);
522	VMXSetupCachedReadVMCS(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
523	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_IDT_INFO);
524	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_IDT_ERRCODE);
525
526	if (pVM->hwaccm.s.fNestedPaging)
527	{
528	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_CR3);
529	VMXSetupCachedReadVMCS(pCache, VMX_VMCS_EXIT_PHYS_ADDR_FULL);
530	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
531	}
532	else
533	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
534	} /* for each VMCPU */
535
536	/* Choose the right TLB setup function. */
537	if (pVM->hwaccm.s.fNestedPaging)
538	{
539	pVM->hwaccm.s.vmx.pfnSetupTaggedTLB = vmxR0SetupTLBEPT;
540
541	/* Default values for flushing. */
542	pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_ALL_CONTEXTS;
543	pVM->hwaccm.s.vmx.enmFlushContext = VMX_FLUSH_ALL_CONTEXTS;
544
545	/* If the capabilities specify we can do more, then make use of it. */
546	if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVEPT_CAPS_INDIV)
547	pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_PAGE;
548	else
549	if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVEPT_CAPS_CONTEXT)
550	pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_SINGLE_CONTEXT;
551
552	if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVEPT_CAPS_CONTEXT)
553	pVM->hwaccm.s.vmx.enmFlushContext = VMX_FLUSH_SINGLE_CONTEXT;
554	}
555	#ifdef HWACCM_VTX_WITH_VPID
556	else
557	if (pVM->hwaccm.s.vmx.fVPID)
558	{
559	pVM->hwaccm.s.vmx.pfnSetupTaggedTLB = vmxR0SetupTLBVPID;
560
561	/* Default values for flushing. */
562	pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_ALL_CONTEXTS;
563	pVM->hwaccm.s.vmx.enmFlushContext = VMX_FLUSH_ALL_CONTEXTS;
564
565	/* If the capabilities specify we can do more, then make use of it. */
566	if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_INDIV)
567	pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_PAGE;
568	else
569	if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_CONTEXT)
570	pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_SINGLE_CONTEXT;
571
572	if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_CONTEXT)
573	pVM->hwaccm.s.vmx.enmFlushContext = VMX_FLUSH_SINGLE_CONTEXT;
574	}
575	#endif /* HWACCM_VTX_WITH_VPID */
576	else
577	pVM->hwaccm.s.vmx.pfnSetupTaggedTLB = vmxR0SetupTLBDummy;
578
579	vmx_end:
580	VMXR0CheckError(pVM, &pVM->aCpus[0], rc);
581	return rc;
582	}
583
584
585	/**
586	* Injects an event (trap or external interrupt)
587	*
588	* @returns VBox status code.
589	* @param pVM The VM to operate on.
590	* @param pVCpu The VMCPU to operate on.
591	* @param pCtx CPU Context
592	* @param intInfo VMX interrupt info
593	* @param cbInstr Opcode length of faulting instruction
594	* @param errCode Error code (optional)
595	*/
596	static int VMXR0InjectEvent(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, uint32_t intInfo, uint32_t cbInstr, uint32_t errCode)
597	{
598	int rc;
599	uint32_t iGate = VMX_EXIT_INTERRUPTION_INFO_VECTOR(intInfo);
600
601	#ifdef VBOX_STRICT
602	if (iGate == 0xE)
603	LogFlow(("VMXR0InjectEvent: Injecting interrupt %d at %RGv error code=%08x CR2=%RGv intInfo=%08x\n", iGate, (RTGCPTR)pCtx->rip, errCode, pCtx->cr2, intInfo));
604	else
605	if (iGate < 0x20)
606	LogFlow(("VMXR0InjectEvent: Injecting interrupt %d at %RGv error code=%08x\n", iGate, (RTGCPTR)pCtx->rip, errCode));
607	else
608	{
609	LogFlow(("INJ-EI: %x at %RGv\n", iGate, (RTGCPTR)pCtx->rip));
610	Assert(VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW \|\| !VM_FF_ISSET(pVM, VM_FF_INHIBIT_INTERRUPTS));
611	Assert(VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW \|\| pCtx->eflags.u32 & X86_EFL_IF);
612	}
613	#endif
614
615	#ifdef HWACCM_VMX_EMULATE_REALMODE
616	if (CPUMIsGuestInRealModeEx(pCtx))
617	{
618	RTGCPHYS GCPhysHandler;
619	uint16_t offset, ip;
620	RTSEL sel;
621
622	/* Injecting events doesn't work right with real mode emulation.
623	* (#GP if we try to inject external hardware interrupts)
624	* Inject the interrupt or trap directly instead.
625	*
626	* ASSUMES no access handlers for the bits we read or write below (should be safe).
627	*/
628	Log(("Manual interrupt/trap '%x' inject (real mode)\n", iGate));
629
630	/* Check if the interrupt handler is present. */
631	if (iGate * 4 + 3 > pCtx->idtr.cbIdt)
632	{
633	Log(("IDT cbIdt violation\n"));
634	if (iGate != X86_XCPT_DF)
635	{
636	RTGCUINTPTR intInfo;
637
638	intInfo = (iGate == X86_XCPT_GP) ? (uint32_t)X86_XCPT_DF : iGate;
639	intInfo \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
640	intInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
641	intInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
642
643	return VMXR0InjectEvent(pVM, pVCpu, pCtx, intInfo, 0, 0 /* no error code according to the Intel docs */);
644	}
645	Log(("Triple fault -> reset the VM!\n"));
646	return VINF_EM_RESET;
647	}
648	if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW
649	\|\| iGate == 3 /* Both #BP and #OF point to the instruction after. */
650	\|\| iGate == 4)
651	{
652	ip = pCtx->ip + cbInstr;
653	}
654	else
655	ip = pCtx->ip;
656
657	/* Read the selector:offset pair of the interrupt handler. */
658	GCPhysHandler = (RTGCPHYS)pCtx->idtr.pIdt + iGate * 4;
659	rc = PGMPhysSimpleReadGCPhys(pVM, &offset, GCPhysHandler, sizeof(offset)); AssertRC(rc);
660	rc = PGMPhysSimpleReadGCPhys(pVM, &sel, GCPhysHandler + 2, sizeof(sel)); AssertRC(rc);
661
662	LogFlow(("IDT handler %04X:%04X\n", sel, offset));
663
664	/* Construct the stack frame. */
665	/** @todo should check stack limit. */
666	pCtx->sp -= 2;
667	LogFlow(("ss:sp %04X:%04X eflags=%x\n", pCtx->ss, pCtx->sp, pCtx->eflags.u));
668	rc = PGMPhysSimpleWriteGCPhys(pVM, pCtx->ssHid.u64Base + pCtx->sp, &pCtx->eflags, sizeof(uint16_t)); AssertRC(rc);
669	pCtx->sp -= 2;
670	LogFlow(("ss:sp %04X:%04X cs=%x\n", pCtx->ss, pCtx->sp, pCtx->cs));
671	rc = PGMPhysSimpleWriteGCPhys(pVM, pCtx->ssHid.u64Base + pCtx->sp, &pCtx->cs, sizeof(uint16_t)); AssertRC(rc);
672	pCtx->sp -= 2;
673	LogFlow(("ss:sp %04X:%04X ip=%x\n", pCtx->ss, pCtx->sp, ip));
674	rc = PGMPhysSimpleWriteGCPhys(pVM, pCtx->ssHid.u64Base + pCtx->sp, &ip, sizeof(ip)); AssertRC(rc);
675
676	/* Update the CPU state for executing the handler. */
677	pCtx->rip = offset;
678	pCtx->cs = sel;
679	pCtx->csHid.u64Base = sel << 4;
680	pCtx->eflags.u &= ~(X86_EFL_IF\|X86_EFL_TF\|X86_EFL_RF\|X86_EFL_AC);
681
682	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_SEGMENT_REGS;
683	return VINF_SUCCESS;
684	}
685	#endif /* HWACCM_VMX_EMULATE_REALMODE */
686
687	/* Set event injection state. */
688	rc = VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_IRQ_INFO, intInfo \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT));
689
690	rc \|= VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
691	rc \|= VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_EXCEPTION_ERRCODE, errCode);
692
693	AssertRC(rc);
694	return rc;
695	}
696
697
698	/**
699	* Checks for pending guest interrupts and injects them
700	*
701	* @returns VBox status code.
702	* @param pVM The VM to operate on.
703	* @param pVCpu The VMCPU to operate on.
704	* @param pCtx CPU Context
705	*/
706	static int VMXR0CheckPendingInterrupt(PVM pVM, PVMCPU pVCpu, CPUMCTX *pCtx)
707	{
708	int rc;
709
710	/* Dispatch any pending interrupts. (injected before, but a VM exit occurred prematurely) */
711	if (pVCpu->hwaccm.s.Event.fPending)
712	{
713	Log(("Reinjecting event %RX64 %08x at %RGv cr2=%RX64\n", pVCpu->hwaccm.s.Event.intInfo, pVCpu->hwaccm.s.Event.errCode, (RTGCPTR)pCtx->rip, pCtx->cr2));
714	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatIntReinject);
715	rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, pVCpu->hwaccm.s.Event.intInfo, 0, pVCpu->hwaccm.s.Event.errCode);
716	AssertRC(rc);
717
718	pVCpu->hwaccm.s.Event.fPending = false;
719	return VINF_SUCCESS;
720	}
721
722	if (pVM->hwaccm.s.fInjectNMI)
723	{
724	RTGCUINTPTR intInfo;
725
726	intInfo = X86_XCPT_NMI;
727	intInfo \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
728	intInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
729
730	rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, intInfo, 0, 0);
731	AssertRC(rc);
732
733	pVM->hwaccm.s.fInjectNMI = false;
734	return VINF_SUCCESS;
735	}
736
737	/* When external interrupts are pending, we should exit the VM when IF is set. */
738	if ( !TRPMHasTrap(pVM)
739	&& VM_FF_ISPENDING(pVM, (VM_FF_INTERRUPT_APIC\|VM_FF_INTERRUPT_PIC)))
740	{
741	if (!(pCtx->eflags.u32 & X86_EFL_IF))
742	{
743	if (!(pVCpu->hwaccm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_IRQ_WINDOW_EXIT))
744	{
745	LogFlow(("Enable irq window exit!\n"));
746	pVCpu->hwaccm.s.vmx.proc_ctls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_IRQ_WINDOW_EXIT;
747	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
748	AssertRC(rc);
749	}
750	/* else nothing to do but wait */
751	}
752	else
753	if (!VM_FF_ISSET(pVM, VM_FF_INHIBIT_INTERRUPTS))
754	{
755	uint8_t u8Interrupt;
756
757	rc = PDMGetInterrupt(pVM, &u8Interrupt);
758	Log(("Dispatch interrupt: u8Interrupt=%x (%d) rc=%Rrc cs:rip=%04X:%RGv\n", u8Interrupt, u8Interrupt, rc, pCtx->cs, (RTGCPTR)pCtx->rip));
759	if (RT_SUCCESS(rc))
760	{
761	rc = TRPMAssertTrap(pVM, u8Interrupt, TRPM_HARDWARE_INT);
762	AssertRC(rc);
763	}
764	else
765	{
766	/* Can only happen in rare cases where a pending interrupt is cleared behind our back */
767	Assert(!VM_FF_ISPENDING(pVM, (VM_FF_INTERRUPT_APIC\|VM_FF_INTERRUPT_PIC)));
768	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatSwitchGuestIrq);
769	/* Just continue */
770	}
771	}
772	else
773	Log(("Pending interrupt blocked at %RGv by VM_FF_INHIBIT_INTERRUPTS!!\n", (RTGCPTR)pCtx->rip));
774	}
775
776	#ifdef VBOX_STRICT
777	if (TRPMHasTrap(pVM))
778	{
779	uint8_t u8Vector;
780	rc = TRPMQueryTrapAll(pVM, &u8Vector, 0, 0, 0);
781	AssertRC(rc);
782	}
783	#endif
784
785	if ( pCtx->eflags.u32 & X86_EFL_IF
786	&& (!VM_FF_ISSET(pVM, VM_FF_INHIBIT_INTERRUPTS))
787	&& TRPMHasTrap(pVM)
788	)
789	{
790	uint8_t u8Vector;
791	int rc;
792	TRPMEVENT enmType;
793	RTGCUINTPTR intInfo;
794	RTGCUINT errCode;
795
796	/* If a new event is pending, then dispatch it now. */
797	rc = TRPMQueryTrapAll(pVM, &u8Vector, &enmType, &errCode, 0);
798	AssertRC(rc);
799	Assert(pCtx->eflags.Bits.u1IF == 1 \|\| enmType == TRPM_TRAP);
800	Assert(enmType != TRPM_SOFTWARE_INT);
801
802	/* Clear the pending trap. */
803	rc = TRPMResetTrap(pVM);
804	AssertRC(rc);
805
806	intInfo = u8Vector;
807	intInfo \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
808
809	if (enmType == TRPM_TRAP)
810	{
811	switch (u8Vector) {
812	case 8:
813	case 10:
814	case 11:
815	case 12:
816	case 13:
817	case 14:
818	case 17:
819	/* Valid error codes. */
820	intInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
821	break;
822	default:
823	break;
824	}
825	if (u8Vector == X86_XCPT_BP \|\| u8Vector == X86_XCPT_OF)
826	intInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
827	else
828	intInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
829	}
830	else
831	intInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
832
833	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatIntInject);
834	rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, intInfo, 0, errCode);
835	AssertRC(rc);
836	} /* if (interrupts can be dispatched) */
837
838	return VINF_SUCCESS;
839	}
840
841	/**
842	* Save the host state
843	*
844	* @returns VBox status code.
845	* @param pVM The VM to operate on.
846	* @param pVCpu The VMCPU to operate on.
847	*/
848	VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
849	{
850	int rc = VINF_SUCCESS;
851
852	/*
853	* Host CPU Context
854	*/
855	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_HOST_CONTEXT)
856	{
857	RTIDTR idtr;
858	RTGDTR gdtr;
859	RTSEL SelTR;
860	PX86DESCHC pDesc;
861	uintptr_t trBase;
862	RTSEL cs;
863	RTSEL ss;
864	uint64_t cr3;
865
866	/* Control registers */
867	rc = VMXWriteVMCS(VMX_VMCS_HOST_CR0, ASMGetCR0());
868	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
869	if (VMX_IS_64BIT_HOST_MODE())
870	{
871	cr3 = hwaccmR0Get64bitCR3();
872	rc \|= VMXWriteVMCS64(VMX_VMCS_HOST_CR3, cr3);
873	}
874	else
875	#endif
876	{
877	cr3 = ASMGetCR3();
878	rc \|= VMXWriteVMCS(VMX_VMCS_HOST_CR3, cr3);
879	}
880	rc \|= VMXWriteVMCS(VMX_VMCS_HOST_CR4, ASMGetCR4());
881	AssertRC(rc);
882	Log2(("VMX_VMCS_HOST_CR0 %08x\n", ASMGetCR0()));
883	Log2(("VMX_VMCS_HOST_CR3 %08RX64\n", cr3));
884	Log2(("VMX_VMCS_HOST_CR4 %08x\n", ASMGetCR4()));
885
886	/* Selector registers. */
887	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
888	if (VMX_IS_64BIT_HOST_MODE())
889	{
890	cs = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
891	ss = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
892	}
893	else
894	{
895	/* sysenter loads LDT cs & ss, VMX doesn't like this. Load the GDT ones (safe). */
896	cs = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
897	ss = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
898	}
899	#else
900	cs = ASMGetCS();
901	ss = ASMGetSS();
902	#endif
903	Assert(!(cs & X86_SEL_LDT)); Assert((cs & X86_SEL_RPL) == 0);
904	Assert(!(ss & X86_SEL_LDT)); Assert((ss & X86_SEL_RPL) == 0);
905	rc = VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_CS, cs);
906	/* Note: VMX is (again) very picky about the RPL of the selectors here; we'll restore them manually. */
907	rc \|= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_DS, 0);
908	rc \|= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_ES, 0);
909	#if HC_ARCH_BITS == 32
910	if (!VMX_IS_64BIT_HOST_MODE())
911	{
912	rc \|= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_FS, 0);
913	rc \|= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_GS, 0);
914	}
915	#endif
916	rc \|= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_SS, ss);
917	SelTR = ASMGetTR();
918	rc \|= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_TR, SelTR);
919	AssertRC(rc);
920	Log2(("VMX_VMCS_HOST_FIELD_CS %08x (%08x)\n", cs, ASMGetSS()));
921	Log2(("VMX_VMCS_HOST_FIELD_DS 00000000 (%08x)\n", ASMGetDS()));
922	Log2(("VMX_VMCS_HOST_FIELD_ES 00000000 (%08x)\n", ASMGetES()));
923	Log2(("VMX_VMCS_HOST_FIELD_FS 00000000 (%08x)\n", ASMGetFS()));
924	Log2(("VMX_VMCS_HOST_FIELD_GS 00000000 (%08x)\n", ASMGetGS()));
925	Log2(("VMX_VMCS_HOST_FIELD_SS %08x (%08x)\n", ss, ASMGetSS()));
926	Log2(("VMX_VMCS_HOST_FIELD_TR %08x\n", ASMGetTR()));
927
928	/* GDTR & IDTR */
929	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
930	if (VMX_IS_64BIT_HOST_MODE())
931	{
932	X86XDTR64 gdtr64, idtr64;
933	hwaccmR0Get64bitGDTRandIDTR(&gdtr64, &idtr64);
934	rc = VMXWriteVMCS64(VMX_VMCS_HOST_GDTR_BASE, gdtr64.uAddr);
935	rc \|= VMXWriteVMCS64(VMX_VMCS_HOST_IDTR_BASE, gdtr64.uAddr);
936	AssertRC(rc);
937	Log2(("VMX_VMCS_HOST_GDTR_BASE %RX64\n", gdtr64.uAddr));
938	Log2(("VMX_VMCS_HOST_IDTR_BASE %RX64\n", idtr64.uAddr));
939	gdtr.cbGdt = gdtr64.cb;
940	gdtr.pGdt = (uintptr_t)gdtr64.uAddr;
941	}
942	else
943	#endif
944	{
945	ASMGetGDTR(&gdtr);
946	rc = VMXWriteVMCS(VMX_VMCS_HOST_GDTR_BASE, gdtr.pGdt);
947	ASMGetIDTR(&idtr);
948	rc \|= VMXWriteVMCS(VMX_VMCS_HOST_IDTR_BASE, idtr.pIdt);
949	AssertRC(rc);
950	Log2(("VMX_VMCS_HOST_GDTR_BASE %RHv\n", gdtr.pGdt));
951	Log2(("VMX_VMCS_HOST_IDTR_BASE %RHv\n", idtr.pIdt));
952	}
953
954
955	/* Save the base address of the TR selector. */
956	if (SelTR > gdtr.cbGdt)
957	{
958	AssertMsgFailed(("Invalid TR selector %x. GDTR.cbGdt=%x\n", SelTR, gdtr.cbGdt));
959	return VERR_VMX_INVALID_HOST_STATE;
960	}
961
962	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
963	if (VMX_IS_64BIT_HOST_MODE())
964	{
965	pDesc = &((PX86DESCHC)gdtr.pGdt)[SelTR >> X86_SEL_SHIFT_HC]; /// ????
966	uint64_t trBase64 = X86DESC64_BASE(*(PX86DESC64)pDesc);
967	rc = VMXWriteVMCS64(VMX_VMCS_HOST_TR_BASE, trBase64);
968	Log2(("VMX_VMCS_HOST_TR_BASE %RX64\n", trBase64));
969	AssertRC(rc);
970	}
971	else
972	#endif
973	{
974	pDesc = &((PX86DESCHC)gdtr.pGdt)[SelTR >> X86_SEL_SHIFT_HC];
975	#if HC_ARCH_BITS == 64
976	trBase = X86DESC64_BASE(*pDesc);
977	#else
978	trBase = X86DESC_BASE(*pDesc);
979	#endif
980	rc = VMXWriteVMCS(VMX_VMCS_HOST_TR_BASE, trBase);
981	AssertRC(rc);
982	Log2(("VMX_VMCS_HOST_TR_BASE %RHv\n", trBase));
983	}
984
985	/* FS and GS base. */
986	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
987	if (VMX_IS_64BIT_HOST_MODE())
988	{
989	Log2(("MSR_K8_FS_BASE = %RX64\n", ASMRdMsr(MSR_K8_FS_BASE)));
990	Log2(("MSR_K8_GS_BASE = %RX64\n", ASMRdMsr(MSR_K8_GS_BASE)));
991	rc = VMXWriteVMCS64(VMX_VMCS_HOST_FS_BASE, ASMRdMsr(MSR_K8_FS_BASE));
992	rc \|= VMXWriteVMCS64(VMX_VMCS_HOST_GS_BASE, ASMRdMsr(MSR_K8_GS_BASE));
993	}
994	#endif
995	AssertRC(rc);
996
997	/* Sysenter MSRs. */
998	/** @todo expensive!! */
999	rc = VMXWriteVMCS(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
1000	Log2(("VMX_VMCS_HOST_SYSENTER_CS %08x\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_CS)));
1001	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1002	if (VMX_IS_64BIT_HOST_MODE())
1003	{
1004	Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_EIP)));
1005	Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_ESP)));
1006	rc \|= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
1007	rc \|= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
1008	}
1009	else
1010	{
1011	rc \|= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
1012	rc \|= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
1013	Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP)));
1014	Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP)));
1015	}
1016	#elif HC_ARCH_BITS == 32
1017	rc \|= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
1018	rc \|= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
1019	Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP)));
1020	Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP)));
1021	#else
1022	Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_EIP)));
1023	Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_ESP)));
1024	rc \|= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
1025	rc \|= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
1026	#endif
1027	AssertRC(rc);
1028
1029	#if 0 /* @todo deal with 32/64 */
1030	/* Restore the host EFER - on CPUs that support it. */
1031	if (pVM->hwaccm.s.vmx.msr.vmx_exit.n.allowed1 & VMX_VMCS_CTRL_EXIT_CONTROLS_LOAD_HOST_EFER_MSR)
1032	{
1033	uint64_t msrEFER = ASMRdMsr(MSR_IA32_EFER);
1034	rc = VMXWriteVMCS64(VMX_VMCS_HOST_FIELD_EFER_FULL, msrEFER);
1035	AssertRC(rc);
1036	}
1037	#endif
1038	pVCpu->hwaccm.s.fContextUseFlags &= ~HWACCM_CHANGED_HOST_CONTEXT;
1039	}
1040	return rc;
1041	}
1042
1043	/**
1044	* Prefetch the 4 PDPT pointers (PAE and nested paging only)
1045	*
1046	* @param pVM The VM to operate on.
1047	* @param pVCpu The VMCPU to operate on.
1048	* @param pCtx Guest context
1049	*/
1050	static void vmxR0PrefetchPAEPdptrs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1051	{
1052	if (CPUMIsGuestInPAEModeEx(pCtx))
1053	{
1054	X86PDPE Pdpe;
1055
1056	for (unsigned i=0;i<4;i++)
1057	{
1058	Pdpe = PGMGstGetPaePDPtr(pVM, i);
1059	int rc = VMXWriteVMCS64(VMX_VMCS_GUEST_PDPTR0_FULL + i*2, Pdpe.u);
1060	AssertRC(rc);
1061	}
1062	}
1063	}
1064
1065	/**
1066	* Update the exception bitmap according to the current CPU state
1067	*
1068	* @param pVM The VM to operate on.
1069	* @param pVCpu The VMCPU to operate on.
1070	* @param pCtx Guest context
1071	*/
1072	static void vmxR0UpdateExceptionBitmap(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1073	{
1074	uint32_t u32TrapMask;
1075	Assert(pCtx);
1076
1077	u32TrapMask = HWACCM_VMX_TRAP_MASK;
1078	#ifndef DEBUG
1079	if (pVM->hwaccm.s.fNestedPaging)
1080	u32TrapMask &= ~RT_BIT(X86_XCPT_PF); /* no longer need to intercept #PF. */
1081	#endif
1082
1083	/* Also catch floating point exceptions as we need to report them to the guest in a different way. */
1084	if ( CPUMIsGuestFPUStateActive(pVCpu) == true
1085	&& !(pCtx->cr0 & X86_CR0_NE)
1086	&& !pVCpu->hwaccm.s.fFPUOldStyleOverride)
1087	{
1088	u32TrapMask \|= RT_BIT(X86_XCPT_MF);
1089	pVCpu->hwaccm.s.fFPUOldStyleOverride = true;
1090	}
1091
1092	#ifdef DEBUG /* till after branching, enable it by default then. */
1093	/* Intercept X86_XCPT_DB if stepping is enabled */
1094	if (DBGFIsStepping(pVM))
1095	u32TrapMask \|= RT_BIT(X86_XCPT_DB);
1096	/** @todo Don't trap it unless the debugger has armed breakpoints. */
1097	u32TrapMask \|= RT_BIT(X86_XCPT_BP);
1098	#endif
1099
1100	#ifdef VBOX_STRICT
1101	Assert(u32TrapMask & RT_BIT(X86_XCPT_GP));
1102	#endif
1103
1104	# ifdef HWACCM_VMX_EMULATE_REALMODE
1105	/* Intercept all exceptions in real mode as none of them can be injected directly (#GP otherwise). */
1106	if (CPUMIsGuestInRealModeEx(pCtx) && pVM->hwaccm.s.vmx.pRealModeTSS)
1107	u32TrapMask \|= HWACCM_VMX_TRAP_MASK_REALMODE;
1108	# endif /* HWACCM_VMX_EMULATE_REALMODE */
1109
1110	int rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXCEPTION_BITMAP, u32TrapMask);
1111	AssertRC(rc);
1112	}
1113
1114	/**
1115	* Loads the guest state
1116	*
1117	* NOTE: Don't do anything here that can cause a jump back to ring 3!!!!!
1118	*
1119	* @returns VBox status code.
1120	* @param pVM The VM to operate on.
1121	* @param pVCpu The VMCPU to operate on.
1122	* @param pCtx Guest context
1123	*/
1124	VMMR0DECL(int) VMXR0LoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1125	{
1126	int rc = VINF_SUCCESS;
1127	RTGCUINTPTR val;
1128	X86EFLAGS eflags;
1129
1130	/* VMX_VMCS_CTRL_ENTRY_CONTROLS
1131	* Set required bits to one and zero according to the MSR capabilities.
1132	*/
1133	val = pVM->hwaccm.s.vmx.msr.vmx_entry.n.disallowed0;
1134	/* Load guest debug controls (dr7 & IA32_DEBUGCTL_MSR) (forced to 1 on the 'first' VT-x capable CPUs; this actually includes the newest Nehalem CPUs) */
1135	val \|= VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_DEBUG;
1136	#if 0 /* @todo deal with 32/64 */
1137	/* Required for the EFER write below, not supported on all CPUs. */
1138	val \|= VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_GUEST_EFER_MSR;
1139	#endif
1140	/* 64 bits guest mode? */
1141	if (pCtx->msrEFER & MSR_K6_EFER_LMA)
1142	val \|= VMX_VMCS_CTRL_ENTRY_CONTROLS_IA64_MODE;
1143	/* else Must be zero when AMD64 is not available. */
1144
1145	/* Mask away the bits that the CPU doesn't support */
1146	val &= pVM->hwaccm.s.vmx.msr.vmx_entry.n.allowed1;
1147	rc = VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_CONTROLS, val);
1148	AssertRC(rc);
1149
1150	/* VMX_VMCS_CTRL_EXIT_CONTROLS
1151	* Set required bits to one and zero according to the MSR capabilities.
1152	*/
1153	val = pVM->hwaccm.s.vmx.msr.vmx_exit.n.disallowed0;
1154
1155	/* Save debug controls (dr7 & IA32_DEBUGCTL_MSR) (forced to 1 on the 'first' VT-x capable CPUs; this actually includes the newest Nehalem CPUs) */
1156	#if 0 /* @todo deal with 32/64 */
1157	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_DEBUG \| VMX_VMCS_CTRL_EXIT_CONTROLS_LOAD_HOST_EFER_MSR;
1158	#else
1159	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_DEBUG;
1160	#endif
1161
1162	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1163	if (VMX_IS_64BIT_HOST_MODE())
1164	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_AMD64;
1165	/* else: Must be zero when AMD64 is not available. */
1166	#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
1167	if (pCtx->msrEFER & MSR_K6_EFER_LMA)
1168	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_AMD64; /* our switcher goes to long mode */
1169	else
1170	Assert(!(val & VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_AMD64));
1171	#endif
1172	val &= pVM->hwaccm.s.vmx.msr.vmx_exit.n.allowed1;
1173	/* Don't acknowledge external interrupts on VM-exit. */
1174	rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_CONTROLS, val);
1175	AssertRC(rc);
1176
1177	/* Guest CPU context: ES, CS, SS, DS, FS, GS. */
1178	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_SEGMENT_REGS)
1179	{
1180	#ifdef HWACCM_VMX_EMULATE_REALMODE
1181	if (pVM->hwaccm.s.vmx.pRealModeTSS)
1182	{
1183	PGMMODE enmGuestMode = PGMGetGuestMode(pVM);
1184	if (pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode != enmGuestMode)
1185	{
1186	/* Correct weird requirements for switching to protected mode. */
1187	if ( pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode == PGMMODE_REAL
1188	&& enmGuestMode >= PGMMODE_PROTECTED)
1189	{
1190	/* Flush the recompiler code cache as it's not unlikely
1191	* the guest will rewrite code it will later execute in real
1192	* mode (OpenBSD 4.0 is one such example)
1193	*/
1194	REMFlushTBs(pVM);
1195
1196	/* DPL of all hidden selector registers must match the current CPL (0). */
1197	pCtx->csHid.Attr.n.u2Dpl = 0;
1198	pCtx->csHid.Attr.n.u4Type = X86_SEL_TYPE_CODE \| X86_SEL_TYPE_RW_ACC;
1199
1200	pCtx->dsHid.Attr.n.u2Dpl = 0;
1201	pCtx->esHid.Attr.n.u2Dpl = 0;
1202	pCtx->fsHid.Attr.n.u2Dpl = 0;
1203	pCtx->gsHid.Attr.n.u2Dpl = 0;
1204	pCtx->ssHid.Attr.n.u2Dpl = 0;
1205
1206	/* The limit must correspond to the granularity bit. */
1207	if (!pCtx->csHid.Attr.n.u1Granularity)
1208	pCtx->csHid.u32Limit &= 0xffff;
1209	if (!pCtx->dsHid.Attr.n.u1Granularity)
1210	pCtx->dsHid.u32Limit &= 0xffff;
1211	if (!pCtx->esHid.Attr.n.u1Granularity)
1212	pCtx->esHid.u32Limit &= 0xffff;
1213	if (!pCtx->fsHid.Attr.n.u1Granularity)
1214	pCtx->fsHid.u32Limit &= 0xffff;
1215	if (!pCtx->gsHid.Attr.n.u1Granularity)
1216	pCtx->gsHid.u32Limit &= 0xffff;
1217	if (!pCtx->ssHid.Attr.n.u1Granularity)
1218	pCtx->ssHid.u32Limit &= 0xffff;
1219	}
1220	else
1221	/* Switching from protected mode to real mode. */
1222	if ( pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode >= PGMMODE_PROTECTED
1223	&& enmGuestMode == PGMMODE_REAL)
1224	{
1225	/* The limit must also be set to 0xffff. */
1226	pCtx->csHid.u32Limit = 0xffff;
1227	pCtx->dsHid.u32Limit = 0xffff;
1228	pCtx->esHid.u32Limit = 0xffff;
1229	pCtx->fsHid.u32Limit = 0xffff;
1230	pCtx->gsHid.u32Limit = 0xffff;
1231	pCtx->ssHid.u32Limit = 0xffff;
1232
1233	Assert(pCtx->csHid.u64Base <= 0xfffff);
1234	Assert(pCtx->dsHid.u64Base <= 0xfffff);
1235	Assert(pCtx->esHid.u64Base <= 0xfffff);
1236	Assert(pCtx->fsHid.u64Base <= 0xfffff);
1237	Assert(pCtx->gsHid.u64Base <= 0xfffff);
1238	}
1239	pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode = enmGuestMode;
1240	}
1241	else
1242	/* VT-x will fail with a guest invalid state otherwise... (CPU state after a reset) */
1243	if ( CPUMIsGuestInRealModeEx(pCtx)
1244	&& pCtx->csHid.u64Base == 0xffff0000)
1245	{
1246	pCtx->csHid.u64Base = 0xf0000;
1247	pCtx->cs = 0xf000;
1248	}
1249	}
1250	#endif /* HWACCM_VMX_EMULATE_REALMODE */
1251
1252	VMX_WRITE_SELREG(ES, es);
1253	AssertRC(rc);
1254
1255	VMX_WRITE_SELREG(CS, cs);
1256	AssertRC(rc);
1257
1258	VMX_WRITE_SELREG(SS, ss);
1259	AssertRC(rc);
1260
1261	VMX_WRITE_SELREG(DS, ds);
1262	AssertRC(rc);
1263
1264	/* The base values in the hidden fs & gs registers are not in sync with the msrs; they are cut to 32 bits. */
1265	VMX_WRITE_SELREG(FS, fs);
1266	AssertRC(rc);
1267
1268	VMX_WRITE_SELREG(GS, gs);
1269	AssertRC(rc);
1270	}
1271
1272	/* Guest CPU context: LDTR. */
1273	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_LDTR)
1274	{
1275	if (pCtx->ldtr == 0)
1276	{
1277	rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_LDTR, 0);
1278	rc \|= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_LIMIT, 0);
1279	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_LDTR_BASE, 0);
1280	/* Note: vmlaunch will fail with 0 or just 0x02. No idea why. */
1281	rc \|= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, 0x82 /* present, LDT */);
1282	}
1283	else
1284	{
1285	rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_LDTR, pCtx->ldtr);
1286	rc \|= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtrHid.u32Limit);
1287	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_LDTR_BASE, pCtx->ldtrHid.u64Base);
1288	rc \|= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, pCtx->ldtrHid.Attr.u);
1289	}
1290	AssertRC(rc);
1291	}
1292	/* Guest CPU context: TR. */
1293	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_TR)
1294	{
1295	#ifdef HWACCM_VMX_EMULATE_REALMODE
1296	/* Real mode emulation using v86 mode with CR4.VME (interrupt redirection using the int bitmap in the TSS) */
1297	if (CPUMIsGuestInRealModeEx(pCtx))
1298	{
1299	RTGCPHYS GCPhys;
1300
1301	/* We convert it here every time as pci regions could be reconfigured. */
1302	rc = PDMVMMDevHeapR3ToGCPhys(pVM, pVM->hwaccm.s.vmx.pRealModeTSS, &GCPhys);
1303	AssertRC(rc);
1304
1305	rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_TR, 0);
1306	rc \|= VMXWriteVMCS(VMX_VMCS32_GUEST_TR_LIMIT, HWACCM_VTX_TSS_SIZE);
1307	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_TR_BASE, GCPhys /* phys = virt in this mode */);
1308
1309	X86DESCATTR attr;
1310
1311	attr.u = 0;
1312	attr.n.u1Present = 1;
1313	attr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
1314	val = attr.u;
1315	}
1316	else
1317	#endif /* HWACCM_VMX_EMULATE_REALMODE */
1318	{
1319	rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_TR, pCtx->tr);
1320	rc \|= VMXWriteVMCS(VMX_VMCS32_GUEST_TR_LIMIT, pCtx->trHid.u32Limit);
1321	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_TR_BASE, pCtx->trHid.u64Base);
1322
1323	val = pCtx->trHid.Attr.u;
1324
1325	/* The TSS selector must be busy. */
1326	if ((val & 0xF) == X86_SEL_TYPE_SYS_286_TSS_AVAIL)
1327	val = (val & ~0xF) \| X86_SEL_TYPE_SYS_286_TSS_BUSY;
1328	else
1329	/* Default even if no TR selector has been set (otherwise vmlaunch will fail!) */
1330	val = (val & ~0xF) \| X86_SEL_TYPE_SYS_386_TSS_BUSY;
1331
1332	}
1333	rc \|= VMXWriteVMCS(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, val);
1334	AssertRC(rc);
1335	}
1336	/* Guest CPU context: GDTR. */
1337	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_GDTR)
1338	{
1339	rc = VMXWriteVMCS(VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt);
1340	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_GDTR_BASE, pCtx->gdtr.pGdt);
1341	AssertRC(rc);
1342	}
1343	/* Guest CPU context: IDTR. */
1344	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_IDTR)
1345	{
1346	rc = VMXWriteVMCS(VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt);
1347	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_IDTR_BASE, pCtx->idtr.pIdt);
1348	AssertRC(rc);
1349	}
1350
1351	/*
1352	* Sysenter MSRs (unconditional)
1353	*/
1354	rc = VMXWriteVMCS(VMX_VMCS32_GUEST_SYSENTER_CS, pCtx->SysEnter.cs);
1355	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_SYSENTER_EIP, pCtx->SysEnter.eip);
1356	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_SYSENTER_ESP, pCtx->SysEnter.esp);
1357	AssertRC(rc);
1358
1359	/* Control registers */
1360	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_CR0)
1361	{
1362	val = pCtx->cr0;
1363	rc = VMXWriteVMCS(VMX_VMCS_CTRL_CR0_READ_SHADOW, val);
1364	Log2(("Guest CR0-shadow %08x\n", val));
1365	if (CPUMIsGuestFPUStateActive(pVCpu) == false)
1366	{
1367	/* Always use #NM exceptions to load the FPU/XMM state on demand. */
1368	val \|= X86_CR0_TS \| X86_CR0_ET \| X86_CR0_NE \| X86_CR0_MP;
1369	}
1370	else
1371	{
1372	/** @todo check if we support the old style mess correctly. */
1373	if (!(val & X86_CR0_NE))
1374	Log(("Forcing X86_CR0_NE!!!\n"));
1375
1376	val \|= X86_CR0_NE; /* always turn on the native mechanism to report FPU errors (old style uses interrupts) */
1377	}
1378	/* Note: protected mode & paging are always enabled; we use them for emulating real and protected mode without paging too. */
1379	val \|= X86_CR0_PE \| X86_CR0_PG;
1380	if (pVM->hwaccm.s.fNestedPaging)
1381	{
1382	if (CPUMIsGuestInPagedProtectedModeEx(pCtx))
1383	{
1384	/* Disable cr3 read/write monitoring as we don't need it for EPT. */
1385	pVCpu->hwaccm.s.vmx.proc_ctls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
1386	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT);
1387	}
1388	else
1389	{
1390	/* Reenable cr3 read/write monitoring as our identity mapped page table is active. */
1391	pVCpu->hwaccm.s.vmx.proc_ctls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
1392	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
1393	}
1394	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1395	AssertRC(rc);
1396	}
1397	else
1398	{
1399	/* Note: We must also set this as we rely on protecting various pages for which supervisor writes must be caught. */
1400	val \|= X86_CR0_WP;
1401	}
1402
1403	/* Always enable caching. */
1404	val &= ~(X86_CR0_CD\|X86_CR0_NW);
1405
1406	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_CR0, val);
1407	Log2(("Guest CR0 %08x\n", val));
1408	/* CR0 flags owned by the host; if the guests attempts to change them, then
1409	* the VM will exit.
1410	*/
1411	val = X86_CR0_PE /* Must monitor this bit (assumptions are made for real mode emulation) */
1412	\| X86_CR0_WP /* Must monitor this bit (it must always be enabled). */
1413	\| X86_CR0_PG /* Must monitor this bit (assumptions are made for real mode & protected mode without paging emulation) */
1414	\| X86_CR0_TS
1415	\| X86_CR0_ET /* Bit not restored during VM-exit! */
1416	\| X86_CR0_CD /* Bit not restored during VM-exit! */
1417	\| X86_CR0_NW /* Bit not restored during VM-exit! */
1418	\| X86_CR0_NE
1419	\| X86_CR0_MP;
1420	pVCpu->hwaccm.s.vmx.cr0_mask = val;
1421
1422	rc \|= VMXWriteVMCS(VMX_VMCS_CTRL_CR0_MASK, val);
1423	Log2(("Guest CR0-mask %08x\n", val));
1424	AssertRC(rc);
1425	}
1426	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_CR4)
1427	{
1428	/* CR4 */
1429	rc = VMXWriteVMCS(VMX_VMCS_CTRL_CR4_READ_SHADOW, pCtx->cr4);
1430	Log2(("Guest CR4-shadow %08x\n", pCtx->cr4));
1431	/* Set the required bits in cr4 too (currently X86_CR4_VMXE). */
1432	val = pCtx->cr4 \| (uint32_t)pVM->hwaccm.s.vmx.msr.vmx_cr4_fixed0;
1433
1434	if (!pVM->hwaccm.s.fNestedPaging)
1435	{
1436	switch(pVCpu->hwaccm.s.enmShadowMode)
1437	{
1438	case PGMMODE_REAL: /* Real mode -> emulated using v86 mode */
1439	case PGMMODE_PROTECTED: /* Protected mode, no paging -> emulated using identity mapping. */
1440	case PGMMODE_32_BIT: /* 32-bit paging. */
1441	val &= ~X86_CR4_PAE;
1442	break;
1443
1444	case PGMMODE_PAE: /* PAE paging. */
1445	case PGMMODE_PAE_NX: /* PAE paging with NX enabled. */
1446	/** @todo use normal 32 bits paging */
1447	val \|= X86_CR4_PAE;
1448	break;
1449
1450	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
1451	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
1452	#ifdef VBOX_ENABLE_64_BITS_GUESTS
1453	break;
1454	#else
1455	AssertFailed();
1456	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1457	#endif
1458	default: /* shut up gcc */
1459	AssertFailed();
1460	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1461	}
1462	}
1463	else
1464	if (!CPUMIsGuestInPagedProtectedModeEx(pCtx))
1465	{
1466	/* We use 4 MB pages in our identity mapping page table for real and protected mode without paging. */
1467	val \|= X86_CR4_PSE;
1468	/* Our identity mapping is a 32 bits page directory. */
1469	val &= ~X86_CR4_PAE;
1470	}
1471
1472	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_CR4, val);
1473	Log2(("Guest CR4 %08x\n", val));
1474	/* CR4 flags owned by the host; if the guests attempts to change them, then
1475	* the VM will exit.
1476	*/
1477	val = 0
1478	\| X86_CR4_PAE
1479	\| X86_CR4_PGE
1480	\| X86_CR4_PSE
1481	\| X86_CR4_VMXE;
1482	pVCpu->hwaccm.s.vmx.cr4_mask = val;
1483
1484	rc \|= VMXWriteVMCS(VMX_VMCS_CTRL_CR4_MASK, val);
1485	Log2(("Guest CR4-mask %08x\n", val));
1486	AssertRC(rc);
1487	}
1488
1489	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_CR3)
1490	{
1491	if (pVM->hwaccm.s.fNestedPaging)
1492	{
1493	Assert(PGMGetHyperCR3(pVM));
1494	pVCpu->hwaccm.s.vmx.GCPhysEPTP = PGMGetHyperCR3(pVM);
1495
1496	Assert(!(pVCpu->hwaccm.s.vmx.GCPhysEPTP & 0xfff));
1497	/** @todo Check the IA32_VMX_EPT_VPID_CAP MSR for other supported memory types. */
1498	pVCpu->hwaccm.s.vmx.GCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
1499	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
1500
1501	rc = VMXWriteVMCS64(VMX_VMCS_CTRL_EPTP_FULL, pVCpu->hwaccm.s.vmx.GCPhysEPTP);
1502	AssertRC(rc);
1503
1504	if (!CPUMIsGuestInPagedProtectedModeEx(pCtx))
1505	{
1506	RTGCPHYS GCPhys;
1507
1508	/* We convert it here every time as pci regions could be reconfigured. */
1509	rc = PDMVMMDevHeapR3ToGCPhys(pVM, pVM->hwaccm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
1510	AssertRC(rc);
1511
1512	/* We use our identity mapping page table here as we need to map guest virtual to guest physical addresses; EPT will
1513	* take care of the translation to host physical addresses.
1514	*/
1515	val = GCPhys;
1516	}
1517	else
1518	{
1519	/* Save the real guest CR3 in VMX_VMCS_GUEST_CR3 */
1520	val = pCtx->cr3;
1521	/* Prefetch the four PDPT entries in PAE mode. */
1522	vmxR0PrefetchPAEPdptrs(pVM, pVCpu, pCtx);
1523	}
1524	}
1525	else
1526	{
1527	val = PGMGetHyperCR3(pVM);
1528	Assert(val \|\| VM_FF_ISPENDING(pVM, VM_FF_PGM_SYNC_CR3 \| VM_FF_PGM_SYNC_CR3_NON_GLOBAL));
1529	}
1530
1531	/* Save our shadow CR3 register. */
1532	rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_CR3, val);
1533	AssertRC(rc);
1534	}
1535
1536	/* Debug registers. */
1537	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_DEBUG)
1538	{
1539	pCtx->dr[6] \|= X86_DR6_INIT_VAL; /* set all reserved bits to 1. */
1540	pCtx->dr[6] &= ~RT_BIT(12); /* must be zero. */
1541
1542	pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
1543	pCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* must be zero */
1544	pCtx->dr[7] \|= 0x400; /* must be one */
1545
1546	/* Resync DR7 */
1547	rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_DR7, pCtx->dr[7]);
1548	AssertRC(rc);
1549
1550	/* Sync the debug state now if any breakpoint is armed. */
1551	if ( (pCtx->dr[7] & (X86_DR7_ENABLED_MASK\|X86_DR7_GD))
1552	&& !CPUMIsGuestDebugStateActive(pVM)
1553	&& !DBGFIsStepping(pVM))
1554	{
1555	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatDRxArmed);
1556
1557	/* Disable drx move intercepts. */
1558	pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
1559	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1560	AssertRC(rc);
1561
1562	/* Save the host and load the guest debug state. */
1563	rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
1564	AssertRC(rc);
1565	}
1566
1567	/* IA32_DEBUGCTL MSR. */
1568	rc = VMXWriteVMCS64(VMX_VMCS_GUEST_DEBUGCTL_FULL, 0);
1569	AssertRC(rc);
1570
1571	/** @todo do we really ever need this? */
1572	rc \|= VMXWriteVMCS(VMX_VMCS_GUEST_DEBUG_EXCEPTIONS, 0);
1573	AssertRC(rc);
1574	}
1575
1576	/* EIP, ESP and EFLAGS */
1577	rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_RIP, pCtx->rip);
1578	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_RSP, pCtx->rsp);
1579	AssertRC(rc);
1580
1581	/* Bits 22-31, 15, 5 & 3 must be zero. Bit 1 must be 1. */
1582	eflags = pCtx->eflags;
1583	eflags.u32 &= VMX_EFLAGS_RESERVED_0;
1584	eflags.u32 \|= VMX_EFLAGS_RESERVED_1;
1585
1586	#ifdef HWACCM_VMX_EMULATE_REALMODE
1587	/* Real mode emulation using v86 mode. */
1588	if (CPUMIsGuestInRealModeEx(pCtx))
1589	{
1590	pVCpu->hwaccm.s.vmx.RealMode.eflags = eflags;
1591
1592	eflags.Bits.u1VM = 1;
1593	eflags.Bits.u2IOPL = 0; /* must always be 0 or else certain instructions won't cause faults. */
1594	}
1595	#endif /* HWACCM_VMX_EMULATE_REALMODE */
1596	rc = VMXWriteVMCS(VMX_VMCS_GUEST_RFLAGS, eflags.u32);
1597	AssertRC(rc);
1598
1599	/* TSC offset. */
1600	uint64_t u64TSCOffset;
1601
1602	if (TMCpuTickCanUseRealTSC(pVM, &u64TSCOffset))
1603	{
1604	/* Note: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT takes precedence over TSC_OFFSET */
1605	rc = VMXWriteVMCS64(VMX_VMCS_CTRL_TSC_OFFSET_FULL, u64TSCOffset);
1606	AssertRC(rc);
1607
1608	pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
1609	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1610	AssertRC(rc);
1611	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTSCOffset);
1612	}
1613	else
1614	{
1615	pVCpu->hwaccm.s.vmx.proc_ctls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
1616	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1617	AssertRC(rc);
1618	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTSCIntercept);
1619	}
1620
1621	/* 64 bits guest mode? */
1622	if (pCtx->msrEFER & MSR_K6_EFER_LMA)
1623	{
1624	#if !defined(VBOX_ENABLE_64_BITS_GUESTS)
1625	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1626	#elif HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1627	pVCpu->hwaccm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
1628	#else
1629	# ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1630	if (!pVM->hwaccm.s.fAllow64BitGuests)
1631	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1632	# endif
1633	pVCpu->hwaccm.s.vmx.pfnStartVM = VMXR0StartVM64;
1634	#endif
1635	/* Unconditionally update these as wrmsr might have changed them. */
1636	rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_FS_BASE, pCtx->fsHid.u64Base);
1637	AssertRC(rc);
1638	rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_GS_BASE, pCtx->gsHid.u64Base);
1639	AssertRC(rc);
1640	}
1641	else
1642	{
1643	pVCpu->hwaccm.s.vmx.pfnStartVM = VMXR0StartVM32;
1644	}
1645
1646	#if 0 /* @todo deal with 32/64 */
1647	/* Unconditionally update the guest EFER - on CPUs that supports it. */
1648	if (pVM->hwaccm.s.vmx.msr.vmx_entry.n.allowed1 & VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_GUEST_EFER_MSR)
1649	{
1650	rc = VMXWriteVMCS64(VMX_VMCS_GUEST_EFER_FULL, pCtx->msrEFER);
1651	AssertRC(rc);
1652	}
1653	#endif
1654
1655	vmxR0UpdateExceptionBitmap(pVM, pVCpu, pCtx);
1656
1657	/* Done. */
1658	pVCpu->hwaccm.s.fContextUseFlags &= ~HWACCM_CHANGED_ALL_GUEST;
1659
1660	return rc;
1661	}
1662
1663	/**
1664	* Syncs back the guest state
1665	*
1666	* @returns VBox status code.
1667	* @param pVM The VM to operate on.
1668	* @param pVCpu The VMCPU to operate on.
1669	* @param pCtx Guest context
1670	*/
1671	DECLINLINE(int) VMXR0SaveGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1672	{
1673	RTGCUINTREG val, valShadow;
1674	RTGCUINTPTR uInterruptState;
1675	int rc;
1676
1677	/* Let's first sync back eip, esp, and eflags. */
1678	rc = VMXReadCachedVMCS(VMX_VMCS64_GUEST_RIP, &val);
1679	AssertRC(rc);
1680	pCtx->rip = val;
1681	rc = VMXReadCachedVMCS(VMX_VMCS64_GUEST_RSP, &val);
1682	AssertRC(rc);
1683	pCtx->rsp = val;
1684	rc = VMXReadCachedVMCS(VMX_VMCS_GUEST_RFLAGS, &val);
1685	AssertRC(rc);
1686	pCtx->eflags.u32 = val;
1687
1688	/* Take care of instruction fusing (sti, mov ss) */
1689	rc \|= VMXReadCachedVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &val);
1690	uInterruptState = val;
1691	if (uInterruptState != 0)
1692	{
1693	Assert(uInterruptState <= 2); /* only sti & mov ss */
1694	Log(("uInterruptState %x eip=%RGv\n", uInterruptState, pCtx->rip));
1695	EMSetInhibitInterruptsPC(pVM, pCtx->rip);
1696	}
1697	else
1698	VM_FF_CLEAR(pVM, VM_FF_INHIBIT_INTERRUPTS);
1699
1700	/* Control registers. */
1701	VMXReadCachedVMCS(VMX_VMCS_CTRL_CR0_READ_SHADOW, &valShadow);
1702	VMXReadCachedVMCS(VMX_VMCS64_GUEST_CR0, &val);
1703	val = (valShadow & pVCpu->hwaccm.s.vmx.cr0_mask) \| (val & ~pVCpu->hwaccm.s.vmx.cr0_mask);
1704	CPUMSetGuestCR0(pVM, val);
1705
1706	VMXReadCachedVMCS(VMX_VMCS_CTRL_CR4_READ_SHADOW, &valShadow);
1707	VMXReadCachedVMCS(VMX_VMCS64_GUEST_CR4, &val);
1708	val = (valShadow & pVCpu->hwaccm.s.vmx.cr4_mask) \| (val & ~pVCpu->hwaccm.s.vmx.cr4_mask);
1709	CPUMSetGuestCR4(pVM, val);
1710
1711	/* Note: no reason to sync back the CRx registers. They can't be changed by the guest. */
1712	/* Note: only in the nested paging case can CR3 & CR4 be changed by the guest. */
1713	if ( pVM->hwaccm.s.fNestedPaging
1714	&& CPUMIsGuestInPagedProtectedModeEx(pCtx))
1715	{
1716	PVMCSCACHE pCache = &pVCpu->hwaccm.s.vmx.VMCSCache;
1717
1718	/* Can be updated behind our back in the nested paging case. */
1719	CPUMSetGuestCR2(pVM, pCache->cr2);
1720
1721	VMXReadCachedVMCS(VMX_VMCS64_GUEST_CR3, &val);
1722
1723	if (val != pCtx->cr3)
1724	{
1725	CPUMSetGuestCR3(pVM, val);
1726	PGMUpdateCR3(pVM, val);
1727	}
1728	/* Prefetch the four PDPT entries in PAE mode. */
1729	vmxR0PrefetchPAEPdptrs(pVM, pVCpu, pCtx);
1730	}
1731
1732	/* Sync back DR7 here. */
1733	VMXReadCachedVMCS(VMX_VMCS64_GUEST_DR7, &val);
1734	pCtx->dr[7] = val;
1735
1736	/* Guest CPU context: ES, CS, SS, DS, FS, GS. */
1737	VMX_READ_SELREG(ES, es);
1738	VMX_READ_SELREG(SS, ss);
1739	VMX_READ_SELREG(CS, cs);
1740	VMX_READ_SELREG(DS, ds);
1741	VMX_READ_SELREG(FS, fs);
1742	VMX_READ_SELREG(GS, gs);
1743
1744	/*
1745	* System MSRs
1746	*/
1747	VMXReadCachedVMCS(VMX_VMCS32_GUEST_SYSENTER_CS, &val);
1748	pCtx->SysEnter.cs = val;
1749	VMXReadCachedVMCS(VMX_VMCS64_GUEST_SYSENTER_EIP, &val);
1750	pCtx->SysEnter.eip = val;
1751	VMXReadCachedVMCS(VMX_VMCS64_GUEST_SYSENTER_ESP, &val);
1752	pCtx->SysEnter.esp = val;
1753
1754	/* Misc. registers; must sync everything otherwise we can get out of sync when jumping to ring 3. */
1755	VMX_READ_SELREG(LDTR, ldtr);
1756
1757	VMXReadCachedVMCS(VMX_VMCS32_GUEST_GDTR_LIMIT, &val);
1758	pCtx->gdtr.cbGdt = val;
1759	VMXReadCachedVMCS(VMX_VMCS64_GUEST_GDTR_BASE, &val);
1760	pCtx->gdtr.pGdt = val;
1761
1762	VMXReadCachedVMCS(VMX_VMCS32_GUEST_IDTR_LIMIT, &val);
1763	pCtx->idtr.cbIdt = val;
1764	VMXReadCachedVMCS(VMX_VMCS64_GUEST_IDTR_BASE, &val);
1765	pCtx->idtr.pIdt = val;
1766
1767	#ifdef HWACCM_VMX_EMULATE_REALMODE
1768	/* Real mode emulation using v86 mode. */
1769	if (CPUMIsGuestInRealModeEx(pCtx))
1770	{
1771	/* Hide our emulation flags */
1772	pCtx->eflags.Bits.u1VM = 0;
1773
1774	/* Restore original IOPL setting as we always use 0. */
1775	pCtx->eflags.Bits.u2IOPL = pVCpu->hwaccm.s.vmx.RealMode.eflags.Bits.u2IOPL;
1776
1777	/* Force a TR resync every time in case we switch modes. */
1778	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_TR;
1779	}
1780	else
1781	#endif /* HWACCM_VMX_EMULATE_REALMODE */
1782	{
1783	/* In real mode we have a fake TSS, so only sync it back when it's supposed to be valid. */
1784	VMX_READ_SELREG(TR, tr);
1785	}
1786	return VINF_SUCCESS;
1787	}
1788
1789	/**
1790	* Dummy placeholder
1791	*
1792	* @param pVM The VM to operate on.
1793	* @param pVCpu The VMCPU to operate on.
1794	*/
1795	static void vmxR0SetupTLBDummy(PVM pVM, PVMCPU pVCpu)
1796	{
1797	NOREF(pVM);
1798	NOREF(pVCpu);
1799	return;
1800	}
1801
1802	/**
1803	* Setup the tagged TLB for EPT
1804	*
1805	* @returns VBox status code.
1806	* @param pVM The VM to operate on.
1807	* @param pVCpu The VMCPU to operate on.
1808	*/
1809	static void vmxR0SetupTLBEPT(PVM pVM, PVMCPU pVCpu)
1810	{
1811	PHWACCM_CPUINFO pCpu;
1812
1813	Assert(pVM->hwaccm.s.fNestedPaging);
1814	Assert(!pVM->hwaccm.s.vmx.fVPID);
1815
1816	/* Deal with tagged TLBs if VPID or EPT is supported. */
1817	pCpu = HWACCMR0GetCurrentCpu();
1818	/* Force a TLB flush for the first world switch if the current cpu differs from the one we ran on last. */
1819	/* Note that this can happen both for start and resume due to long jumps back to ring 3. */
1820	if ( pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu
1821	/* if the tlb flush count has changed, another VM has flushed the TLB of this cpu, so we can't use our current ASID anymore. */
1822	\|\| pVCpu->hwaccm.s.cTLBFlushes != pCpu->cTLBFlushes)
1823	{
1824	/* Force a TLB flush on VM entry. */
1825	pVCpu->hwaccm.s.fForceTLBFlush = true;
1826	}
1827	else
1828	Assert(!pCpu->fFlushTLB);
1829
1830	pVCpu->hwaccm.s.idLastCpu = pCpu->idCpu;
1831	pCpu->fFlushTLB = false;
1832
1833	if (pVCpu->hwaccm.s.fForceTLBFlush)
1834	vmxR0FlushEPT(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushContext, 0);
1835
1836	#ifdef VBOX_WITH_STATISTICS
1837	if (pVCpu->hwaccm.s.fForceTLBFlush)
1838	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushTLBWorldSwitch);
1839	else
1840	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatNoFlushTLBWorldSwitch);
1841	#endif
1842	}
1843
1844	#ifdef HWACCM_VTX_WITH_VPID
1845	/**
1846	* Setup the tagged TLB for VPID
1847	*
1848	* @returns VBox status code.
1849	* @param pVM The VM to operate on.
1850	* @param pVCpu The VMCPU to operate on.
1851	*/
1852	static void vmxR0SetupTLBVPID(PVM pVM, PVMCPU pVCpu)
1853	{
1854	PHWACCM_CPUINFO pCpu;
1855
1856	Assert(pVM->hwaccm.s.vmx.fVPID);
1857	Assert(!pVM->hwaccm.s.fNestedPaging);
1858
1859	/* Deal with tagged TLBs if VPID or EPT is supported. */
1860	pCpu = HWACCMR0GetCurrentCpu();
1861	/* Force a TLB flush for the first world switch if the current cpu differs from the one we ran on last. */
1862	/* Note that this can happen both for start and resume due to long jumps back to ring 3. */
1863	if ( pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu
1864	/* if the tlb flush count has changed, another VM has flushed the TLB of this cpu, so we can't use our current ASID anymore. */
1865	\|\| pVCpu->hwaccm.s.cTLBFlushes != pCpu->cTLBFlushes)
1866	{
1867	/* Force a TLB flush on VM entry. */
1868	pVCpu->hwaccm.s.fForceTLBFlush = true;
1869	}
1870	else
1871	Assert(!pCpu->fFlushTLB);
1872
1873	pVCpu->hwaccm.s.idLastCpu = pCpu->idCpu;
1874
1875	/* Make sure we flush the TLB when required. Switch ASID to achieve the same thing, but without actually flushing the whole TLB (which is expensive). */
1876	if (pVCpu->hwaccm.s.fForceTLBFlush)
1877	{
1878	if ( ++pCpu->uCurrentASID >= pVM->hwaccm.s.uMaxASID
1879	\|\| pCpu->fFlushTLB)
1880	{
1881	pCpu->fFlushTLB = false;
1882	pCpu->uCurrentASID = 1; /* start at 1; host uses 0 */
1883	pCpu->cTLBFlushes++;
1884	}
1885	else
1886	{
1887	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushASID);
1888	pVCpu->hwaccm.s.fForceTLBFlush = false;
1889	}
1890
1891	pVCpu->hwaccm.s.cTLBFlushes = pCpu->cTLBFlushes;
1892	pVCpu->hwaccm.s.uCurrentASID = pCpu->uCurrentASID;
1893	}
1894	else
1895	{
1896	Assert(!pCpu->fFlushTLB);
1897
1898	if (!pCpu->uCurrentASID \|\| !pVCpu->hwaccm.s.uCurrentASID)
1899	pVCpu->hwaccm.s.uCurrentASID = pCpu->uCurrentASID = 1;
1900	}
1901	AssertMsg(pVCpu->hwaccm.s.cTLBFlushes == pCpu->cTLBFlushes, ("Flush count mismatch for cpu %d (%x vs %x)\n", pCpu->idCpu, pVCpu->hwaccm.s.cTLBFlushes, pCpu->cTLBFlushes));
1902	AssertMsg(pCpu->uCurrentASID >= 1 && pCpu->uCurrentASID < pVM->hwaccm.s.uMaxASID, ("cpu%d uCurrentASID = %x\n", pCpu->idCpu, pCpu->uCurrentASID));
1903	AssertMsg(pVCpu->hwaccm.s.uCurrentASID >= 1 && pVCpu->hwaccm.s.uCurrentASID < pVM->hwaccm.s.uMaxASID, ("cpu%d VM uCurrentASID = %x\n", pCpu->idCpu, pVCpu->hwaccm.s.uCurrentASID));
1904
1905	int rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hwaccm.s.uCurrentASID);
1906	AssertRC(rc);
1907
1908	if (pVCpu->hwaccm.s.fForceTLBFlush)
1909	vmxR0FlushVPID(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushContext, 0);
1910
1911	#ifdef VBOX_WITH_STATISTICS
1912	if (pVCpu->hwaccm.s.fForceTLBFlush)
1913	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushTLBWorldSwitch);
1914	else
1915	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatNoFlushTLBWorldSwitch);
1916	#endif
1917	}
1918	#endif /* HWACCM_VTX_WITH_VPID */
1919
1920	/**
1921	* Runs guest code in a VT-x VM.
1922	*
1923	* @returns VBox status code.
1924	* @param pVM The VM to operate on.
1925	* @param pVCpu The VMCPU to operate on.
1926	* @param pCtx Guest context
1927	*/
1928	VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1929	{
1930	int rc = VINF_SUCCESS;
1931	RTGCUINTREG val;
1932	RTGCUINTREG exitReason, instrError, cbInstr;
1933	RTGCUINTPTR exitQualification;
1934	RTGCUINTPTR intInfo = 0; /* shut up buggy gcc 4 */
1935	RTGCUINTPTR errCode, instrInfo;
1936	bool fSyncTPR = false;
1937	PHWACCM_CPUINFO pCpu = 0;
1938	unsigned cResume = 0;
1939	#ifdef VBOX_STRICT
1940	RTCPUID idCpuCheck;
1941	#endif
1942	#ifdef VBOX_WITH_STATISTICS
1943	bool fStatEntryStarted = true;
1944	bool fStatExit2Started = false;
1945	#endif
1946
1947	Log2(("\nE"));
1948
1949	STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatEntry, x);
1950
1951	#ifdef VBOX_STRICT
1952	{
1953	RTCCUINTREG val;
1954
1955	rc = VMXReadVMCS(VMX_VMCS_CTRL_PIN_EXEC_CONTROLS, &val);
1956	AssertRC(rc);
1957	Log2(("VMX_VMCS_CTRL_PIN_EXEC_CONTROLS = %08x\n", val));
1958
1959	/* allowed zero */
1960	if ((val & pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.disallowed0)
1961	Log(("Invalid VMX_VMCS_CTRL_PIN_EXEC_CONTROLS: zero\n"));
1962
1963	/* allowed one */
1964	if ((val & ~pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.allowed1) != 0)
1965	Log(("Invalid VMX_VMCS_CTRL_PIN_EXEC_CONTROLS: one\n"));
1966
1967	rc = VMXReadVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, &val);
1968	AssertRC(rc);
1969	Log2(("VMX_VMCS_CTRL_PROC_EXEC_CONTROLS = %08x\n", val));
1970
1971	/* Must be set according to the MSR, but can be cleared in case of EPT. */
1972	if (pVM->hwaccm.s.fNestedPaging)
1973	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT
1974	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
1975	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
1976
1977	/* allowed zero */
1978	if ((val & pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.disallowed0)
1979	Log(("Invalid VMX_VMCS_CTRL_PROC_EXEC_CONTROLS: zero\n"));
1980
1981	/* allowed one */
1982	if ((val & ~pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1) != 0)
1983	Log(("Invalid VMX_VMCS_CTRL_PROC_EXEC_CONTROLS: one\n"));
1984
1985	rc = VMXReadVMCS(VMX_VMCS_CTRL_ENTRY_CONTROLS, &val);
1986	AssertRC(rc);
1987	Log2(("VMX_VMCS_CTRL_ENTRY_CONTROLS = %08x\n", val));
1988
1989	/* allowed zero */
1990	if ((val & pVM->hwaccm.s.vmx.msr.vmx_entry.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_entry.n.disallowed0)
1991	Log(("Invalid VMX_VMCS_CTRL_ENTRY_CONTROLS: zero\n"));
1992
1993	/* allowed one */
1994	if ((val & ~pVM->hwaccm.s.vmx.msr.vmx_entry.n.allowed1) != 0)
1995	Log(("Invalid VMX_VMCS_CTRL_ENTRY_CONTROLS: one\n"));
1996
1997	rc = VMXReadVMCS(VMX_VMCS_CTRL_EXIT_CONTROLS, &val);
1998	AssertRC(rc);
1999	Log2(("VMX_VMCS_CTRL_EXIT_CONTROLS = %08x\n", val));
2000
2001	/* allowed zero */
2002	if ((val & pVM->hwaccm.s.vmx.msr.vmx_exit.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_exit.n.disallowed0)
2003	Log(("Invalid VMX_VMCS_CTRL_EXIT_CONTROLS: zero\n"));
2004
2005	/* allowed one */
2006	if ((val & ~pVM->hwaccm.s.vmx.msr.vmx_exit.n.allowed1) != 0)
2007	Log(("Invalid VMX_VMCS_CTRL_EXIT_CONTROLS: one\n"));
2008	}
2009	#endif
2010
2011	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2012	pVCpu->hwaccm.s.vmx.VMCSCache.u64TimeEntry = RTTimeNanoTS();
2013	#endif
2014
2015	/* We can jump to this point to resume execution after determining that a VM-exit is innocent.
2016	*/
2017	ResumeExecution:
2018	STAM_STATS({
2019	if (fStatExit2Started) { STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2, y); fStatExit2Started = false; }
2020	if (!fStatEntryStarted) { STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatEntry, x); fStatEntryStarted = true; }
2021	});
2022	AssertMsg(pVCpu->hwaccm.s.idEnteredCpu == RTMpCpuId(),
2023	("Expected %d, I'm %d; cResume=%d exitReason=%RGv exitQualification=%RGv\n",
2024	(int)pVCpu->hwaccm.s.idEnteredCpu, (int)RTMpCpuId(), cResume, exitReason, exitQualification));
2025	Assert(!HWACCMR0SuspendPending());
2026
2027	/* Safety precaution; looping for too long here can have a very bad effect on the host */
2028	if (++cResume > HWACCM_MAX_RESUME_LOOPS)
2029	{
2030	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitMaxResume);
2031	rc = VINF_EM_RAW_INTERRUPT;
2032	goto end;
2033	}
2034
2035	/* Check for irq inhibition due to instruction fusing (sti, mov ss). */
2036	if (VM_FF_ISSET(pVM, VM_FF_INHIBIT_INTERRUPTS))
2037	{
2038	Log(("VM_FF_INHIBIT_INTERRUPTS at %RGv successor %RGv\n", (RTGCPTR)pCtx->rip, EMGetInhibitInterruptsPC(pVM)));
2039	if (pCtx->rip != EMGetInhibitInterruptsPC(pVM))
2040	{
2041	/* Note: we intentionally don't clear VM_FF_INHIBIT_INTERRUPTS here.
2042	* Before we are able to execute this instruction in raw mode (iret to guest code) an external interrupt might
2043	* force a world switch again. Possibly allowing a guest interrupt to be dispatched in the process. This could
2044	* break the guest. Sounds very unlikely, but such timing sensitive problems are not as rare as you might think.
2045	*/
2046	VM_FF_CLEAR(pVM, VM_FF_INHIBIT_INTERRUPTS);
2047	/* Irq inhibition is no longer active; clear the corresponding VMX state. */
2048	rc = VMXWriteVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, 0);
2049	AssertRC(rc);
2050	}
2051	}
2052	else
2053	{
2054	/* Irq inhibition is no longer active; clear the corresponding VMX state. */
2055	rc = VMXWriteVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, 0);
2056	AssertRC(rc);
2057	}
2058
2059	/* Check for pending actions that force us to go back to ring 3. */
2060	if (VM_FF_ISPENDING(pVM, VM_FF_HWACCM_TO_R3_MASK))
2061	{
2062	VM_FF_CLEAR(pVM, VM_FF_TO_R3);
2063	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatSwitchToR3);
2064	rc = RT_UNLIKELY(VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
2065	goto end;
2066	}
2067	/* Pending request packets might contain actions that need immediate attention, such as pending hardware interrupts. */
2068	if (VM_FF_ISPENDING(pVM, VM_FF_REQUEST))
2069	{
2070	rc = VINF_EM_PENDING_REQUEST;
2071	goto end;
2072	}
2073
2074	/* When external interrupts are pending, we should exit the VM when IF is set. */
2075	/* Note! After VM_FF_INHIBIT_INTERRUPTS check!!! */
2076	rc = VMXR0CheckPendingInterrupt(pVM, pVCpu, pCtx);
2077	if (RT_FAILURE(rc))
2078	goto end;
2079
2080	/** @todo check timers?? */
2081
2082	/* TPR caching using CR8 is only available in 64 bits mode */
2083	/* Note the 32 bits exception for AMD (X86_CPUID_AMD_FEATURE_ECX_CR8L), but that appears missing in Intel CPUs */
2084	/* Note: we can't do this in LoadGuestState as PDMApicGetTPR can jump back to ring 3 (lock)!!!!! */
2085	/**
2086	* @todo reduce overhead
2087	*/
2088	if ( (pCtx->msrEFER & MSR_K6_EFER_LMA)
2089	&& pVM->hwaccm.s.vmx.pAPIC)
2090	{
2091	/* TPR caching in CR8 */
2092	uint8_t u8TPR;
2093	bool fPending;
2094
2095	int rc = PDMApicGetTPR(pVM, &u8TPR, &fPending);
2096	AssertRC(rc);
2097	/* The TPR can be found at offset 0x80 in the APIC mmio page. */
2098	pVM->hwaccm.s.vmx.pAPIC[0x80] = u8TPR << 4; /* bits 7-4 contain the task priority */
2099
2100	/* Two options here:
2101	* - external interrupt pending, but masked by the TPR value.
2102	* -> a CR8 update that lower the current TPR value should cause an exit
2103	* - no pending interrupts
2104	* -> We don't need to be explicitely notified. There are enough world switches for detecting pending interrupts.
2105	*/
2106	rc = VMXWriteVMCS(VMX_VMCS_CTRL_TPR_THRESHOLD, (fPending) ? u8TPR : 0);
2107	AssertRC(rc);
2108
2109	/* Always sync back the TPR; we should optimize this though / /* @todo optimize TPR sync. */
2110	fSyncTPR = true;
2111	}
2112
2113	#if defined(HWACCM_VTX_WITH_EPT) && defined(LOG_ENABLED)
2114	if ( pVM->hwaccm.s.fNestedPaging
2115	# ifdef HWACCM_VTX_WITH_VPID
2116	\|\| pVM->hwaccm.s.vmx.fVPID
2117	# endif /* HWACCM_VTX_WITH_VPID */
2118	)
2119	{
2120	pCpu = HWACCMR0GetCurrentCpu();
2121	if ( pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu
2122	\|\| pVCpu->hwaccm.s.cTLBFlushes != pCpu->cTLBFlushes)
2123	{
2124	if (pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu)
2125	Log(("Force TLB flush due to rescheduling to a different cpu (%d vs %d)\n", pVCpu->hwaccm.s.idLastCpu, pCpu->idCpu));
2126	else
2127	Log(("Force TLB flush due to changed TLB flush count (%x vs %x)\n", pVCpu->hwaccm.s.cTLBFlushes, pCpu->cTLBFlushes));
2128	}
2129	if (pCpu->fFlushTLB)
2130	Log(("Force TLB flush: first time cpu %d is used -> flush\n", pCpu->idCpu));
2131	else
2132	if (pVCpu->hwaccm.s.fForceTLBFlush)
2133	LogFlow(("Manual TLB flush\n"));
2134	}
2135	#endif
2136	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
2137	PGMDynMapFlushAutoSet(pVCpu);
2138	#endif
2139
2140	/*
2141	* NOTE: DO NOT DO ANYTHING AFTER THIS POINT THAT MIGHT JUMP BACK TO RING 3!
2142	* (until the actual world switch)
2143	*/
2144	#ifdef VBOX_STRICT
2145	idCpuCheck = RTMpCpuId();
2146	#endif
2147	#ifdef LOG_LOGGING
2148	VMMR0LogFlushDisable(pVCpu);
2149	#endif
2150	/* Save the host state first. */
2151	rc = VMXR0SaveHostState(pVM, pVCpu);
2152	if (rc != VINF_SUCCESS)
2153	goto end;
2154	/* Load the guest state */
2155	rc = VMXR0LoadGuestState(pVM, pVCpu, pCtx);
2156	if (rc != VINF_SUCCESS)
2157	goto end;
2158
2159	/* Deal with tagged TLB setup and invalidation. */
2160	pVM->hwaccm.s.vmx.pfnSetupTaggedTLB(pVM, pVCpu);
2161
2162	/* Non-register state Guest Context */
2163	/** @todo change me according to cpu state */
2164	rc = VMXWriteVMCS(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_CMS_GUEST_ACTIVITY_ACTIVE);
2165	AssertRC(rc);
2166
2167	STAM_STATS({ STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatEntry, x); fStatEntryStarted = false; });
2168
2169	/* Manual save and restore:
2170	* - General purpose registers except RIP, RSP
2171	*
2172	* Trashed:
2173	* - CR2 (we don't care)
2174	* - LDTR (reset to 0)
2175	* - DRx (presumably not changed at all)
2176	* - DR7 (reset to 0x400)
2177	* - EFLAGS (reset to RT_BIT(1); not relevant)
2178	*
2179	*/
2180
2181
2182	/* All done! Let's start VM execution. */
2183	STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatInGC, z);
2184	#ifdef VBOX_STRICT
2185	Assert(idCpuCheck == RTMpCpuId());
2186	#endif
2187
2188	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2189	pVCpu->hwaccm.s.vmx.VMCSCache.cResume = cResume;
2190	pVCpu->hwaccm.s.vmx.VMCSCache.u64TimeSwitch = RTTimeNanoTS();
2191	#endif
2192
2193	TMNotifyStartOfExecution(pVM);
2194	rc = pVCpu->hwaccm.s.vmx.pfnStartVM(pVCpu->hwaccm.s.fResumeVM, pCtx, &pVCpu->hwaccm.s.vmx.VMCSCache, pVM, pVCpu);
2195	TMNotifyEndOfExecution(pVM);
2196
2197	AssertMsg(!pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries, ("pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries=%d\n", pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries));
2198
2199	/* In case we execute a goto ResumeExecution later on. */
2200	pVCpu->hwaccm.s.fResumeVM = true;
2201	pVCpu->hwaccm.s.fForceTLBFlush = false;
2202
2203	/*
2204	* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
2205	* IMPORTANT: WE CAN'T DO ANY LOGGING OR OPERATIONS THAT CAN DO A LONGJMP BACK TO RING 3 BEFORE WE'VE SYNCED BACK (MOST OF) THE GUEST STATE
2206	* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
2207	*/
2208	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatInGC, z);
2209	STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatExit1, v);
2210
2211	if (rc != VINF_SUCCESS)
2212	{
2213	VMXR0ReportWorldSwitchError(pVM, pVCpu, rc, pCtx);
2214	goto end;
2215	}
2216
2217	/* Success. Query the guest state and figure out what has happened. */
2218
2219	/* Investigate why there was a VM-exit. */
2220	rc = VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_REASON, &exitReason);
2221	STAM_COUNTER_INC(&pVCpu->hwaccm.s.paStatExitReasonR0[exitReason & MASK_EXITREASON_STAT]);
2222
2223	exitReason &= 0xffff; /* bit 0-15 contain the exit code. */
2224	rc \|= VMXReadCachedVMCS(VMX_VMCS32_RO_VM_INSTR_ERROR, &instrError);
2225	rc \|= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &cbInstr);
2226	rc \|= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &intInfo);
2227	/* might not be valid; depends on VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID. */
2228	rc \|= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERRCODE, &errCode);
2229	rc \|= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INSTR_INFO, &instrInfo);
2230	rc \|= VMXReadCachedVMCS(VMX_VMCS_RO_EXIT_QUALIFICATION, &exitQualification);
2231	AssertRC(rc);
2232
2233	/* Sync back the guest state */
2234	rc = VMXR0SaveGuestState(pVM, pVCpu, pCtx);
2235	AssertRC(rc);
2236
2237	/* Note! NOW IT'S SAFE FOR LOGGING! */
2238	#ifdef LOG_LOGGING
2239	VMMR0LogFlushEnable(pVCpu);
2240	#endif
2241	Log2(("Raw exit reason %08x\n", exitReason));
2242
2243	/* Check if an injected event was interrupted prematurely. */
2244	rc = VMXReadCachedVMCS(VMX_VMCS32_RO_IDT_INFO, &val);
2245	AssertRC(rc);
2246	pVCpu->hwaccm.s.Event.intInfo = VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(val);
2247	if ( VMX_EXIT_INTERRUPTION_INFO_VALID(pVCpu->hwaccm.s.Event.intInfo)
2248	/* Ignore 'int xx' as they'll be restarted anyway. */
2249	&& VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hwaccm.s.Event.intInfo) != VMX_EXIT_INTERRUPTION_INFO_TYPE_SW
2250	/* Ignore software exceptions (such as int3) as they're reoccur when we restart the instruction anyway. */
2251	&& VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hwaccm.s.Event.intInfo) != VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT)
2252	{
2253	pVCpu->hwaccm.s.Event.fPending = true;
2254	/* Error code present? */
2255	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVCpu->hwaccm.s.Event.intInfo))
2256	{
2257	rc = VMXReadCachedVMCS(VMX_VMCS32_RO_IDT_ERRCODE, &val);
2258	AssertRC(rc);
2259	pVCpu->hwaccm.s.Event.errCode = val;
2260	Log(("Pending inject %RX64 at %RGv exit=%08x intInfo=%08x exitQualification=%RGv pending error=%RX64\n", pVCpu->hwaccm.s.Event.intInfo, (RTGCPTR)pCtx->rip, exitReason, intInfo, exitQualification, val));
2261	}
2262	else
2263	{
2264	Log(("Pending inject %RX64 at %RGv exit=%08x intInfo=%08x exitQualification=%RGv\n", pVCpu->hwaccm.s.Event.intInfo, (RTGCPTR)pCtx->rip, exitReason, intInfo, exitQualification));
2265	pVCpu->hwaccm.s.Event.errCode = 0;
2266	}
2267	}
2268	#ifdef VBOX_STRICT
2269	else
2270	if ( VMX_EXIT_INTERRUPTION_INFO_VALID(pVCpu->hwaccm.s.Event.intInfo)
2271	/* Ignore software exceptions (such as int3) as they're reoccur when we restart the instruction anyway. */
2272	&& VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hwaccm.s.Event.intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT)
2273	{
2274	Log(("Ignore pending inject %RX64 at %RGv exit=%08x intInfo=%08x exitQualification=%RGv\n", pVCpu->hwaccm.s.Event.intInfo, (RTGCPTR)pCtx->rip, exitReason, intInfo, exitQualification));
2275	}
2276
2277	if (exitReason == VMX_EXIT_ERR_INVALID_GUEST_STATE)
2278	HWACCMDumpRegs(pVM, pCtx);
2279	#endif
2280
2281	Log2(("E%d: New EIP=%RGv\n", exitReason, (RTGCPTR)pCtx->rip));
2282	Log2(("Exit reason %d, exitQualification %RGv\n", (uint32_t)exitReason, exitQualification));
2283	Log2(("instrInfo=%d instrError=%d instr length=%d\n", (uint32_t)instrInfo, (uint32_t)instrError, (uint32_t)cbInstr));
2284	Log2(("Interruption error code %d\n", (uint32_t)errCode));
2285	Log2(("IntInfo = %08x\n", (uint32_t)intInfo));
2286
2287	if (fSyncTPR)
2288	{
2289	rc = PDMApicSetTPR(pVM, pVM->hwaccm.s.vmx.pAPIC[0x80] >> 4);
2290	AssertRC(rc);
2291	}
2292
2293	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit1, v);
2294	STAM_STATS({ STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatExit2, y); fStatExit2Started = true; });
2295
2296	/* Some cases don't need a complete resync of the guest CPU state; handle them here. */
2297	switch (exitReason)
2298	{
2299	case VMX_EXIT_EXCEPTION: /* 0 Exception or non-maskable interrupt (NMI). */
2300	case VMX_EXIT_EXTERNAL_IRQ: /* 1 External interrupt. */
2301	{
2302	uint32_t vector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(intInfo);
2303
2304	if (!VMX_EXIT_INTERRUPTION_INFO_VALID(intInfo))
2305	{
2306	Assert(exitReason == VMX_EXIT_EXTERNAL_IRQ);
2307	/* External interrupt; leave to allow it to be dispatched again. */
2308	rc = VINF_EM_RAW_INTERRUPT;
2309	break;
2310	}
2311	STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2312	switch (VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo))
2313	{
2314	case VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI: /* Non-maskable interrupt. */
2315	/* External interrupt; leave to allow it to be dispatched again. */
2316	rc = VINF_EM_RAW_INTERRUPT;
2317	break;
2318
2319	case VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT: /* External hardware interrupt. */
2320	AssertFailed(); /* can't come here; fails the first check. */
2321	break;
2322
2323	case VMX_EXIT_INTERRUPTION_INFO_TYPE_DBEXCPT: /* Unknown why we get this type for #DB */
2324	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT: /* Software exception. (#BP or #OF) */
2325	Assert(vector == 1 \|\| vector == 3 \|\| vector == 4);
2326	/* no break */
2327	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT: /* Hardware exception. */
2328	Log2(("Hardware/software interrupt %d\n", vector));
2329	switch (vector)
2330	{
2331	case X86_XCPT_NM:
2332	{
2333	Log(("#NM fault at %RGv error code %x\n", (RTGCPTR)pCtx->rip, errCode));
2334
2335	/** @todo don't intercept #NM exceptions anymore when we've activated the guest FPU state. */
2336	/* If we sync the FPU/XMM state on-demand, then we can continue execution as if nothing has happened. */
2337	rc = CPUMR0LoadGuestFPU(pVM, pVCpu, pCtx);
2338	if (rc == VINF_SUCCESS)
2339	{
2340	Assert(CPUMIsGuestFPUStateActive(pVCpu));
2341
2342	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitShadowNM);
2343
2344	/* Continue execution. */
2345	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR0;
2346
2347	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2348	goto ResumeExecution;
2349	}
2350
2351	Log(("Forward #NM fault to the guest\n"));
2352	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestNM);
2353	rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, 0);
2354	AssertRC(rc);
2355	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2356	goto ResumeExecution;
2357	}
2358
2359	case X86_XCPT_PF: /* Page fault */
2360	{
2361	#ifdef DEBUG
2362	if (pVM->hwaccm.s.fNestedPaging)
2363	{ /* A genuine pagefault.
2364	* Forward the trap to the guest by injecting the exception and resuming execution.
2365	*/
2366	Log(("Guest page fault at %RGv cr2=%RGv error code %x rsp=%RGv\n", (RTGCPTR)pCtx->rip, exitQualification, errCode, (RTGCPTR)pCtx->rsp));
2367
2368	Assert(CPUMIsGuestInPagedProtectedModeEx(pCtx));
2369
2370	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestPF);
2371
2372	/* Now we must update CR2. */
2373	pCtx->cr2 = exitQualification;
2374	rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
2375	AssertRC(rc);
2376
2377	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2378	goto ResumeExecution;
2379	}
2380	#endif
2381	Assert(!pVM->hwaccm.s.fNestedPaging);
2382
2383	Log2(("Page fault at %RGv error code %x\n", exitQualification, errCode));
2384	/* Exit qualification contains the linear address of the page fault. */
2385	TRPMAssertTrap(pVM, X86_XCPT_PF, TRPM_TRAP);
2386	TRPMSetErrorCode(pVM, errCode);
2387	TRPMSetFaultAddress(pVM, exitQualification);
2388
2389	/* Forward it to our trap handler first, in case our shadow pages are out of sync. */
2390	rc = PGMTrap0eHandler(pVM, errCode, CPUMCTX2CORE(pCtx), (RTGCPTR)exitQualification);
2391	Log2(("PGMTrap0eHandler %RGv returned %Rrc\n", (RTGCPTR)pCtx->rip, rc));
2392	if (rc == VINF_SUCCESS)
2393	{ /* We've successfully synced our shadow pages, so let's just continue execution. */
2394	Log2(("Shadow page fault at %RGv cr2=%RGv error code %x\n", (RTGCPTR)pCtx->rip, exitQualification ,errCode));
2395	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitShadowPF);
2396
2397	TRPMResetTrap(pVM);
2398
2399	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2400	goto ResumeExecution;
2401	}
2402	else
2403	if (rc == VINF_EM_RAW_GUEST_TRAP)
2404	{ /* A genuine pagefault.
2405	* Forward the trap to the guest by injecting the exception and resuming execution.
2406	*/
2407	Log2(("Forward page fault to the guest\n"));
2408
2409	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestPF);
2410	/* The error code might have been changed. */
2411	errCode = TRPMGetErrorCode(pVM);
2412
2413	TRPMResetTrap(pVM);
2414
2415	/* Now we must update CR2. */
2416	pCtx->cr2 = exitQualification;
2417	rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
2418	AssertRC(rc);
2419
2420	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2421	goto ResumeExecution;
2422	}
2423	#ifdef VBOX_STRICT
2424	if (rc != VINF_EM_RAW_EMULATE_INSTR && rc != VINF_EM_RAW_EMULATE_IO_BLOCK)
2425	Log2(("PGMTrap0eHandler failed with %d\n", rc));
2426	#endif
2427	/* Need to go back to the recompiler to emulate the instruction. */
2428	TRPMResetTrap(pVM);
2429	break;
2430	}
2431
2432	case X86_XCPT_MF: /* Floating point exception. */
2433	{
2434	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestMF);
2435	if (!(pCtx->cr0 & X86_CR0_NE))
2436	{
2437	/* old style FPU error reporting needs some extra work. */
2438	/** @todo don't fall back to the recompiler, but do it manually. */
2439	rc = VINF_EM_RAW_EMULATE_INSTR;
2440	break;
2441	}
2442	Log(("Trap %x at %04X:%RGv\n", vector, pCtx->cs, (RTGCPTR)pCtx->rip));
2443	rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
2444	AssertRC(rc);
2445
2446	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2447	goto ResumeExecution;
2448	}
2449
2450	case X86_XCPT_DB: /* Debug exception. */
2451	{
2452	uint64_t uDR6;
2453
2454	/* DR6, DR7.GD and IA32_DEBUGCTL.LBR are not updated yet.
2455	*
2456	* Exit qualification bits:
2457	* 3:0 B0-B3 which breakpoint condition was met
2458	* 12:4 Reserved (0)
2459	* 13 BD - debug register access detected
2460	* 14 BS - single step execution or branch taken
2461	* 63:15 Reserved (0)
2462	*/
2463	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestDB);
2464
2465	/* Note that we don't support guest and host-initiated debugging at the same time. */
2466	Assert(DBGFIsStepping(pVM) \|\| CPUMIsGuestInRealModeEx(pCtx));
2467
2468	uDR6 = X86_DR6_INIT_VAL;
2469	uDR6 \|= (exitQualification & (X86_DR6_B0\|X86_DR6_B1\|X86_DR6_B2\|X86_DR6_B3\|X86_DR6_BD\|X86_DR6_BS));
2470	rc = DBGFR0Trap01Handler(pVM, CPUMCTX2CORE(pCtx), uDR6);
2471	if (rc == VINF_EM_RAW_GUEST_TRAP)
2472	{
2473	/** @todo this isn't working, but we'll never get here normally. */
2474
2475	/* Update DR6 here. */
2476	pCtx->dr[6] = uDR6;
2477
2478	/* X86_DR7_GD will be cleared if drx accesses should be trapped inside the guest. */
2479	pCtx->dr[7] &= ~X86_DR7_GD;
2480
2481	/* Paranoia. */
2482	pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
2483	pCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* must be zero */
2484	pCtx->dr[7] \|= 0x400; /* must be one */
2485
2486	/* Resync DR7 */
2487	rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_DR7, pCtx->dr[7]);
2488	AssertRC(rc);
2489
2490	Log(("Trap %x (debug) at %RGv exit qualification %RX64\n", vector, (RTGCPTR)pCtx->rip, exitQualification));
2491	rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
2492	AssertRC(rc);
2493
2494	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2495	goto ResumeExecution;
2496	}
2497	/* Return to ring 3 to deal with the debug exit code. */
2498	break;
2499	}
2500
2501	#ifdef DEBUG /* till after branching, enable by default after that. */
2502	case X86_XCPT_BP: /* Breakpoint. */
2503	{
2504	rc = DBGFR0Trap03Handler(pVM, CPUMCTX2CORE(pCtx));
2505	if (rc == VINF_EM_RAW_GUEST_TRAP)
2506	{
2507	Log(("Guest #BP at %04x:%RGv\n", pCtx->cs, pCtx->rip));
2508	rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
2509	AssertRC(rc);
2510	goto ResumeExecution;
2511	}
2512	if (rc == VINF_SUCCESS)
2513	goto ResumeExecution;
2514	Log(("Debugger BP at %04x:%RGv (rc=%Rrc)\n", pCtx->cs, pCtx->rip, rc));
2515	break;
2516	}
2517	#endif
2518
2519	case X86_XCPT_GP: /* General protection failure exception.*/
2520	{
2521	uint32_t cbOp;
2522	uint32_t cbSize;
2523	DISCPUSTATE Cpu;
2524
2525	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestGP);
2526	#ifdef VBOX_STRICT
2527	if (!CPUMIsGuestInRealModeEx(pCtx))
2528	{
2529	Log(("Trap %x at %04X:%RGv errorCode=%x\n", vector, pCtx->cs, (RTGCPTR)pCtx->rip, errCode));
2530	rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
2531	AssertRC(rc);
2532	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2533	goto ResumeExecution;
2534	}
2535	#endif
2536	Assert(CPUMIsGuestInRealModeEx(pCtx));
2537
2538	LogFlow(("Real mode X86_XCPT_GP instruction emulation at %RGv\n", (RTGCPTR)pCtx->rip));
2539
2540	rc = EMInterpretDisasOne(pVM, CPUMCTX2CORE(pCtx), &Cpu, &cbOp);
2541	if (RT_SUCCESS(rc))
2542	{
2543	bool fUpdateRIP = true;
2544
2545	Assert(cbOp == Cpu.opsize);
2546	switch (Cpu.pCurInstr->opcode)
2547	{
2548	case OP_CLI:
2549	pCtx->eflags.Bits.u1IF = 0;
2550	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCli);
2551	break;
2552
2553	case OP_STI:
2554	pCtx->eflags.Bits.u1IF = 1;
2555	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitSti);
2556	break;
2557
2558	case OP_HLT:
2559	fUpdateRIP = false;
2560	rc = VINF_EM_HALT;
2561	pCtx->rip += Cpu.opsize;
2562	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitHlt);
2563	break;
2564
2565	case OP_POPF:
2566	{
2567	RTGCPTR GCPtrStack;
2568	uint32_t cbParm;
2569	uint32_t uMask;
2570	X86EFLAGS eflags;
2571
2572	if (Cpu.prefix & PREFIX_OPSIZE)
2573	{
2574	cbParm = 4;
2575	uMask = 0xffffffff;
2576	}
2577	else
2578	{
2579	cbParm = 2;
2580	uMask = 0xffff;
2581	}
2582
2583	rc = SELMToFlatEx(pVM, DIS_SELREG_SS, CPUMCTX2CORE(pCtx), pCtx->esp & uMask, 0, &GCPtrStack);
2584	if (RT_FAILURE(rc))
2585	{
2586	rc = VERR_EM_INTERPRETER;
2587	break;
2588	}
2589	eflags.u = 0;
2590	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &eflags.u, cbParm);
2591	if (RT_FAILURE(rc))
2592	{
2593	rc = VERR_EM_INTERPRETER;
2594	break;
2595	}
2596	LogFlow(("POPF %x -> %RGv mask=%x\n", eflags.u, pCtx->rsp, uMask));
2597	pCtx->eflags.u = (pCtx->eflags.u & ~(X86_EFL_POPF_BITS & uMask)) \| (eflags.u & X86_EFL_POPF_BITS & uMask);
2598	/* RF cleared when popped in real mode; see pushf description in AMD manual. */
2599	pCtx->eflags.Bits.u1RF = 0;
2600	pCtx->esp += cbParm;
2601	pCtx->esp &= uMask;
2602
2603	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitPopf);
2604	break;
2605	}
2606
2607	case OP_PUSHF:
2608	{
2609	RTGCPTR GCPtrStack;
2610	uint32_t cbParm;
2611	uint32_t uMask;
2612	X86EFLAGS eflags;
2613
2614	if (Cpu.prefix & PREFIX_OPSIZE)
2615	{
2616	cbParm = 4;
2617	uMask = 0xffffffff;
2618	}
2619	else
2620	{
2621	cbParm = 2;
2622	uMask = 0xffff;
2623	}
2624
2625	rc = SELMToFlatEx(pVM, DIS_SELREG_SS, CPUMCTX2CORE(pCtx), (pCtx->esp - cbParm) & uMask, 0, &GCPtrStack);
2626	if (RT_FAILURE(rc))
2627	{
2628	rc = VERR_EM_INTERPRETER;
2629	break;
2630	}
2631	eflags = pCtx->eflags;
2632	/* RF & VM cleared when pushed in real mode; see pushf description in AMD manual. */
2633	eflags.Bits.u1RF = 0;
2634	eflags.Bits.u1VM = 0;
2635
2636	rc = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &eflags.u, cbParm);
2637	if (RT_FAILURE(rc))
2638	{
2639	rc = VERR_EM_INTERPRETER;
2640	break;
2641	}
2642	LogFlow(("PUSHF %x -> %RGv\n", eflags.u, GCPtrStack));
2643	pCtx->esp -= cbParm;
2644	pCtx->esp &= uMask;
2645	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitPushf);
2646	break;
2647	}
2648
2649	case OP_IRET:
2650	{
2651	RTGCPTR GCPtrStack;
2652	uint32_t uMask = 0xffff;
2653	uint16_t aIretFrame[3];
2654
2655	if (Cpu.prefix & (PREFIX_OPSIZE \| PREFIX_ADDRSIZE))
2656	{
2657	rc = VERR_EM_INTERPRETER;
2658	break;
2659	}
2660
2661	rc = SELMToFlatEx(pVM, DIS_SELREG_SS, CPUMCTX2CORE(pCtx), pCtx->esp & uMask, 0, &GCPtrStack);
2662	if (RT_FAILURE(rc))
2663	{
2664	rc = VERR_EM_INTERPRETER;
2665	break;
2666	}
2667	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
2668	if (RT_FAILURE(rc))
2669	{
2670	rc = VERR_EM_INTERPRETER;
2671	break;
2672	}
2673	pCtx->ip = aIretFrame[0];
2674	pCtx->cs = aIretFrame[1];
2675	pCtx->csHid.u64Base = pCtx->cs << 4;
2676	pCtx->eflags.u = (pCtx->eflags.u & ~(X86_EFL_POPF_BITS & uMask)) \| (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
2677	pCtx->sp += sizeof(aIretFrame);
2678
2679	LogFlow(("iret to %04x:%x\n", pCtx->cs, pCtx->ip));
2680	fUpdateRIP = false;
2681	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIret);
2682	break;
2683	}
2684
2685	case OP_INT:
2686	{
2687	RTGCUINTPTR intInfo;
2688
2689	LogFlow(("Realmode: INT %x\n", Cpu.param1.parval & 0xff));
2690	intInfo = Cpu.param1.parval & 0xff;
2691	intInfo \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
2692	intInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
2693
2694	rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, intInfo, cbOp, 0);
2695	AssertRC(rc);
2696	fUpdateRIP = false;
2697	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInt);
2698	break;
2699	}
2700
2701	case OP_INTO:
2702	{
2703	if (pCtx->eflags.Bits.u1OF)
2704	{
2705	RTGCUINTPTR intInfo;
2706
2707	LogFlow(("Realmode: INTO\n"));
2708	intInfo = X86_XCPT_OF;
2709	intInfo \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
2710	intInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
2711
2712	rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, intInfo, cbOp, 0);
2713	AssertRC(rc);
2714	fUpdateRIP = false;
2715	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInt);
2716	}
2717	break;
2718	}
2719
2720	case OP_INT3:
2721	{
2722	RTGCUINTPTR intInfo;
2723
2724	LogFlow(("Realmode: INT 3\n"));
2725	intInfo = 3;
2726	intInfo \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
2727	intInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
2728
2729	rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, intInfo, cbOp, 0);
2730	AssertRC(rc);
2731	fUpdateRIP = false;
2732	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInt);
2733	break;
2734	}
2735
2736	default:
2737	rc = EMInterpretInstructionCPU(pVM, &Cpu, CPUMCTX2CORE(pCtx), 0, &cbSize);
2738	break;
2739	}
2740
2741	if (rc == VINF_SUCCESS)
2742	{
2743	if (fUpdateRIP)
2744	pCtx->rip += cbOp; /* Move on to the next instruction. */
2745
2746	/* lidt, lgdt can end up here. In the future crx changes as well. Just reload the whole context to be done with it. */
2747	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_ALL;
2748
2749	/* Only resume if successful. */
2750	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2751	goto ResumeExecution;
2752	}
2753	}
2754	else
2755	rc = VERR_EM_INTERPRETER;
2756
2757	AssertMsg(rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_EM_HALT, ("Unexpected rc=%Rrc\n", rc));
2758	break;
2759	}
2760
2761	#ifdef VBOX_STRICT
2762	case X86_XCPT_DE: /* Divide error. */
2763	case X86_XCPT_UD: /* Unknown opcode exception. */
2764	case X86_XCPT_SS: /* Stack segment exception. */
2765	case X86_XCPT_NP: /* Segment not present exception. */
2766	{
2767	switch(vector)
2768	{
2769	case X86_XCPT_DE:
2770	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestDE);
2771	break;
2772	case X86_XCPT_UD:
2773	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestUD);
2774	break;
2775	case X86_XCPT_SS:
2776	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestSS);
2777	break;
2778	case X86_XCPT_NP:
2779	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestNP);
2780	break;
2781	}
2782
2783	Log(("Trap %x at %04X:%RGv\n", vector, pCtx->cs, (RTGCPTR)pCtx->rip));
2784	rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
2785	AssertRC(rc);
2786
2787	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2788	goto ResumeExecution;
2789	}
2790	#endif
2791	default:
2792	#ifdef HWACCM_VMX_EMULATE_REALMODE
2793	if (CPUMIsGuestInRealModeEx(pCtx))
2794	{
2795	Log(("Real Mode Trap %x at %04x:%04X error code %x\n", vector, pCtx->cs, pCtx->eip, errCode));
2796	rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
2797	AssertRC(rc);
2798
2799	/* Go back to ring 3 in case of a triple fault. */
2800	if ( vector == X86_XCPT_DF
2801	&& rc == VINF_EM_RESET)
2802	break;
2803
2804	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2805	goto ResumeExecution;
2806	}
2807	#endif
2808	AssertMsgFailed(("Unexpected vm-exit caused by exception %x\n", vector));
2809	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
2810	break;
2811	} /* switch (vector) */
2812
2813	break;
2814
2815	default:
2816	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_CODE;
2817	AssertMsgFailed(("Unexpected interuption code %x\n", intInfo));
2818	break;
2819	}
2820
2821	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2822	break;
2823	}
2824
2825	case VMX_EXIT_EPT_VIOLATION: /* 48 EPT violation. An attempt to access memory with a guest-physical address was disallowed by the configuration of the EPT paging structures. */
2826	{
2827	RTGCPHYS GCPhys;
2828
2829	Assert(pVM->hwaccm.s.fNestedPaging);
2830
2831	rc = VMXReadVMCS64(VMX_VMCS_EXIT_PHYS_ADDR_FULL, &GCPhys);
2832	AssertRC(rc);
2833	Assert(((exitQualification >> 7) & 3) != 2);
2834
2835	/* Determine the kind of violation. */
2836	errCode = 0;
2837	if (exitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
2838	errCode \|= X86_TRAP_PF_ID;
2839
2840	if (exitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
2841	errCode \|= X86_TRAP_PF_RW;
2842
2843	/* If the page is present, then it's a page level protection fault. */
2844	if (exitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
2845	errCode \|= X86_TRAP_PF_P;
2846
2847	Log(("EPT Page fault %x at %RGp error code %x\n", (uint32_t)exitQualification, GCPhys, errCode));
2848
2849	/* GCPhys contains the guest physical address of the page fault. */
2850	TRPMAssertTrap(pVM, X86_XCPT_PF, TRPM_TRAP);
2851	TRPMSetErrorCode(pVM, errCode);
2852	TRPMSetFaultAddress(pVM, GCPhys);
2853
2854	/* Handle the pagefault trap for the nested shadow table. */
2855	rc = PGMR0Trap0eHandlerNestedPaging(pVM, PGMMODE_EPT, errCode, CPUMCTX2CORE(pCtx), GCPhys);
2856	Log2(("PGMR0Trap0eHandlerNestedPaging %RGv returned %Rrc\n", (RTGCPTR)pCtx->rip, rc));
2857	if (rc == VINF_SUCCESS)
2858	{ /* We've successfully synced our shadow pages, so let's just continue execution. */
2859	Log2(("Shadow page fault at %RGv cr2=%RGp error code %x\n", (RTGCPTR)pCtx->rip, exitQualification , errCode));
2860	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitReasonNPF);
2861
2862	TRPMResetTrap(pVM);
2863
2864	goto ResumeExecution;
2865	}
2866
2867	#ifdef VBOX_STRICT
2868	if (rc != VINF_EM_RAW_EMULATE_INSTR)
2869	LogFlow(("PGMTrap0eHandlerNestedPaging failed with %d\n", rc));
2870	#endif
2871	/* Need to go back to the recompiler to emulate the instruction. */
2872	TRPMResetTrap(pVM);
2873	break;
2874	}
2875
2876	case VMX_EXIT_EPT_MISCONFIG:
2877	{
2878	RTGCPHYS GCPhys;
2879
2880	Assert(pVM->hwaccm.s.fNestedPaging);
2881
2882	rc = VMXReadVMCS64(VMX_VMCS_EXIT_PHYS_ADDR_FULL, &GCPhys);
2883	AssertRC(rc);
2884
2885	Log(("VMX_EXIT_EPT_MISCONFIG for %VGp\n", GCPhys));
2886	break;
2887	}
2888
2889	case VMX_EXIT_IRQ_WINDOW: /* 7 Interrupt window. */
2890	/* Clear VM-exit on IF=1 change. */
2891	LogFlow(("VMX_EXIT_IRQ_WINDOW %RGv pending=%d IF=%d\n", (RTGCPTR)pCtx->rip, VM_FF_ISPENDING(pVM, (VM_FF_INTERRUPT_APIC\|VM_FF_INTERRUPT_PIC)), pCtx->eflags.Bits.u1IF));
2892	pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_IRQ_WINDOW_EXIT;
2893	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
2894	AssertRC(rc);
2895	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIrqWindow);
2896	goto ResumeExecution; /* we check for pending guest interrupts there */
2897
2898	case VMX_EXIT_WBINVD: /* 54 Guest software attempted to execute WBINVD. (conditional) */
2899	case VMX_EXIT_INVD: /* 13 Guest software attempted to execute INVD. (unconditional) */
2900	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInvd);
2901	/* Skip instruction and continue directly. */
2902	pCtx->rip += cbInstr;
2903	/* Continue execution.*/
2904	goto ResumeExecution;
2905
2906	case VMX_EXIT_CPUID: /* 10 Guest software attempted to execute CPUID. */
2907	{
2908	Log2(("VMX: Cpuid %x\n", pCtx->eax));
2909	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCpuid);
2910	rc = EMInterpretCpuId(pVM, CPUMCTX2CORE(pCtx));
2911	if (rc == VINF_SUCCESS)
2912	{
2913	/* Update EIP and continue execution. */
2914	Assert(cbInstr == 2);
2915	pCtx->rip += cbInstr;
2916	goto ResumeExecution;
2917	}
2918	AssertMsgFailed(("EMU: cpuid failed with %Rrc\n", rc));
2919	rc = VINF_EM_RAW_EMULATE_INSTR;
2920	break;
2921	}
2922
2923	case VMX_EXIT_RDTSC: /* 16 Guest software attempted to execute RDTSC. */
2924	{
2925	Log2(("VMX: Rdtsc\n"));
2926	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitRdtsc);
2927	rc = EMInterpretRdtsc(pVM, CPUMCTX2CORE(pCtx));
2928	if (rc == VINF_SUCCESS)
2929	{
2930	/* Update EIP and continue execution. */
2931	Assert(cbInstr == 2);
2932	pCtx->rip += cbInstr;
2933	goto ResumeExecution;
2934	}
2935	AssertMsgFailed(("EMU: rdtsc failed with %Rrc\n", rc));
2936	rc = VINF_EM_RAW_EMULATE_INSTR;
2937	break;
2938	}
2939
2940	case VMX_EXIT_INVPG: /* 14 Guest software attempted to execute INVPG. */
2941	{
2942	Log2(("VMX: invlpg\n"));
2943	Assert(!pVM->hwaccm.s.fNestedPaging);
2944
2945	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInvpg);
2946	rc = EMInterpretInvlpg(pVM, CPUMCTX2CORE(pCtx), exitQualification);
2947	if (rc == VINF_SUCCESS)
2948	{
2949	/* Update EIP and continue execution. */
2950	pCtx->rip += cbInstr;
2951	goto ResumeExecution;
2952	}
2953	AssertMsg(rc == VERR_EM_INTERPRETER, ("EMU: invlpg %RGv failed with %Rrc\n", exitQualification, rc));
2954	break;
2955	}
2956
2957	case VMX_EXIT_RDMSR: /* 31 RDMSR. Guest software attempted to execute RDMSR. */
2958	case VMX_EXIT_WRMSR: /* 32 WRMSR. Guest software attempted to execute WRMSR. */
2959	{
2960	uint32_t cbSize;
2961
2962	/* Note: the intel manual claims there's a REX version of RDMSR that's slightly different, so we play safe by completely disassembling the instruction. */
2963	Log2(("VMX: %s\n", (exitReason == VMX_EXIT_RDMSR) ? "rdmsr" : "wrmsr"));
2964	rc = EMInterpretInstruction(pVM, CPUMCTX2CORE(pCtx), 0, &cbSize);
2965	if (rc == VINF_SUCCESS)
2966	{
2967	/* EIP has been updated already. */
2968
2969	/* Only resume if successful. */
2970	goto ResumeExecution;
2971	}
2972	AssertMsg(rc == VERR_EM_INTERPRETER, ("EMU: %s failed with %Rrc\n", (exitReason == VMX_EXIT_RDMSR) ? "rdmsr" : "wrmsr", rc));
2973	break;
2974	}
2975
2976	case VMX_EXIT_CRX_MOVE: /* 28 Control-register accesses. */
2977	{
2978	STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatExit2Sub2, y2);
2979
2980	switch (VMX_EXIT_QUALIFICATION_CRX_ACCESS(exitQualification))
2981	{
2982	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE:
2983	Log2(("VMX: %RGv mov cr%d, x\n", (RTGCPTR)pCtx->rip, VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification)));
2984	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification)]);
2985	rc = EMInterpretCRxWrite(pVM, CPUMCTX2CORE(pCtx),
2986	VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification),
2987	VMX_EXIT_QUALIFICATION_CRX_GENREG(exitQualification));
2988
2989	switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification))
2990	{
2991	case 0:
2992	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR0 \| HWACCM_CHANGED_GUEST_CR3;
2993	break;
2994	case 2:
2995	break;
2996	case 3:
2997	Assert(!pVM->hwaccm.s.fNestedPaging \|\| !CPUMIsGuestInPagedProtectedModeEx(pCtx));
2998	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR3;
2999	break;
3000	case 4:
3001	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR4;
3002	break;
3003	case 8:
3004	/* CR8 contains the APIC TPR */
3005	Assert(!(pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
3006	break;
3007
3008	default:
3009	AssertFailed();
3010	break;
3011	}
3012	/* Check if a sync operation is pending. */
3013	if ( rc == VINF_SUCCESS /* don't bother if we are going to ring 3 anyway */
3014	&& VM_FF_ISPENDING(pVM, VM_FF_PGM_SYNC_CR3 \| VM_FF_PGM_SYNC_CR3_NON_GLOBAL))
3015	{
3016	rc = PGMSyncCR3(pVM, CPUMGetGuestCR0(pVM), CPUMGetGuestCR3(pVM), CPUMGetGuestCR4(pVM), VM_FF_ISSET(pVM, VM_FF_PGM_SYNC_CR3));
3017	AssertRC(rc);
3018	}
3019	break;
3020
3021	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ:
3022	Log2(("VMX: mov x, crx\n"));
3023	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification)]);
3024
3025	Assert(!pVM->hwaccm.s.fNestedPaging \|\| !CPUMIsGuestInPagedProtectedModeEx(pCtx) \|\| VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification) != USE_REG_CR3);
3026
3027	/* CR8 reads only cause an exit when the TPR shadow feature isn't present. */
3028	Assert(VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification) != 8 \|\| !(pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
3029
3030	rc = EMInterpretCRxRead(pVM, CPUMCTX2CORE(pCtx),
3031	VMX_EXIT_QUALIFICATION_CRX_GENREG(exitQualification),
3032	VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification));
3033	break;
3034
3035	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS:
3036	Log2(("VMX: clts\n"));
3037	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCLTS);
3038	rc = EMInterpretCLTS(pVM);
3039	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR0;
3040	break;
3041
3042	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW:
3043	Log2(("VMX: lmsw %x\n", VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(exitQualification)));
3044	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitLMSW);
3045	rc = EMInterpretLMSW(pVM, CPUMCTX2CORE(pCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(exitQualification));
3046	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR0;
3047	break;
3048	}
3049
3050	/* Update EIP if no error occurred. */
3051	if (RT_SUCCESS(rc))
3052	pCtx->rip += cbInstr;
3053
3054	if (rc == VINF_SUCCESS)
3055	{
3056	/* Only resume if successful. */
3057	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub2, y2);
3058	goto ResumeExecution;
3059	}
3060	Assert(rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3);
3061	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub2, y2);
3062	break;
3063	}
3064
3065	case VMX_EXIT_DRX_MOVE: /* 29 Debug-register accesses. */
3066	{
3067	if (!DBGFIsStepping(pVM))
3068	{
3069	/* Disable drx move intercepts. */
3070	pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
3071	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
3072	AssertRC(rc);
3073
3074	/* Save the host and load the guest debug state. */
3075	rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
3076	AssertRC(rc);
3077
3078	#ifdef VBOX_WITH_STATISTICS
3079	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatDRxContextSwitch);
3080	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
3081	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxWrite);
3082	else
3083	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxRead);
3084	#endif
3085
3086	goto ResumeExecution;
3087	}
3088
3089	/** @todo clear VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT after the first time and restore drx registers afterwards */
3090	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
3091	{
3092	Log2(("VMX: mov drx%d, genreg%d\n", VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification), VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification)));
3093	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxWrite);
3094	rc = EMInterpretDRxWrite(pVM, CPUMCTX2CORE(pCtx),
3095	VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification),
3096	VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification));
3097	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_DEBUG;
3098	Log2(("DR7=%08x\n", pCtx->dr[7]));
3099	}
3100	else
3101	{
3102	Log2(("VMX: mov x, drx\n"));
3103	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxRead);
3104	rc = EMInterpretDRxRead(pVM, CPUMCTX2CORE(pCtx),
3105	VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification),
3106	VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification));
3107	}
3108	/* Update EIP if no error occurred. */
3109	if (RT_SUCCESS(rc))
3110	pCtx->rip += cbInstr;
3111
3112	if (rc == VINF_SUCCESS)
3113	{
3114	/* Only resume if successful. */
3115	goto ResumeExecution;
3116	}
3117	Assert(rc == VERR_EM_INTERPRETER);
3118	break;
3119	}
3120
3121	/* Note: We'll get a #GP if the IO instruction isn't allowed (IOPL or TSS bitmap); no need to double check. */
3122	case VMX_EXIT_PORT_IO: /* 30 I/O instruction. */
3123	{
3124	STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3125	uint32_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(exitQualification);
3126	uint32_t uPort;
3127	bool fIOWrite = (VMX_EXIT_QUALIFICATION_IO_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
3128
3129	/** @todo necessary to make the distinction? */
3130	if (VMX_EXIT_QUALIFICATION_IO_ENCODING(exitQualification) == VMX_EXIT_QUALIFICATION_IO_ENCODING_DX)
3131	{
3132	uPort = pCtx->edx & 0xffff;
3133	}
3134	else
3135	uPort = VMX_EXIT_QUALIFICATION_IO_PORT(exitQualification); /* Immediate encoding. */
3136
3137	/* paranoia */
3138	if (RT_UNLIKELY(uIOWidth == 2 \|\| uIOWidth >= 4))
3139	{
3140	rc = fIOWrite ? VINF_IOM_HC_IOPORT_WRITE : VINF_IOM_HC_IOPORT_READ;
3141	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3142	break;
3143	}
3144
3145	uint32_t cbSize = g_aIOSize[uIOWidth];
3146
3147	if (VMX_EXIT_QUALIFICATION_IO_STRING(exitQualification))
3148	{
3149	/* ins/outs */
3150	DISCPUSTATE Cpu;
3151
3152	/* Disassemble manually to deal with segment prefixes. */
3153	/** @todo VMX_VMCS_EXIT_GUEST_LINEAR_ADDR contains the flat pointer operand of the instruction. */
3154	/** @todo VMX_VMCS32_RO_EXIT_INSTR_INFO also contains segment prefix info. */
3155	rc = EMInterpretDisasOne(pVM, CPUMCTX2CORE(pCtx), &Cpu, NULL);
3156	if (rc == VINF_SUCCESS)
3157	{
3158	if (fIOWrite)
3159	{
3160	Log2(("IOMInterpretOUTSEx %RGv %x size=%d\n", (RTGCPTR)pCtx->rip, uPort, cbSize));
3161	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIOStringWrite);
3162	rc = IOMInterpretOUTSEx(pVM, CPUMCTX2CORE(pCtx), uPort, Cpu.prefix, cbSize);
3163	}
3164	else
3165	{
3166	Log2(("IOMInterpretINSEx %RGv %x size=%d\n", (RTGCPTR)pCtx->rip, uPort, cbSize));
3167	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIOStringRead);
3168	rc = IOMInterpretINSEx(pVM, CPUMCTX2CORE(pCtx), uPort, Cpu.prefix, cbSize);
3169	}
3170	}
3171	else
3172	rc = VINF_EM_RAW_EMULATE_INSTR;
3173	}
3174	else
3175	{
3176	/* normal in/out */
3177	uint32_t uAndVal = g_aIOOpAnd[uIOWidth];
3178
3179	Assert(!VMX_EXIT_QUALIFICATION_IO_REP(exitQualification));
3180
3181	if (fIOWrite)
3182	{
3183	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIOWrite);
3184	rc = IOMIOPortWrite(pVM, uPort, pCtx->eax & uAndVal, cbSize);
3185	}
3186	else
3187	{
3188	uint32_t u32Val = 0;
3189
3190	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIORead);
3191	rc = IOMIOPortRead(pVM, uPort, &u32Val, cbSize);
3192	if (IOM_SUCCESS(rc))
3193	{
3194	/* Write back to the EAX register. */
3195	pCtx->eax = (pCtx->eax & ~uAndVal) \| (u32Val & uAndVal);
3196	}
3197	}
3198	}
3199	/*
3200	* Handled the I/O return codes.
3201	* (The unhandled cases end up with rc == VINF_EM_RAW_EMULATE_INSTR.)
3202	*/
3203	if (IOM_SUCCESS(rc))
3204	{
3205	/* Update EIP and continue execution. */
3206	pCtx->rip += cbInstr;
3207	if (RT_LIKELY(rc == VINF_SUCCESS))
3208	{
3209	/* If any IO breakpoints are armed, then we should check if a debug trap needs to be generated. */
3210	if (pCtx->dr[7] & X86_DR7_ENABLED_MASK)
3211	{
3212	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatDRxIOCheck);
3213	for (unsigned i=0;i<4;i++)
3214	{
3215	unsigned uBPLen = g_aIOSize[X86_DR7_GET_LEN(pCtx->dr[7], i)];
3216
3217	if ( (uPort >= pCtx->dr[i] && uPort < pCtx->dr[i] + uBPLen)
3218	&& (pCtx->dr[7] & (X86_DR7_L(i) \| X86_DR7_G(i)))
3219	&& (pCtx->dr[7] & X86_DR7_RW(i, X86_DR7_RW_IO)) == X86_DR7_RW(i, X86_DR7_RW_IO))
3220	{
3221	uint64_t uDR6;
3222
3223	Assert(CPUMIsGuestDebugStateActive(pVM));
3224
3225	uDR6 = ASMGetDR6();
3226
3227	/* Clear all breakpoint status flags and set the one we just hit. */
3228	uDR6 &= ~(X86_DR6_B0\|X86_DR6_B1\|X86_DR6_B2\|X86_DR6_B3);
3229	uDR6 \|= (uint64_t)RT_BIT(i);
3230
3231	/* Note: AMD64 Architecture Programmer's Manual 13.1:
3232	* Bits 15:13 of the DR6 register is never cleared by the processor and must be cleared by software after
3233	* the contents have been read.
3234	*/
3235	ASMSetDR6(uDR6);
3236
3237	/* X86_DR7_GD will be cleared if drx accesses should be trapped inside the guest. */
3238	pCtx->dr[7] &= ~X86_DR7_GD;
3239
3240	/* Paranoia. */
3241	pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
3242	pCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* must be zero */
3243	pCtx->dr[7] \|= 0x400; /* must be one */
3244
3245	/* Resync DR7 */
3246	rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_DR7, pCtx->dr[7]);
3247	AssertRC(rc);
3248
3249	/* Construct inject info. */
3250	intInfo = X86_XCPT_DB;
3251	intInfo \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
3252	intInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
3253
3254	Log(("Inject IO debug trap at %RGv\n", (RTGCPTR)pCtx->rip));
3255	rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), 0, 0);
3256	AssertRC(rc);
3257
3258	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3259	goto ResumeExecution;
3260	}
3261	}
3262	}
3263
3264	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3265	goto ResumeExecution;
3266	}
3267	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3268	break;
3269	}
3270
3271	#ifdef VBOX_STRICT
3272	if (rc == VINF_IOM_HC_IOPORT_READ)
3273	Assert(!fIOWrite);
3274	else if (rc == VINF_IOM_HC_IOPORT_WRITE)
3275	Assert(fIOWrite);
3276	else
3277	AssertMsg(RT_FAILURE(rc) \|\| rc == VINF_EM_RAW_EMULATE_INSTR \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", rc));
3278	#endif
3279	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3280	break;
3281	}
3282
3283	case VMX_EXIT_TPR: /* 43 TPR below threshold. Guest software executed MOV to CR8. */
3284	LogFlow(("VMX_EXIT_TPR\n"));
3285	/* RIP is already set to the next instruction and the TPR has been synced back. Just resume. */
3286	goto ResumeExecution;
3287
3288	case VMX_EXIT_PREEMPTION_TIMER: /* 52 VMX-preemption timer expired. The preemption timer counted down to zero. */
3289	goto ResumeExecution;
3290
3291	default:
3292	/* The rest is handled after syncing the entire CPU state. */
3293	break;
3294	}
3295
3296	/* Note: the guest state isn't entirely synced back at this stage. */
3297
3298	/* Investigate why there was a VM-exit. (part 2) */
3299	switch (exitReason)
3300	{
3301	case VMX_EXIT_EXCEPTION: /* 0 Exception or non-maskable interrupt (NMI). */
3302	case VMX_EXIT_EXTERNAL_IRQ: /* 1 External interrupt. */
3303	case VMX_EXIT_EPT_VIOLATION:
3304	case VMX_EXIT_PREEMPTION_TIMER: /* 52 VMX-preemption timer expired. The preemption timer counted down to zero. */
3305	/* Already handled above. */
3306	break;
3307
3308	case VMX_EXIT_TRIPLE_FAULT: /* 2 Triple fault. */
3309	rc = VINF_EM_RESET; /* Triple fault equals a reset. */
3310	break;
3311
3312	case VMX_EXIT_INIT_SIGNAL: /* 3 INIT signal. */
3313	case VMX_EXIT_SIPI: /* 4 Start-up IPI (SIPI). */
3314	rc = VINF_EM_RAW_INTERRUPT;
3315	AssertFailed(); /* Can't happen. Yet. */
3316	break;
3317
3318	case VMX_EXIT_IO_SMI_IRQ: /* 5 I/O system-management interrupt (SMI). */
3319	case VMX_EXIT_SMI_IRQ: /* 6 Other SMI. */
3320	rc = VINF_EM_RAW_INTERRUPT;
3321	AssertFailed(); /* Can't happen afaik. */
3322	break;
3323
3324	case VMX_EXIT_TASK_SWITCH: /* 9 Task switch. */
3325	rc = VERR_EM_INTERPRETER;
3326	break;
3327
3328	case VMX_EXIT_HLT: /* 12 Guest software attempted to execute HLT. */
3329	/** Check if external interrupts are pending; if so, don't switch back. */
3330	pCtx->rip++; /* skip hlt */
3331	if ( pCtx->eflags.Bits.u1IF
3332	&& VM_FF_ISPENDING(pVM, (VM_FF_INTERRUPT_APIC\|VM_FF_INTERRUPT_PIC)))
3333	goto ResumeExecution;
3334
3335	rc = VINF_EM_HALT;
3336	break;
3337
3338	case VMX_EXIT_RSM: /* 17 Guest software attempted to execute RSM in SMM. */
3339	AssertFailed(); /* can't happen. */
3340	rc = VINF_EM_RAW_EXCEPTION_PRIVILEGED;
3341	break;
3342
3343	case VMX_EXIT_VMCALL: /* 18 Guest software executed VMCALL. */
3344	case VMX_EXIT_VMCLEAR: /* 19 Guest software executed VMCLEAR. */
3345	case VMX_EXIT_VMLAUNCH: /* 20 Guest software executed VMLAUNCH. */
3346	case VMX_EXIT_VMPTRLD: /* 21 Guest software executed VMPTRLD. */
3347	case VMX_EXIT_VMPTRST: /* 22 Guest software executed VMPTRST. */
3348	case VMX_EXIT_VMREAD: /* 23 Guest software executed VMREAD. */
3349	case VMX_EXIT_VMRESUME: /* 24 Guest software executed VMRESUME. */
3350	case VMX_EXIT_VMWRITE: /* 25 Guest software executed VMWRITE. */
3351	case VMX_EXIT_VMXOFF: /* 26 Guest software executed VMXOFF. */
3352	case VMX_EXIT_VMXON: /* 27 Guest software executed VMXON. */
3353	/** @todo inject #UD immediately */
3354	rc = VINF_EM_RAW_EXCEPTION_PRIVILEGED;
3355	break;
3356
3357	case VMX_EXIT_CPUID: /* 10 Guest software attempted to execute CPUID. */
3358	case VMX_EXIT_RDTSC: /* 16 Guest software attempted to execute RDTSC. */
3359	case VMX_EXIT_INVPG: /* 14 Guest software attempted to execute INVPG. */
3360	case VMX_EXIT_CRX_MOVE: /* 28 Control-register accesses. */
3361	case VMX_EXIT_DRX_MOVE: /* 29 Debug-register accesses. */
3362	case VMX_EXIT_PORT_IO: /* 30 I/O instruction. */
3363	/* already handled above */
3364	AssertMsg( rc == VINF_PGM_CHANGE_MODE
3365	\|\| rc == VINF_EM_RAW_INTERRUPT
3366	\|\| rc == VERR_EM_INTERPRETER
3367	\|\| rc == VINF_EM_RAW_EMULATE_INSTR
3368	\|\| rc == VINF_PGM_SYNC_CR3
3369	\|\| rc == VINF_IOM_HC_IOPORT_READ
3370	\|\| rc == VINF_IOM_HC_IOPORT_WRITE
3371	\|\| rc == VINF_EM_RAW_GUEST_TRAP
3372	\|\| rc == VINF_TRPM_XCPT_DISPATCHED
3373	\|\| rc == VINF_EM_RESCHEDULE_REM,
3374	("rc = %d\n", rc));
3375	break;
3376
3377	case VMX_EXIT_TPR: /* 43 TPR below threshold. Guest software executed MOV to CR8. */
3378	case VMX_EXIT_RDMSR: /* 31 RDMSR. Guest software attempted to execute RDMSR. */
3379	case VMX_EXIT_WRMSR: /* 32 WRMSR. Guest software attempted to execute WRMSR. */
3380	/* Note: If we decide to emulate them here, then we must sync the MSRs that could have been changed (sysenter, fs/gs base)!!! */
3381	rc = VERR_EM_INTERPRETER;
3382	break;
3383
3384	case VMX_EXIT_RDPMC: /* 15 Guest software attempted to execute RDPMC. */
3385	case VMX_EXIT_MWAIT: /* 36 Guest software executed MWAIT. */
3386	case VMX_EXIT_MONITOR: /* 39 Guest software attempted to execute MONITOR. */
3387	case VMX_EXIT_PAUSE: /* 40 Guest software attempted to execute PAUSE. */
3388	rc = VINF_EM_RAW_EXCEPTION_PRIVILEGED;
3389	break;
3390
3391	case VMX_EXIT_IRQ_WINDOW: /* 7 Interrupt window. */
3392	Assert(rc == VINF_EM_RAW_INTERRUPT);
3393	break;
3394
3395	case VMX_EXIT_ERR_INVALID_GUEST_STATE: /* 33 VM-entry failure due to invalid guest state. */
3396	{
3397	#ifdef VBOX_STRICT
3398	RTCCUINTREG val = 0;
3399
3400	Log(("VMX_EXIT_ERR_INVALID_GUEST_STATE\n"));
3401
3402	VMXReadVMCS(VMX_VMCS64_GUEST_RIP, &val);
3403	Log(("Old eip %RGv new %RGv\n", (RTGCPTR)pCtx->rip, (RTGCPTR)val));
3404
3405	VMXReadVMCS(VMX_VMCS64_GUEST_CR0, &val);
3406	Log(("VMX_VMCS_GUEST_CR0 %RX64\n", (uint64_t)val));
3407
3408	VMXReadVMCS(VMX_VMCS64_GUEST_CR3, &val);
3409	Log(("VMX_VMCS_GUEST_CR3 %RX64\n", (uint64_t)val));
3410
3411	VMXReadVMCS(VMX_VMCS64_GUEST_CR4, &val);
3412	Log(("VMX_VMCS_GUEST_CR4 %RX64\n", (uint64_t)val));
3413
3414	VMXReadVMCS(VMX_VMCS_GUEST_RFLAGS, &val);
3415	Log(("VMX_VMCS_GUEST_RFLAGS %08x\n", val));
3416
3417	VMX_LOG_SELREG(CS, "CS");
3418	VMX_LOG_SELREG(DS, "DS");
3419	VMX_LOG_SELREG(ES, "ES");
3420	VMX_LOG_SELREG(FS, "FS");
3421	VMX_LOG_SELREG(GS, "GS");
3422	VMX_LOG_SELREG(SS, "SS");
3423	VMX_LOG_SELREG(TR, "TR");
3424	VMX_LOG_SELREG(LDTR, "LDTR");
3425
3426	VMXReadVMCS(VMX_VMCS64_GUEST_GDTR_BASE, &val);
3427	Log(("VMX_VMCS_GUEST_GDTR_BASE %RX64\n", (uint64_t)val));
3428	VMXReadVMCS(VMX_VMCS64_GUEST_IDTR_BASE, &val);
3429	Log(("VMX_VMCS_GUEST_IDTR_BASE %RX64\n", (uint64_t)val));
3430	#endif /* VBOX_STRICT */
3431	rc = VERR_VMX_INVALID_GUEST_STATE;
3432	break;
3433	}
3434
3435	case VMX_EXIT_ERR_MSR_LOAD: /* 34 VM-entry failure due to MSR loading. */
3436	case VMX_EXIT_ERR_MACHINE_CHECK: /* 41 VM-entry failure due to machine-check. */
3437	default:
3438	rc = VERR_VMX_UNEXPECTED_EXIT_CODE;
3439	AssertMsgFailed(("Unexpected exit code %d\n", exitReason)); /* Can't happen. */
3440	break;
3441
3442	}
3443	end:
3444
3445	/* Signal changes for the recompiler. */
3446	CPUMSetChangedFlags(pVM, CPUM_CHANGED_SYSENTER_MSR \| CPUM_CHANGED_LDTR \| CPUM_CHANGED_GDTR \| CPUM_CHANGED_IDTR \| CPUM_CHANGED_TR \| CPUM_CHANGED_HIDDEN_SEL_REGS);
3447
3448	/* If we executed vmlaunch/vmresume and an external irq was pending, then we don't have to do a full sync the next time. */
3449	if ( exitReason == VMX_EXIT_EXTERNAL_IRQ
3450	&& !VMX_EXIT_INTERRUPTION_INFO_VALID(intInfo))
3451	{
3452	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatPendingHostIrq);
3453	/* On the next entry we'll only sync the host context. */
3454	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_HOST_CONTEXT;
3455	}
3456	else
3457	{
3458	/* On the next entry we'll sync everything. */
3459	/** @todo we can do better than this */
3460	/* Not in the VINF_PGM_CHANGE_MODE though! */
3461	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_ALL;
3462	}
3463
3464	/* translate into a less severe return code */
3465	if (rc == VERR_EM_INTERPRETER)
3466	rc = VINF_EM_RAW_EMULATE_INSTR;
3467	else
3468	/* Try to extract more information about what might have gone wrong here. */
3469	if (rc == VERR_VMX_INVALID_VMCS_PTR)
3470	{
3471	VMXGetActivateVMCS(&pVCpu->hwaccm.s.vmx.lasterror.u64VMCSPhys);
3472	pVCpu->hwaccm.s.vmx.lasterror.ulVMCSRevision = (uint32_t )pVCpu->hwaccm.s.vmx.pVMCS;
3473	pVCpu->hwaccm.s.vmx.lasterror.idEnteredCpu = pVCpu->hwaccm.s.idEnteredCpu;
3474	pVCpu->hwaccm.s.vmx.lasterror.idCurrentCpu = RTMpCpuId();
3475	}
3476
3477	STAM_STATS({
3478	if (fStatExit2Started) STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2, y);
3479	else if (fStatEntryStarted) STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatEntry, x);
3480	});
3481	Log2(("X"));
3482	return rc;
3483	}
3484
3485
3486	/**
3487	* Enters the VT-x session
3488	*
3489	* @returns VBox status code.
3490	* @param pVM The VM to operate on.
3491	* @param pVCpu The VMCPU to operate on.
3492	* @param pCpu CPU info struct
3493	*/
3494	VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHWACCM_CPUINFO pCpu)
3495	{
3496	Assert(pVM->hwaccm.s.vmx.fSupported);
3497
3498	unsigned cr4 = ASMGetCR4();
3499	if (!(cr4 & X86_CR4_VMXE))
3500	{
3501	AssertMsgFailed(("X86_CR4_VMXE should be set!\n"));
3502	return VERR_VMX_X86_CR4_VMXE_CLEARED;
3503	}
3504
3505	/* Activate the VM Control Structure. */
3506	int rc = VMXActivateVMCS(pVCpu->hwaccm.s.vmx.pVMCSPhys);
3507	if (RT_FAILURE(rc))
3508	return rc;
3509
3510	pVCpu->hwaccm.s.fResumeVM = false;
3511	return VINF_SUCCESS;
3512	}
3513
3514
3515	/**
3516	* Leaves the VT-x session
3517	*
3518	* @returns VBox status code.
3519	* @param pVM The VM to operate on.
3520	* @param pVCpu The VMCPU to operate on.
3521	* @param pCtx CPU context
3522	*/
3523	VMMR0DECL(int) VMXR0Leave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3524	{
3525	Assert(pVM->hwaccm.s.vmx.fSupported);
3526
3527	/* Save the guest debug state if necessary. */
3528	if (CPUMIsGuestDebugStateActive(pVM))
3529	{
3530	CPUMR0SaveGuestDebugState(pVM, pVCpu, pCtx, true /* save DR6 */);
3531
3532	/* Enable drx move intercepts again. */
3533	pVCpu->hwaccm.s.vmx.proc_ctls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
3534	int rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
3535	AssertRC(rc);
3536
3537	/* Resync the debug registers the next time. */
3538	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_DEBUG;
3539	}
3540	else
3541	Assert(pVCpu->hwaccm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT);
3542
3543	/* Clear VM Control Structure. Marking it inactive, clearing implementation specific data and writing back VMCS data to memory. */
3544	int rc = VMXClearVMCS(pVCpu->hwaccm.s.vmx.pVMCSPhys);
3545	AssertRC(rc);
3546
3547	return VINF_SUCCESS;
3548	}
3549
3550	/**
3551	* Flush the TLB (EPT)
3552	*
3553	* @returns VBox status code.
3554	* @param pVM The VM to operate on.
3555	* @param pVCpu The VM CPU to operate on.
3556	* @param enmFlush Type of flush
3557	* @param GCPhys Physical address of the page to flush
3558	*/
3559	static void vmxR0FlushEPT(PVM pVM, PVMCPU pVCpu, VMX_FLUSH enmFlush, RTGCPHYS GCPhys)
3560	{
3561	uint64_t descriptor[2];
3562
3563	LogFlow(("vmxR0FlushEPT %d %RGv\n", enmFlush, GCPhys));
3564	Assert(pVM->hwaccm.s.fNestedPaging);
3565	descriptor[0] = pVCpu->hwaccm.s.vmx.GCPhysEPTP;
3566	descriptor[1] = GCPhys;
3567	int rc = VMXR0InvEPT(enmFlush, &descriptor[0]);
3568	AssertRC(rc);
3569	}
3570
3571	#ifdef HWACCM_VTX_WITH_VPID
3572	/**
3573	* Flush the TLB (EPT)
3574	*
3575	* @returns VBox status code.
3576	* @param pVM The VM to operate on.
3577	* @param pVCpu The VM CPU to operate on.
3578	* @param enmFlush Type of flush
3579	* @param GCPtr Virtual address of the page to flush
3580	*/
3581	static void vmxR0FlushVPID(PVM pVM, PVMCPU pVCpu, VMX_FLUSH enmFlush, RTGCPTR GCPtr)
3582	{
3583	#if HC_ARCH_BITS == 32
3584	/* If we get a flush in 64 bits guest mode, then force a full TLB flush. Invvpid probably takes only 32 bits addresses. (@todo) */
3585	if ( CPUMIsGuestInLongMode(pVM)
3586	&& !VMX_IS_64BIT_HOST_MODE())
3587	{
3588	pVCpu->hwaccm.s.fForceTLBFlush = true;
3589	}
3590	else
3591	#endif
3592	{
3593	uint64_t descriptor[2];
3594
3595	Assert(pVM->hwaccm.s.vmx.fVPID);
3596	descriptor[0] = pVCpu->hwaccm.s.uCurrentASID;
3597	descriptor[1] = GCPtr;
3598	int rc = VMXR0InvVPID(enmFlush, &descriptor[0]);
3599	AssertRC(rc);
3600	}
3601	}
3602	#endif /* HWACCM_VTX_WITH_VPID */
3603
3604	/**
3605	* Invalidates a guest page
3606	*
3607	* @returns VBox status code.
3608	* @param pVM The VM to operate on.
3609	* @param pVCpu The VM CPU to operate on.
3610	* @param GCVirt Page to invalidate
3611	*/
3612	VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
3613	{
3614	bool fFlushPending = pVCpu->hwaccm.s.fForceTLBFlush;
3615
3616	Log2(("VMXR0InvalidatePage %RGv\n", GCVirt));
3617
3618	/* Only relevant if we want to use VPID.
3619	* In the nested paging case we still see such calls, but
3620	* can safely ignore them. (e.g. after cr3 updates)
3621	*/
3622	#ifdef HWACCM_VTX_WITH_VPID
3623	/* Skip it if a TLB flush is already pending. */
3624	if ( !fFlushPending
3625	&& pVM->hwaccm.s.vmx.fVPID)
3626	vmxR0FlushVPID(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushPage, GCVirt);
3627	#endif /* HWACCM_VTX_WITH_VPID */
3628
3629	return VINF_SUCCESS;
3630	}
3631
3632	/**
3633	* Invalidates a guest page by physical address
3634	*
3635	* NOTE: Assumes the current instruction references this physical page though a virtual address!!
3636	*
3637	* @returns VBox status code.
3638	* @param pVM The VM to operate on.
3639	* @param pVCpu The VM CPU to operate on.
3640	* @param GCPhys Page to invalidate
3641	*/
3642	VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
3643	{
3644	bool fFlushPending = pVCpu->hwaccm.s.fForceTLBFlush;
3645
3646	Assert(pVM->hwaccm.s.fNestedPaging);
3647
3648	LogFlow(("VMXR0InvalidatePhysPage %RGp\n", GCPhys));
3649
3650	/* Skip it if a TLB flush is already pending. */
3651	if (!fFlushPending)
3652	vmxR0FlushEPT(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushPage, GCPhys);
3653
3654	return VINF_SUCCESS;
3655	}
3656
3657	/**
3658	* Report world switch error and dump some useful debug info
3659	*
3660	* @param pVM The VM to operate on.
3661	* @param pVCpu The VMCPU to operate on.
3662	* @param rc Return code
3663	* @param pCtx Current CPU context (not updated)
3664	*/
3665	static void VMXR0ReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rc, PCPUMCTX pCtx)
3666	{
3667	switch (rc)
3668	{
3669	case VERR_VMX_INVALID_VMXON_PTR:
3670	AssertFailed();
3671	break;
3672
3673	case VERR_VMX_UNABLE_TO_START_VM:
3674	case VERR_VMX_UNABLE_TO_RESUME_VM:
3675	{
3676	int rc;
3677	RTCCUINTREG exitReason, instrError;
3678
3679	rc = VMXReadVMCS(VMX_VMCS32_RO_EXIT_REASON, &exitReason);
3680	rc \|= VMXReadVMCS(VMX_VMCS32_RO_VM_INSTR_ERROR, &instrError);
3681	AssertRC(rc);
3682	if (rc == VINF_SUCCESS)
3683	{
3684	Log(("Unable to start/resume VM for reason: %x. Instruction error %x\n", (uint32_t)exitReason, (uint32_t)instrError));
3685	Log(("Current stack %08x\n", &rc));
3686
3687	pVCpu->hwaccm.s.vmx.lasterror.ulInstrError = instrError;
3688	pVCpu->hwaccm.s.vmx.lasterror.ulExitReason = exitReason;
3689
3690	#ifdef VBOX_STRICT
3691	RTGDTR gdtr;
3692	PX86DESCHC pDesc;
3693	RTCCUINTREG val;
3694
3695	ASMGetGDTR(&gdtr);
3696
3697	VMXReadVMCS(VMX_VMCS64_GUEST_RIP, &val);
3698	Log(("Old eip %RGv new %RGv\n", (RTGCPTR)pCtx->rip, (RTGCPTR)val));
3699	VMXReadVMCS(VMX_VMCS_CTRL_PIN_EXEC_CONTROLS, &val);
3700	Log(("VMX_VMCS_CTRL_PIN_EXEC_CONTROLS %08x\n", val));
3701	VMXReadVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, &val);
3702	Log(("VMX_VMCS_CTRL_PROC_EXEC_CONTROLS %08x\n", val));
3703	VMXReadVMCS(VMX_VMCS_CTRL_ENTRY_CONTROLS, &val);
3704	Log(("VMX_VMCS_CTRL_ENTRY_CONTROLS %08x\n", val));
3705	VMXReadVMCS(VMX_VMCS_CTRL_EXIT_CONTROLS, &val);
3706	Log(("VMX_VMCS_CTRL_EXIT_CONTROLS %08x\n", val));
3707
3708	VMXReadVMCS(VMX_VMCS_HOST_CR0, &val);
3709	Log(("VMX_VMCS_HOST_CR0 %08x\n", val));
3710
3711	VMXReadVMCS(VMX_VMCS_HOST_CR3, &val);
3712	Log(("VMX_VMCS_HOST_CR3 %08x\n", val));
3713
3714	VMXReadVMCS(VMX_VMCS_HOST_CR4, &val);
3715	Log(("VMX_VMCS_HOST_CR4 %08x\n", val));
3716
3717	VMXReadVMCS(VMX_VMCS16_HOST_FIELD_CS, &val);
3718	Log(("VMX_VMCS_HOST_FIELD_CS %08x\n", val));
3719
3720	VMXReadVMCS(VMX_VMCS_GUEST_RFLAGS, &val);
3721	Log(("VMX_VMCS_GUEST_RFLAGS %08x\n", val));
3722
3723	if (val < gdtr.cbGdt)
3724	{
3725	pDesc = &((PX86DESCHC)gdtr.pGdt)[val >> X86_SEL_SHIFT_HC];
3726	HWACCMR0DumpDescriptor(pDesc, val, "CS: ");
3727	}
3728
3729	VMXReadVMCS(VMX_VMCS16_HOST_FIELD_DS, &val);
3730	Log(("VMX_VMCS_HOST_FIELD_DS %08x\n", val));
3731	if (val < gdtr.cbGdt)
3732	{
3733	pDesc = &((PX86DESCHC)gdtr.pGdt)[val >> X86_SEL_SHIFT_HC];
3734	HWACCMR0DumpDescriptor(pDesc, val, "DS: ");
3735	}
3736
3737	VMXReadVMCS(VMX_VMCS16_HOST_FIELD_ES, &val);
3738	Log(("VMX_VMCS_HOST_FIELD_ES %08x\n", val));
3739	if (val < gdtr.cbGdt)
3740	{
3741	pDesc = &((PX86DESCHC)gdtr.pGdt)[val >> X86_SEL_SHIFT_HC];
3742	HWACCMR0DumpDescriptor(pDesc, val, "ES: ");
3743	}
3744
3745	VMXReadVMCS(VMX_VMCS16_HOST_FIELD_FS, &val);
3746	Log(("VMX_VMCS16_HOST_FIELD_FS %08x\n", val));
3747	if (val < gdtr.cbGdt)
3748	{
3749	pDesc = &((PX86DESCHC)gdtr.pGdt)[val >> X86_SEL_SHIFT_HC];
3750	HWACCMR0DumpDescriptor(pDesc, val, "FS: ");
3751	}
3752
3753	VMXReadVMCS(VMX_VMCS16_HOST_FIELD_GS, &val);
3754	Log(("VMX_VMCS16_HOST_FIELD_GS %08x\n", val));
3755	if (val < gdtr.cbGdt)
3756	{
3757	pDesc = &((PX86DESCHC)gdtr.pGdt)[val >> X86_SEL_SHIFT_HC];
3758	HWACCMR0DumpDescriptor(pDesc, val, "GS: ");
3759	}
3760
3761	VMXReadVMCS(VMX_VMCS16_HOST_FIELD_SS, &val);
3762	Log(("VMX_VMCS16_HOST_FIELD_SS %08x\n", val));
3763	if (val < gdtr.cbGdt)
3764	{
3765	pDesc = &((PX86DESCHC)gdtr.pGdt)[val >> X86_SEL_SHIFT_HC];
3766	HWACCMR0DumpDescriptor(pDesc, val, "SS: ");
3767	}
3768
3769	VMXReadVMCS(VMX_VMCS16_HOST_FIELD_TR, &val);
3770	Log(("VMX_VMCS16_HOST_FIELD_TR %08x\n", val));
3771	if (val < gdtr.cbGdt)
3772	{
3773	pDesc = &((PX86DESCHC)gdtr.pGdt)[val >> X86_SEL_SHIFT_HC];
3774	HWACCMR0DumpDescriptor(pDesc, val, "TR: ");
3775	}
3776
3777	VMXReadVMCS(VMX_VMCS_HOST_TR_BASE, &val);
3778	Log(("VMX_VMCS_HOST_TR_BASE %RHv\n", val));
3779
3780	VMXReadVMCS(VMX_VMCS_HOST_GDTR_BASE, &val);
3781	Log(("VMX_VMCS_HOST_GDTR_BASE %RHv\n", val));
3782	VMXReadVMCS(VMX_VMCS_HOST_IDTR_BASE, &val);
3783	Log(("VMX_VMCS_HOST_IDTR_BASE %RHv\n", val));
3784
3785	VMXReadVMCS(VMX_VMCS32_HOST_SYSENTER_CS, &val);
3786	Log(("VMX_VMCS_HOST_SYSENTER_CS %08x\n", val));
3787
3788	VMXReadVMCS(VMX_VMCS_HOST_SYSENTER_EIP, &val);
3789	Log(("VMX_VMCS_HOST_SYSENTER_EIP %RHv\n", val));
3790
3791	VMXReadVMCS(VMX_VMCS_HOST_SYSENTER_ESP, &val);
3792	Log(("VMX_VMCS_HOST_SYSENTER_ESP %RHv\n", val));
3793
3794	VMXReadVMCS(VMX_VMCS_HOST_RSP, &val);
3795	Log(("VMX_VMCS_HOST_RSP %RHv\n", val));
3796	VMXReadVMCS(VMX_VMCS_HOST_RIP, &val);
3797	Log(("VMX_VMCS_HOST_RIP %RHv\n", val));
3798
3799	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3800	if (VMX_IS_64BIT_HOST_MODE())
3801	{
3802	Log(("MSR_K6_EFER = %RX64\n", ASMRdMsr(MSR_K6_EFER)));
3803	Log(("MSR_K6_STAR = %RX64\n", ASMRdMsr(MSR_K6_STAR)));
3804	Log(("MSR_K8_LSTAR = %RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
3805	Log(("MSR_K8_CSTAR = %RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
3806	Log(("MSR_K8_SF_MASK = %RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
3807	}
3808	# endif
3809	#endif /* VBOX_STRICT */
3810	}
3811	break;
3812	}
3813
3814	default:
3815	/* impossible */
3816	AssertMsgFailed(("%Rrc (%#x)\n", rc, rc));
3817	break;
3818	}
3819	}
3820
3821	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3822	/**
3823	* Prepares for and executes VMLAUNCH (64 bits guest mode)
3824	*
3825	* @returns VBox status code
3826	* @param fResume vmlauch/vmresume
3827	* @param pCtx Guest context
3828	* @param pCache VMCS cache
3829	* @param pVM The VM to operate on.
3830	* @param pVCpu The VMCPU to operate on.
3831	*/
3832	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
3833	{
3834	uint32_t aParam[6];
3835	PHWACCM_CPUINFO pCpu;
3836	RTHCPHYS pPageCpuPhys;
3837	int rc;
3838
3839	pCpu = HWACCMR0GetCurrentCpu();
3840	pPageCpuPhys = RTR0MemObjGetPagePhysAddr(pCpu->pMemObj, 0);
3841
3842	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
3843	pCache->uPos = 1;
3844	pCache->interPD = PGMGetInterPaeCR3(pVM);
3845	pCache->pSwitcher = (uint64_t)pVM->hwaccm.s.pfnHost32ToGuest64R0;
3846	#endif
3847
3848	#ifdef DEBUG
3849	pCache->TestIn.pPageCpuPhys = 0;
3850	pCache->TestIn.pVMCSPhys = 0;
3851	pCache->TestIn.pCache = 0;
3852	pCache->TestOut.pVMCSPhys = 0;
3853	pCache->TestOut.pCache = 0;
3854	pCache->TestOut.pCtx = 0;
3855	pCache->TestOut.eflags = 0;
3856	#endif
3857
3858	aParam[0] = (uint32_t)(pPageCpuPhys); /* Param 1: VMXON physical address - Lo. */
3859	aParam[1] = (uint32_t)(pPageCpuPhys >> 32); /* Param 1: VMXON physical address - Hi. */
3860	aParam[2] = (uint32_t)(pVCpu->hwaccm.s.vmx.pVMCSPhys); /* Param 2: VMCS physical address - Lo. */
3861	aParam[3] = (uint32_t)(pVCpu->hwaccm.s.vmx.pVMCSPhys >> 32); /* Param 2: VMCS physical address - Hi. */
3862	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hwaccm.s.vmx.VMCSCache);
3863	aParam[5] = 0;
3864
3865	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
3866	pCtx->dr[4] = pVM->hwaccm.s.vmx.pScratchPhys + 16 + 8;
3867	(uint32_t )(pVM->hwaccm.s.vmx.pScratch + 16 + 8) = 1;
3868	#endif
3869	rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, pVM->hwaccm.s.pfnVMXGCStartVM64, 6, &aParam[0]);
3870
3871	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
3872	Assert((uint32_t )(pVM->hwaccm.s.vmx.pScratch + 16 + 8) == 5);
3873	Assert(pCtx->dr[4] == 10);
3874	(uint32_t )(pVM->hwaccm.s.vmx.pScratch + 16 + 8) = 0xff;
3875	#endif
3876
3877	#ifdef DEBUG
3878	AssertMsg(pCache->TestIn.pPageCpuPhys == pPageCpuPhys, ("%RHp vs %RHp\n", pCache->TestIn.pPageCpuPhys, pPageCpuPhys));
3879	AssertMsg(pCache->TestIn.pVMCSPhys == pVCpu->hwaccm.s.vmx.pVMCSPhys, ("%RHp vs %RHp\n", pCache->TestIn.pVMCSPhys, pVCpu->hwaccm.s.vmx.pVMCSPhys));
3880	AssertMsg(pCache->TestIn.pVMCSPhys == pCache->TestOut.pVMCSPhys, ("%RHp vs %RHp\n", pCache->TestIn.pVMCSPhys, pCache->TestOut.pVMCSPhys));
3881	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache, pCache->TestOut.pCache));
3882	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hwaccm.s.vmx.VMCSCache), ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hwaccm.s.vmx.VMCSCache)));
3883	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx, pCache->TestOut.pCtx));
3884	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
3885	#endif
3886	return rc;
3887	}
3888
3889	/**
3890	* Executes the specified handler in 64 mode
3891	*
3892	* @returns VBox status code.
3893	* @param pVM The VM to operate on.
3894	* @param pVCpu The VMCPU to operate on.
3895	* @param pCtx Guest context
3896	* @param pfnHandler RC handler
3897	* @param cbParam Number of parameters
3898	* @param paParam Array of 32 bits parameters
3899	*/
3900	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, RTRCPTR pfnHandler, uint32_t cbParam, uint32_t *paParam)
3901	{
3902	int rc, rc2;
3903	PHWACCM_CPUINFO pCpu;
3904	RTHCPHYS pPageCpuPhys;
3905
3906	/* @todo This code is not guest SMP safe (hyper context) */
3907	AssertReturn(pVM->cCPUs == 1, VERR_ACCESS_DENIED);
3908	AssertReturn(pVM->hwaccm.s.pfnHost32ToGuest64R0, VERR_INTERNAL_ERROR);
3909	Assert(pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hwaccm.s.vmx.VMCSCache.Write.aField));
3910	Assert(pVCpu->hwaccm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hwaccm.s.vmx.VMCSCache.Read.aField));
3911
3912	#ifdef VBOX_STRICT
3913	for (unsigned i=0;i<pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries;i++)
3914	Assert(vmxR0IsValidWriteField(pVCpu->hwaccm.s.vmx.VMCSCache.Write.aField[i]));
3915
3916	for (unsigned i=0;i<pVCpu->hwaccm.s.vmx.VMCSCache.Read.cValidEntries;i++)
3917	Assert(vmxR0IsValidReadField(pVCpu->hwaccm.s.vmx.VMCSCache.Read.aField[i]));
3918	#endif
3919
3920	pCpu = HWACCMR0GetCurrentCpu();
3921	pPageCpuPhys = RTR0MemObjGetPagePhysAddr(pCpu->pMemObj, 0);
3922
3923	/* Clear VM Control Structure. Marking it inactive, clearing implementation specific data and writing back VMCS data to memory. */
3924	VMXClearVMCS(pVCpu->hwaccm.s.vmx.pVMCSPhys);
3925
3926	/* Leave VMX Root Mode. */
3927	VMXDisable();
3928
3929	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
3930
3931	CPUMSetHyperESP(pVM, VMMGetStackRC(pVM));
3932	CPUMSetHyperEIP(pVM, pfnHandler);
3933	for (int i=(int)cbParam-1;i>=0;i--)
3934	CPUMPushHyper(pVM, paParam[i]);
3935
3936	STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatWorldSwitch3264, z);
3937	/* Call switcher. */
3938	rc = pVM->hwaccm.s.pfnHost32ToGuest64R0(pVM);
3939	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatWorldSwitch3264, z);
3940
3941	/* Make sure the VMX instructions don't cause #UD faults. */
3942	ASMSetCR4(ASMGetCR4() \| X86_CR4_VMXE);
3943
3944	/* Enter VMX Root Mode */
3945	rc2 = VMXEnable(pPageCpuPhys);
3946	if (RT_FAILURE(rc2))
3947	{
3948	if (pVM)
3949	VMXR0CheckError(pVM, pVCpu, rc2);
3950	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
3951	return VERR_VMX_VMXON_FAILED;
3952	}
3953
3954	rc2 = VMXActivateVMCS(pVCpu->hwaccm.s.vmx.pVMCSPhys);
3955	AssertRCReturn(rc2, rc2);
3956	#ifdef RT_OS_WINDOWS
3957	Assert(ASMGetFlags() & X86_EFL_IF);
3958	#else
3959	Assert(!(ASMGetFlags() & X86_EFL_IF));
3960	#endif
3961	return rc;
3962	}
3963
3964	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
3965
3966
3967	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
3968	/**
3969	* Executes VMWRITE
3970	*
3971	* @returns VBox status code
3972	* @param pVCpu The VMCPU to operate on.
3973	* @param idxField VMCS index
3974	* @param u64Val 16, 32 or 64 bits value
3975	*/
3976	VMMR0DECL(int) VMXWriteVMCS64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
3977	{
3978	int rc;
3979
3980	switch (idxField)
3981	{
3982	case VMX_VMCS_CTRL_TSC_OFFSET_FULL:
3983	case VMX_VMCS_CTRL_IO_BITMAP_A_FULL:
3984	case VMX_VMCS_CTRL_IO_BITMAP_B_FULL:
3985	case VMX_VMCS_CTRL_MSR_BITMAP_FULL:
3986	case VMX_VMCS_CTRL_VMEXIT_MSR_STORE_FULL:
3987	case VMX_VMCS_CTRL_VMEXIT_MSR_LOAD_FULL:
3988	case VMX_VMCS_CTRL_VMENTRY_MSR_LOAD_FULL:
3989	case VMX_VMCS_CTRL_VAPIC_PAGEADDR_FULL:
3990	case VMX_VMCS_GUEST_LINK_PTR_FULL:
3991	case VMX_VMCS_GUEST_PDPTR0_FULL:
3992	case VMX_VMCS_GUEST_PDPTR1_FULL:
3993	case VMX_VMCS_GUEST_PDPTR2_FULL:
3994	case VMX_VMCS_GUEST_PDPTR3_FULL:
3995	case VMX_VMCS_GUEST_DEBUGCTL_FULL:
3996	case VMX_VMCS_GUEST_EFER_FULL:
3997	case VMX_VMCS_CTRL_EPTP_FULL:
3998	/* These fields consist of two parts, which are both writable in 32 bits mode. */
3999	rc = VMXWriteVMCS32(idxField, u64Val);
4000	rc \|= VMXWriteVMCS32(idxField + 1, (uint32_t)(u64Val >> 32ULL));
4001	AssertRC(rc);
4002	return rc;
4003
4004	case VMX_VMCS64_GUEST_LDTR_BASE:
4005	case VMX_VMCS64_GUEST_TR_BASE:
4006	case VMX_VMCS64_GUEST_GDTR_BASE:
4007	case VMX_VMCS64_GUEST_IDTR_BASE:
4008	case VMX_VMCS64_GUEST_SYSENTER_EIP:
4009	case VMX_VMCS64_GUEST_SYSENTER_ESP:
4010	case VMX_VMCS64_GUEST_CR0:
4011	case VMX_VMCS64_GUEST_CR4:
4012	case VMX_VMCS64_GUEST_CR3:
4013	case VMX_VMCS64_GUEST_DR7:
4014	case VMX_VMCS64_GUEST_RIP:
4015	case VMX_VMCS64_GUEST_RSP:
4016	case VMX_VMCS64_GUEST_CS_BASE:
4017	case VMX_VMCS64_GUEST_DS_BASE:
4018	case VMX_VMCS64_GUEST_ES_BASE:
4019	case VMX_VMCS64_GUEST_FS_BASE:
4020	case VMX_VMCS64_GUEST_GS_BASE:
4021	case VMX_VMCS64_GUEST_SS_BASE:
4022	/* Queue a 64 bits value as we can't set it in 32 bits host mode. */
4023	if (u64Val >> 32ULL)
4024	rc = VMXWriteCachedVMCSEx(pVCpu, idxField, u64Val);
4025	else
4026	rc = VMXWriteVMCS32(idxField, (uint32_t)u64Val);
4027
4028	return rc;
4029
4030	default:
4031	AssertMsgFailed(("Unexpected field %x\n", idxField));
4032	return VERR_INVALID_PARAMETER;
4033	}
4034	}
4035
4036	/**
4037	* Cache VMCS writes for performance reasons (Darwin) and for running 64 bits guests on 32 bits hosts.
4038	*
4039	* @param pVCpu The VMCPU to operate on.
4040	* @param idxField VMCS field
4041	* @param u64Val Value
4042	*/
4043	VMMR0DECL(int) VMXWriteCachedVMCSEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4044	{
4045	PVMCSCACHE pCache = &pVCpu->hwaccm.s.vmx.VMCSCache;
4046
4047	AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1, ("entries=%x\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
4048
4049	/* Make sure there are no duplicates. */
4050	for (unsigned i=0;i<pCache->Write.cValidEntries;i++)
4051	{
4052	if (pCache->Write.aField[i] == idxField)
4053	{
4054	pCache->Write.aFieldVal[i] = u64Val;
4055	return VINF_SUCCESS;
4056	}
4057	}
4058
4059	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
4060	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
4061	pCache->Write.cValidEntries++;
4062	return VINF_SUCCESS;
4063	}
4064
4065	#endif /* HC_ARCH_BITS == 32 && !VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0 */
4066
4067	#ifdef VBOX_STRICT
4068	static bool vmxR0IsValidReadField(uint32_t idxField)
4069	{
4070	switch(idxField)
4071	{
4072	case VMX_VMCS64_GUEST_RIP:
4073	case VMX_VMCS64_GUEST_RSP:
4074	case VMX_VMCS_GUEST_RFLAGS:
4075	case VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE:
4076	case VMX_VMCS_CTRL_CR0_READ_SHADOW:
4077	case VMX_VMCS64_GUEST_CR0:
4078	case VMX_VMCS_CTRL_CR4_READ_SHADOW:
4079	case VMX_VMCS64_GUEST_CR4:
4080	case VMX_VMCS64_GUEST_DR7:
4081	case VMX_VMCS32_GUEST_SYSENTER_CS:
4082	case VMX_VMCS64_GUEST_SYSENTER_EIP:
4083	case VMX_VMCS64_GUEST_SYSENTER_ESP:
4084	case VMX_VMCS32_GUEST_GDTR_LIMIT:
4085	case VMX_VMCS64_GUEST_GDTR_BASE:
4086	case VMX_VMCS32_GUEST_IDTR_LIMIT:
4087	case VMX_VMCS64_GUEST_IDTR_BASE:
4088	case VMX_VMCS16_GUEST_FIELD_CS:
4089	case VMX_VMCS32_GUEST_CS_LIMIT:
4090	case VMX_VMCS64_GUEST_CS_BASE:
4091	case VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS:
4092	case VMX_VMCS16_GUEST_FIELD_DS:
4093	case VMX_VMCS32_GUEST_DS_LIMIT:
4094	case VMX_VMCS64_GUEST_DS_BASE:
4095	case VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS:
4096	case VMX_VMCS16_GUEST_FIELD_ES:
4097	case VMX_VMCS32_GUEST_ES_LIMIT:
4098	case VMX_VMCS64_GUEST_ES_BASE:
4099	case VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS:
4100	case VMX_VMCS16_GUEST_FIELD_FS:
4101	case VMX_VMCS32_GUEST_FS_LIMIT:
4102	case VMX_VMCS64_GUEST_FS_BASE:
4103	case VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS:
4104	case VMX_VMCS16_GUEST_FIELD_GS:
4105	case VMX_VMCS32_GUEST_GS_LIMIT:
4106	case VMX_VMCS64_GUEST_GS_BASE:
4107	case VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS:
4108	case VMX_VMCS16_GUEST_FIELD_SS:
4109	case VMX_VMCS32_GUEST_SS_LIMIT:
4110	case VMX_VMCS64_GUEST_SS_BASE:
4111	case VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS:
4112	case VMX_VMCS16_GUEST_FIELD_LDTR:
4113	case VMX_VMCS32_GUEST_LDTR_LIMIT:
4114	case VMX_VMCS64_GUEST_LDTR_BASE:
4115	case VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS:
4116	case VMX_VMCS16_GUEST_FIELD_TR:
4117	case VMX_VMCS32_GUEST_TR_LIMIT:
4118	case VMX_VMCS64_GUEST_TR_BASE:
4119	case VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS:
4120	case VMX_VMCS32_RO_EXIT_REASON:
4121	case VMX_VMCS32_RO_VM_INSTR_ERROR:
4122	case VMX_VMCS32_RO_EXIT_INSTR_LENGTH:
4123	case VMX_VMCS32_RO_EXIT_INTERRUPTION_ERRCODE:
4124	case VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO:
4125	case VMX_VMCS32_RO_EXIT_INSTR_INFO:
4126	case VMX_VMCS_RO_EXIT_QUALIFICATION:
4127	case VMX_VMCS32_RO_IDT_INFO:
4128	case VMX_VMCS32_RO_IDT_ERRCODE:
4129	case VMX_VMCS64_GUEST_CR3:
4130	case VMX_VMCS_EXIT_PHYS_ADDR_FULL:
4131	return true;
4132	}
4133	return false;
4134	}
4135
4136	static bool vmxR0IsValidWriteField(uint32_t idxField)
4137	{
4138	switch(idxField)
4139	{
4140	case VMX_VMCS64_GUEST_LDTR_BASE:
4141	case VMX_VMCS64_GUEST_TR_BASE:
4142	case VMX_VMCS64_GUEST_GDTR_BASE:
4143	case VMX_VMCS64_GUEST_IDTR_BASE:
4144	case VMX_VMCS64_GUEST_SYSENTER_EIP:
4145	case VMX_VMCS64_GUEST_SYSENTER_ESP:
4146	case VMX_VMCS64_GUEST_CR0:
4147	case VMX_VMCS64_GUEST_CR4:
4148	case VMX_VMCS64_GUEST_CR3:
4149	case VMX_VMCS64_GUEST_DR7:
4150	case VMX_VMCS64_GUEST_RIP:
4151	case VMX_VMCS64_GUEST_RSP:
4152	case VMX_VMCS64_GUEST_CS_BASE:
4153	case VMX_VMCS64_GUEST_DS_BASE:
4154	case VMX_VMCS64_GUEST_ES_BASE:
4155	case VMX_VMCS64_GUEST_FS_BASE:
4156	case VMX_VMCS64_GUEST_GS_BASE:
4157	case VMX_VMCS64_GUEST_SS_BASE:
4158	return true;
4159	}
4160	return false;
4161	}
4162
4163	#endif

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HWVMXR0.cpp@ 18768

Download in other formats: