HMVMXR0.cpp@ 46587

Last change on this file since 46587 was 46580, checked in by vboxsync, 12 years ago
VMM/HMSVMR0, HMVMXR0: AMD-V bits, VT-x minor change.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 382.6 KB

Line
1	/* $Id: HMVMXR0.cpp 46580 2013-06-17 10:20:31Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012-2013 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18	/*******************************************************************************
19	* Header Files *
20	*******************************************************************************/
21	#define LOG_GROUP LOG_GROUP_HM
22	#include <iprt/asm-amd64-x86.h>
23	#include <iprt/thread.h>
24	#include <iprt/string.h>
25
26	#include "HMInternal.h"
27	#include <VBox/vmm/vm.h>
28	#include "HWVMXR0.h"
29	#include <VBox/vmm/pdmapi.h>
30	#include <VBox/vmm/dbgf.h>
31	#include <VBox/vmm/iom.h>
32	#include <VBox/vmm/selm.h>
33	#include <VBox/vmm/tm.h>
34	#ifdef VBOX_WITH_REM
35	# include <VBox/vmm/rem.h>
36	#endif
37	#ifdef DEBUG_ramshankar
38	#define HMVMX_SAVE_FULL_GUEST_STATE
39	#define HMVMX_SYNC_FULL_GUEST_STATE
40	#define HMVMX_ALWAYS_TRAP_ALL_XCPTS
41	#define HMVMX_ALWAYS_TRAP_PF
42	#endif
43
44
45	/*******************************************************************************
46	* Defined Constants And Macros *
47	*******************************************************************************/
48	#define HMVMXHCUINTREG RTHCUINTREG
49	#if defined(RT_ARCH_AMD64)
50	# define HMVMX_IS_64BIT_HOST_MODE() (true)
51	#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
52	extern "C" uint32_t g_fVMXIs64bitHost;
53	# define HMVMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
54	# undef HMVMXHCUINTREG
55	# define HMVMXHCUINTREG uint64_t
56	#else
57	# define HMVMX_IS_64BIT_HOST_MODE() (false)
58	#endif
59
60	/** Use the function table. */
61	#define HMVMX_USE_FUNCTION_TABLE
62
63	/** This bit indicates the segment selector is unusable in VT-x. */
64	#define HMVMX_SEL_UNUSABLE RT_BIT(16)
65
66	/** Determine which tagged-TLB flush handler to use. */
67	#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
68	#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
69	#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
70	#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
71
72	/** @name Updated-guest-state flags.
73	* @{ */
74	#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
75	#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
76	#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
77	#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
78	#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
79	#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
80	#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
81	#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
82	#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
83	#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
84	#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
85	#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
86	#define HMVMX_UPDATED_GUEST_FS_BASE_MSR RT_BIT(12)
87	#define HMVMX_UPDATED_GUEST_GS_BASE_MSR RT_BIT(13)
88	#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(14)
89	#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(15)
90	#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(16)
91	#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(17)
92	#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(18)
93	#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(19)
94	#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
95	\| HMVMX_UPDATED_GUEST_RSP \
96	\| HMVMX_UPDATED_GUEST_RFLAGS \
97	\| HMVMX_UPDATED_GUEST_CR0 \
98	\| HMVMX_UPDATED_GUEST_CR3 \
99	\| HMVMX_UPDATED_GUEST_CR4 \
100	\| HMVMX_UPDATED_GUEST_GDTR \
101	\| HMVMX_UPDATED_GUEST_IDTR \
102	\| HMVMX_UPDATED_GUEST_LDTR \
103	\| HMVMX_UPDATED_GUEST_TR \
104	\| HMVMX_UPDATED_GUEST_SEGMENT_REGS \
105	\| HMVMX_UPDATED_GUEST_DEBUG \
106	\| HMVMX_UPDATED_GUEST_FS_BASE_MSR \
107	\| HMVMX_UPDATED_GUEST_GS_BASE_MSR \
108	\| HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
109	\| HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
110	\| HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
111	\| HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
112	\| HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
113	\| HMVMX_UPDATED_GUEST_APIC_STATE)
114	/** @} */
115
116	/**
117	* Flags to skip redundant reads of some common VMCS fields that are not part of
118	* the guest-CPU state but are in the transient structure.
119	*/
120	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
121	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
122	#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
123	#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
124	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
125	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
126
127	/**
128	* Exception bitmap mask for real-mode guests (real-on-v86). We need to intercept all exceptions manually (except #PF).
129	* #NM is also handled spearetely, see hmR0VmxLoadGuestControlRegs(). #PF need not be intercepted even in real-mode if
130	* we have Nested Paging support.
131	*/
132	#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) \| RT_BIT(X86_XCPT_DB) \| RT_BIT(X86_XCPT_NMI) \
133	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
134	\| RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ \| RT_BIT(X86_XCPT_DF) \
135	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
136	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
137	\| RT_BIT(X86_XCPT_MF) \| RT_BIT(X86_XCPT_AC) \| RT_BIT(X86_XCPT_MC) \
138	\| RT_BIT(X86_XCPT_XF))
139
140	/**
141	* Exception bitmap mask for all contributory exceptions.
142	*/
143	#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) \| RT_BIT(X86_XCPT_NP) \| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_TS) \
144	\| RT_BIT(X86_XCPT_DE))
145
146	/** Maximum VM-instruction error number. */
147	#define HMVMX_INSTR_ERROR_MAX 28
148
149	/** Profiling macro. */
150	#ifdef HM_PROFILE_EXIT_DISPATCH
151	# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
152	# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
153	#else
154	# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
155	# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
156	#endif
157
158
159	/*******************************************************************************
160	* Structures and Typedefs *
161	*******************************************************************************/
162	/** @name VMX transient.
163	*
164	* A state structure for holding miscellaneous information across
165	* VMX non-root operation and restored after the transition.
166	*
167	* @{ */
168	typedef struct VMXTRANSIENT
169	{
170	/** The host's rflags/eflags. */
171	RTCCUINTREG uEFlags;
172	#if HC_ARCH_BITS == 32
173	uint32_t u32Alignment0;
174	#endif
175	/** The guest's LSTAR MSR value used for TPR patching for 32-bit guests. */
176	uint64_t u64LStarMsr;
177	/** The guest's TPR value used for TPR shadowing. */
178	uint8_t u8GuestTpr;
179	/** Alignment. */
180	uint8_t abAlignment0[6];
181
182	/** The basic VM-exit reason. */
183	uint16_t uExitReason;
184	/** Alignment. */
185	uint16_t u16Alignment0;
186	/** The VM-exit interruption error code. */
187	uint32_t uExitIntrErrorCode;
188	/** The VM-exit exit qualification. */
189	uint64_t uExitQualification;
190
191	/** The VM-exit interruption-information field. */
192	uint32_t uExitIntrInfo;
193	/** The VM-exit instruction-length field. */
194	uint32_t cbInstr;
195	/** Whether the VM-entry failed or not. */
196	bool fVMEntryFailed;
197	/** Alignment. */
198	uint8_t abAlignment1[5];
199
200	/** The VM-entry interruption-information field. */
201	uint32_t uEntryIntrInfo;
202	/** The VM-entry exception error code field. */
203	uint32_t uEntryXcptErrorCode;
204	/** The VM-entry instruction length field. */
205	uint32_t cbEntryInstr;
206
207	/** IDT-vectoring information field. */
208	uint32_t uIdtVectoringInfo;
209	/** IDT-vectoring error code. */
210	uint32_t uIdtVectoringErrorCode;
211
212	/** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_. /
213	uint32_t fVmcsFieldsRead;
214	/** Whether TSC-offsetting should be setup before VM-entry. */
215	bool fUpdateTscOffsettingAndPreemptTimer;
216	/** Whether the VM-exit was caused by a page-fault during delivery of a
217	* contributary exception or a page-fault. */
218	bool fVectoringPF;
219	} VMXTRANSIENT, *PVMXTRANSIENT;
220	AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
221	AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntrInfo, sizeof(uint64_t));
222	AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntrInfo, sizeof(uint64_t));
223	/** @} */
224
225
226	/**
227	* MSR-bitmap read permissions.
228	*/
229	typedef enum VMXMSREXITREAD
230	{
231	/** Reading this MSR causes a VM-exit. */
232	VMXMSREXIT_INTERCEPT_READ = 0xb,
233	/** Reading this MSR does not cause a VM-exit. */
234	VMXMSREXIT_PASSTHRU_READ
235	} VMXMSREXITREAD;
236
237	/**
238	* MSR-bitmap write permissions.
239	*/
240	typedef enum VMXMSREXITWRITE
241	{
242	/** Writing to this MSR causes a VM-exit. */
243	VMXMSREXIT_INTERCEPT_WRITE = 0xd,
244	/** Writing to this MSR does not cause a VM-exit. */
245	VMXMSREXIT_PASSTHRU_WRITE
246	} VMXMSREXITWRITE;
247
248
249	/*******************************************************************************
250	* Internal Functions *
251	*******************************************************************************/
252	static void hmR0VmxFlushEpt(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush);
253	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr);
254	static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntrInfo, uint32_t cbInstr,
255	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState);
256	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
257	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
258	#endif
259	#ifndef HMVMX_USE_FUNCTION_TABLE
260	DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
261	#define HMVMX_EXIT_DECL static int
262	#else
263	#define HMVMX_EXIT_DECL static DECLCALLBACK(int)
264	#endif
265
266	HMVMX_EXIT_DECL hmR0VmxExitXcptNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
267	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
268	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
269	HMVMX_EXIT_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
270	HMVMX_EXIT_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
271	HMVMX_EXIT_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
272	HMVMX_EXIT_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
273	HMVMX_EXIT_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
274	HMVMX_EXIT_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
275	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
276	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
277	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
278	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
279	HMVMX_EXIT_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
280	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
281	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
282	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
283	HMVMX_EXIT_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
284	HMVMX_EXIT_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
285	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
286	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
287	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
288	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
289	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
290	HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
291	HMVMX_EXIT_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
292	HMVMX_EXIT_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
293	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
294	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
295	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
296	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
297	HMVMX_EXIT_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
298	HMVMX_EXIT_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
299	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
300	HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
301	HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
302	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
303	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
304	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
305	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
306	HMVMX_EXIT_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
307	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
308	HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
309	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
310
311	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
312	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
313	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
314	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
315	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
316	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
317	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
318
319
320	/*******************************************************************************
321	* Global Variables *
322	*******************************************************************************/
323	#ifdef HMVMX_USE_FUNCTION_TABLE
324	/**
325	* VM-exit handler.
326	*
327	* @returns VBox status code.
328	* @param pVCpu Pointer to the VMCPU.
329	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
330	* out-of-sync. Make sure to update the required
331	* fields before using them.
332	* @param pVmxTransient Pointer to the VMX-transient structure.
333	*/
334	typedef DECLCALLBACK(int) FNVMEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
335	/** Pointer to VM-exit handler. */
336	typedef FNVMEXITHANDLER *const PFNVMEXITHANDLER;
337
338	/**
339	* VMX_EXIT dispatch table.
340	*/
341	static const PFNVMEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
342	{
343	/* 00 VMX_EXIT_XCPT_NMI */ hmR0VmxExitXcptNmi,
344	/* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
345	/* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
346	/* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
347	/* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
348	/* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
349	/* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
350	/* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
351	/* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
352	/* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
353	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
354	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
355	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
356	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
357	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
358	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
359	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
360	/* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
361	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitSetPendingXcptUD,
362	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
363	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
364	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
365	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
366	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
367	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
368	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
369	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
370	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
371	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
372	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
373	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
374	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
375	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
376	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
377	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
378	/* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
379	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
380	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
381	/* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
382	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
383	/* 40 UNDEFINED */ hmR0VmxExitPause,
384	/* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
385	/* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
386	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
387	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
388	/* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
389	/* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
390	/* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
391	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
392	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
393	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
394	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
395	/* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
396	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
397	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
398	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
399	/* 56 UNDEFINED */ hmR0VmxExitErrUndefined,
400	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
401	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
402	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD
403	};
404	#endif /* HMVMX_USE_FUNCTION_TABLE */
405
406	#ifdef VBOX_STRICT
407	static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
408	{
409	/* 0 */ "(Not Used)",
410	/* 1 */ "VMCALL executed in VMX root operation.",
411	/* 2 */ "VMCLEAR with invalid physical address.",
412	/* 3 */ "VMCLEAR with VMXON pointer.",
413	/* 4 */ "VMLAUNCH with non-clear VMCS.",
414	/* 5 */ "VMRESUME with non-launched VMCS.",
415	/* 6 */ "VMRESUME after VMXOFF",
416	/* 7 */ "VM entry with invalid control fields.",
417	/* 8 */ "VM entry with invalid host state fields.",
418	/* 9 */ "VMPTRLD with invalid physical address.",
419	/* 10 */ "VMPTRLD with VMXON pointer.",
420	/* 11 */ "VMPTRLD with incorrect revision identifier.",
421	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
422	/* 13 */ "VMWRITE to read-only VMCS component.",
423	/* 14 */ "(Not Used)",
424	/* 15 */ "VMXON executed in VMX root operation.",
425	/* 16 */ "VM entry with invalid executive-VMCS pointer.",
426	/* 17 */ "VM entry with non-launched executing VMCS.",
427	/* 18 */ "VM entry with executive-VMCS pointer not VMXON pointer.",
428	/* 19 */ "VMCALL with non-clear VMCS.",
429	/* 20 */ "VMCALL with invalid VM-exit control fields.",
430	/* 21 */ "(Not Used)",
431	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
432	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
433	/* 24 */ "VMCALL with invalid SMM-monitor features.",
434	/* 25 */ "VM entry with invalid VM-execution control fields in executive VMCS.",
435	/* 26 */ "VM entry with events blocked by MOV SS.",
436	/* 27 */ "(Not Used)",
437	/* 28 */ "Invalid operand to INVEPT/INVVPID."
438	};
439	#endif /* VBOX_STRICT */
440
441
442
443	/**
444	* Updates the VM's last error record. If there was a VMX instruction error,
445	* reads the error data from the VMCS and updates VCPU's last error record as
446	* well.
447	*
448	* @param pVM Pointer to the VM.
449	* @param pVCpu Pointer to the VMCPU (can be NULL if @a rc is not
450	* VERR_VMX_UNABLE_TO_START_VM or
451	* VERR_VMX_INVALID_VMCS_FIELD).
452	* @param rc The error code.
453	*/
454	static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
455	{
456	AssertPtr(pVM);
457	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
458	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
459	{
460	AssertPtrReturnVoid(pVCpu);
461	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.lasterror.u32InstrError);
462	}
463	pVM->hm.s.lLastError = rc;
464	}
465
466
467	/**
468	* Reads the VM-entry interruption-information field from the VMCS into the VMX
469	* transient structure.
470	*
471	* @returns VBox status code.
472	* @param pVmxTransient Pointer to the VMX transient structure.
473	*
474	* @remarks No-long-jump zone!!!
475	*/
476	DECLINLINE(int) hmR0VmxReadEntryIntrInfoVmcs(PVMXTRANSIENT pVmxTransient)
477	{
478	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntrInfo);
479	AssertRCReturn(rc, rc);
480	return VINF_SUCCESS;
481	}
482
483
484	/**
485	* Reads the VM-entry exception error code field from the VMCS into
486	* the VMX transient structure.
487	*
488	* @returns VBox status code.
489	* @param pVmxTransient Pointer to the VMX transient structure.
490	*
491	* @remarks No-long-jump zone!!!
492	*/
493	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
494	{
495	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
496	AssertRCReturn(rc, rc);
497	return VINF_SUCCESS;
498	}
499
500
501	/**
502	* Reads the VM-entry exception error code field from the VMCS into
503	* the VMX transient structure.
504	*
505	* @returns VBox status code.
506	* @param pVCpu Pointer to the VMCPU.
507	* @param pVmxTransient Pointer to the VMX transient structure.
508	*
509	* @remarks No-long-jump zone!!!
510	*/
511	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
512	{
513	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
514	AssertRCReturn(rc, rc);
515	return VINF_SUCCESS;
516	}
517
518
519	/**
520	* Reads the VM-exit interruption-information field from the VMCS into the VMX
521	* transient structure.
522	*
523	* @returns VBox status code.
524	* @param pVCpu Pointer to the VMCPU.
525	* @param pVmxTransient Pointer to the VMX transient structure.
526	*/
527	DECLINLINE(int) hmR0VmxReadExitIntrInfoVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
528	{
529	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
530	{
531	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntrInfo);
532	AssertRCReturn(rc, rc);
533	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
534	}
535	return VINF_SUCCESS;
536	}
537
538
539	/**
540	* Reads the VM-exit interruption error code from the VMCS into the VMX
541	* transient structure.
542	*
543	* @returns VBox status code.
544	* @param pVCpu Pointer to the VMCPU.
545	* @param pVmxTransient Pointer to the VMX transient structure.
546	*/
547	DECLINLINE(int) hmR0VmxReadExitIntrErrorCodeVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
548	{
549	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
550	{
551	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntrErrorCode);
552	AssertRCReturn(rc, rc);
553	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
554	}
555	return VINF_SUCCESS;
556	}
557
558
559	/**
560	* Reads the VM-exit instruction length field from the VMCS into the VMX
561	* transient structure.
562	*
563	* @returns VBox status code.
564	* @param pVCpu Pointer to the VMCPU.
565	* @param pVmxTransient Pointer to the VMX transient structure.
566	*/
567	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
568	{
569	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
570	{
571	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
572	AssertRCReturn(rc, rc);
573	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
574	}
575	return VINF_SUCCESS;
576	}
577
578
579	/**
580	* Reads the exit qualification from the VMCS into the VMX transient structure.
581	*
582	* @returns VBox status code.
583	* @param pVCpu Pointer to the VMCPU.
584	* @param pVmxTransient Pointer to the VMX transient structure.
585	*/
586	DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
587	{
588	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
589	{
590	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification);
591	AssertRCReturn(rc, rc);
592	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
593	}
594	return VINF_SUCCESS;
595	}
596
597
598	/**
599	* Reads the IDT-vectoring information field from the VMCS into the VMX
600	* transient structure.
601	*
602	* @returns VBox status code.
603	* @param pVmxTransient Pointer to the VMX transient structure.
604	*
605	* @remarks No-long-jump zone!!!
606	*/
607	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
608	{
609	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
610	{
611	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
612	AssertRCReturn(rc, rc);
613	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
614	}
615	return VINF_SUCCESS;
616	}
617
618
619	/**
620	* Reads the IDT-vectoring error code from the VMCS into the VMX
621	* transient structure.
622	*
623	* @returns VBox status code.
624	* @param pVmxTransient Pointer to the VMX transient structure.
625	*/
626	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
627	{
628	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
629	{
630	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
631	AssertRCReturn(rc, rc);
632	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
633	}
634	return VINF_SUCCESS;
635	}
636
637
638	/**
639	* Enters VMX root mode operation on the current CPU.
640	*
641	* @returns VBox status code.
642	* @param pVM Pointer to the VM (optional, can be NULL, after
643	* a resume).
644	* @param HCPhysCpuPage Physical address of the VMXON region.
645	* @param pvCpuPage Pointer to the VMXON region.
646	*/
647	static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
648	{
649	AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
650	AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
651	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
652
653	if (pVM)
654	{
655	/* Write the VMCS revision dword to the VMXON region. */
656	(uint32_t )pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.msr.vmx_basic_info);
657	}
658
659	/* Enable the VMX bit in CR4 if necessary. */
660	RTCCUINTREG uCr4 = ASMGetCR4();
661	if (!(uCr4 & X86_CR4_VMXE))
662	ASMSetCR4(uCr4 \| X86_CR4_VMXE);
663
664	/* Enter VMX root mode. */
665	int rc = VMXEnable(HCPhysCpuPage);
666	if (RT_FAILURE(rc))
667	ASMSetCR4(uCr4);
668
669	return rc;
670	}
671
672
673	/**
674	* Exits VMX root mode operation on the current CPU.
675	*
676	* @returns VBox status code.
677	*/
678	static int hmR0VmxLeaveRootMode(void)
679	{
680	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
681
682	/* If we're for some reason not in VMX root mode, then don't leave it. */
683	RTCCUINTREG uHostCR4 = ASMGetCR4();
684	if (uHostCR4 & X86_CR4_VMXE)
685	{
686	/* Exit VMX root mode and clear the VMX bit in CR4. */
687	VMXDisable();
688	ASMSetCR4(uHostCR4 & ~X86_CR4_VMXE);
689	return VINF_SUCCESS;
690	}
691
692	return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
693	}
694
695
696	/**
697	* Allocates and maps one physically contiguous page. The allocated page is
698	* zero'd out. (Used by various VT-x structures).
699	*
700	* @returns IPRT status code.
701	* @param pMemObj Pointer to the ring-0 memory object.
702	* @param ppVirt Where to store the virtual address of the
703	* allocation.
704	* @param pPhys Where to store the physical address of the
705	* allocation.
706	*/
707	DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
708	{
709	AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
710	AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
711	AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
712
713	int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
714	if (RT_FAILURE(rc))
715	return rc;
716	ppVirt = RTR0MemObjAddress(pMemObj);
717	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
718	ASMMemZero32(*ppVirt, PAGE_SIZE);
719	return VINF_SUCCESS;
720	}
721
722
723	/**
724	* Frees and unmaps an allocated physical page.
725	*
726	* @param pMemObj Pointer to the ring-0 memory object.
727	* @param ppVirt Where to re-initialize the virtual address of
728	* allocation as 0.
729	* @param pHCPhys Where to re-initialize the physical address of the
730	* allocation as 0.
731	*/
732	DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
733	{
734	AssertPtr(pMemObj);
735	AssertPtr(ppVirt);
736	AssertPtr(pHCPhys);
737	if (*pMemObj != NIL_RTR0MEMOBJ)
738	{
739	int rc = RTR0MemObjFree(pMemObj, true / fFreeMappings */);
740	AssertRC(rc);
741	*pMemObj = NIL_RTR0MEMOBJ;
742	*ppVirt = 0;
743	*pHCPhys = 0;
744	}
745	}
746
747
748	/**
749	* Worker function to free VT-x related structures.
750	*
751	* @returns IPRT status code.
752	* @param pVM Pointer to the VM.
753	*/
754	static void hmR0VmxStructsFree(PVM pVM)
755	{
756	for (VMCPUID i = 0; i < pVM->cCpus; i++)
757	{
758	PVMCPU pVCpu = &pVM->aCpus[i];
759	AssertPtr(pVCpu);
760
761	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
762	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
763	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
764	#endif
765
766	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
767	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
768
769	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
770	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
771	}
772
773	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
774	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
775	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
776	#endif
777	}
778
779
780	/**
781	* Worker function to allocate VT-x related VM structures.
782	*
783	* @returns IPRT status code.
784	* @param pVM Pointer to the VM.
785	*/
786	static int hmR0VmxStructsAlloc(PVM pVM)
787	{
788	/*
789	* Initialize members up-front so we can cleanup properly on allocation failure.
790	*/
791	#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
792	pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
793	pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
794	pVM->hm.s.vmx.HCPhys##a_Name = 0;
795
796	#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
797	pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
798	pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
799	pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
800
801	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
802	VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
803	#endif
804	VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
805
806	AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
807	for (VMCPUID i = 0; i < pVM->cCpus; i++)
808	{
809	PVMCPU pVCpu = &pVM->aCpus[i];
810	VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
811	VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
812	VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
813	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
814	VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
815	VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
816	#endif
817	}
818	#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
819	#undef VMXLOCAL_INIT_VM_MEMOBJ
820
821	/*
822	* Allocate all the VT-x structures.
823	*/
824	int rc = VINF_SUCCESS;
825	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
826	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
827	if (RT_FAILURE(rc))
828	goto cleanup;
829	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
830	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
831	#endif
832
833	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
834	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
835	{
836	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
837	&pVM->hm.s.vmx.HCPhysApicAccess);
838	if (RT_FAILURE(rc))
839	goto cleanup;
840	}
841
842	/*
843	* Initialize per-VCPU VT-x structures.
844	*/
845	for (VMCPUID i = 0; i < pVM->cCpus; i++)
846	{
847	PVMCPU pVCpu = &pVM->aCpus[i];
848	AssertPtr(pVCpu);
849
850	/* Allocate the VM control structure (VMCS). */
851	AssertReturn(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.msr.vmx_basic_info) <= PAGE_SIZE, VERR_INTERNAL_ERROR);
852	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
853	if (RT_FAILURE(rc))
854	goto cleanup;
855
856	/* Allocate the Virtual-APIC page for transparent TPR accesses. */
857	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
858	{
859	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
860	&pVCpu->hm.s.vmx.HCPhysVirtApic);
861	if (RT_FAILURE(rc))
862	goto cleanup;
863	}
864
865	/* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for transparent accesses of specific MSRs. */
866	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
867	{
868	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
869	&pVCpu->hm.s.vmx.HCPhysMsrBitmap);
870	if (RT_FAILURE(rc))
871	goto cleanup;
872	memset(pVCpu->hm.s.vmx.pvMsrBitmap, 0xff, PAGE_SIZE);
873	}
874
875	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
876	/* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
877	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
878	if (RT_FAILURE(rc))
879	goto cleanup;
880
881	/* Allocate the VM-exit MSR-load page for the host MSRs. */
882	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
883	if (RT_FAILURE(rc))
884	goto cleanup;
885	#endif
886	}
887
888	return VINF_SUCCESS;
889
890	cleanup:
891	hmR0VmxStructsFree(pVM);
892	return rc;
893	}
894
895
896	/**
897	* Does global VT-x initialization (called during module initialization).
898	*
899	* @returns VBox status code.
900	*/
901	VMMR0DECL(int) VMXR0GlobalInit(void)
902	{
903	#ifdef HMVMX_USE_FUNCTION_TABLE
904	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
905	# ifdef VBOX_STRICT
906	for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
907	Assert(g_apfnVMExitHandlers[i]);
908	# endif
909	#endif
910	return VINF_SUCCESS;
911	}
912
913
914	/**
915	* Does global VT-x termination (called during module termination).
916	*/
917	VMMR0DECL(void) VMXR0GlobalTerm()
918	{
919	/* Nothing to do currently. */
920	}
921
922
923	/**
924	* Sets up and activates VT-x on the current CPU.
925	*
926	* @returns VBox status code.
927	* @param pCpu Pointer to the global CPU info struct.
928	* @param pVM Pointer to the VM (can be NULL after a host resume
929	* operation).
930	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
931	* fEnabledByHost is true).
932	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
933	* @a fEnabledByHost is true).
934	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
935	* enable VT-x on the host.
936	*/
937	VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBLCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost)
938	{
939	AssertReturn(pCpu, VERR_INVALID_PARAMETER);
940	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
941
942	if (!fEnabledByHost)
943	{
944	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
945	if (RT_FAILURE(rc))
946	return rc;
947	}
948
949	/*
950	* Flush all EPTP tagged-TLB entries (in case any other hypervisor have been using EPTPs) so that
951	* we can avoid an explicit flush while using new VPIDs. We would still need to flush
952	* each time while reusing a VPID after hitting the MaxASID limit once.
953	*/
954	if ( pVM
955	&& pVM->hm.s.fNestedPaging)
956	{
957	/* We require ALL_CONTEXT flush-type to be available on the CPU. See hmR0VmxSetupTaggedTlb(). */
958	Assert(pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS);
959	hmR0VmxFlushEpt(pVM, NULL /* pVCpu */, VMX_FLUSH_EPT_ALL_CONTEXTS);
960	pCpu->fFlushAsidBeforeUse = false;
961	}
962	else
963	{
964	/** @todo This is still not perfect. If on host resume (pVM is NULL or a VM
965	* without Nested Paging triggered this function) we still have the risk
966	* of potentially running with stale TLB-entries from other hypervisors
967	* when later we use a VM with NestedPaging. To fix this properly we will
968	* have to pass '&g_HvmR0' (see HMR0.cpp) to this function and read
969	* 'vmx_ept_vpid_caps' from it. Sigh. */
970	pCpu->fFlushAsidBeforeUse = true;
971	}
972
973	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
974	++pCpu->cTlbFlushes;
975
976	return VINF_SUCCESS;
977	}
978
979
980	/**
981	* Deactivates VT-x on the current CPU.
982	*
983	* @returns VBox status code.
984	* @param pCpu Pointer to the global CPU info struct.
985	* @param pvCpuPage Pointer to the VMXON region.
986	* @param HCPhysCpuPage Physical address of the VMXON region.
987	*
988	* @remarks This function should never be called when SUPR0EnableVTx() or
989	* similar was used to enable VT-x on the host.
990	*/
991	VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBLCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
992	{
993	NOREF(pCpu);
994	NOREF(pvCpuPage);
995	NOREF(HCPhysCpuPage);
996
997	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
998	return hmR0VmxLeaveRootMode();
999	}
1000
1001
1002	/**
1003	* Sets the permission bits for the specified MSR in the MSR bitmap.
1004	*
1005	* @param pVCpu Pointer to the VMCPU.
1006	* @param uMSR The MSR value.
1007	* @param enmRead Whether reading this MSR causes a VM-exit.
1008	* @param enmWrite Whether writing this MSR causes a VM-exit.
1009	*/
1010	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1011	{
1012	int32_t iBit;
1013	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
1014
1015	/*
1016	* Layout:
1017	* 0x000 - 0x3ff - Low MSR read bits
1018	* 0x400 - 0x7ff - High MSR read bits
1019	* 0x800 - 0xbff - Low MSR write bits
1020	* 0xc00 - 0xfff - High MSR write bits
1021	*/
1022	if (uMsr <= 0x00001FFF)
1023	iBit = uMsr;
1024	else if ( uMsr >= 0xC0000000
1025	&& uMsr <= 0xC0001FFF)
1026	{
1027	iBit = (uMsr - 0xC0000000);
1028	pbMsrBitmap += 0x400;
1029	}
1030	else
1031	{
1032	AssertMsgFailed(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1033	return;
1034	}
1035
1036	Assert(iBit <= 0x1fff);
1037	if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1038	ASMBitSet(pbMsrBitmap, iBit);
1039	else
1040	ASMBitClear(pbMsrBitmap, iBit);
1041
1042	if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1043	ASMBitSet(pbMsrBitmap + 0x800, iBit);
1044	else
1045	ASMBitClear(pbMsrBitmap + 0x800, iBit);
1046	}
1047
1048
1049	/**
1050	* Flushes the TLB using EPT.
1051	*
1052	* @returns VBox status code.
1053	* @param pVM Pointer to the VM.
1054	* @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1055	* enmFlush).
1056	* @param enmFlush Type of flush.
1057	*/
1058	static void hmR0VmxFlushEpt(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush)
1059	{
1060	AssertPtr(pVM);
1061	Assert(pVM->hm.s.fNestedPaging);
1062
1063	uint64_t descriptor[2];
1064	if (enmFlush == VMX_FLUSH_EPT_ALL_CONTEXTS)
1065	descriptor[0] = 0;
1066	else
1067	{
1068	Assert(pVCpu);
1069	descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1070	}
1071	descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1072
1073	int rc = VMXR0InvEPT(enmFlush, &descriptor[0]);
1074	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1075	rc));
1076	if ( RT_SUCCESS(rc)
1077	&& pVCpu)
1078	{
1079	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1080	}
1081	}
1082
1083
1084	/**
1085	* Flushes the TLB using VPID.
1086	*
1087	* @returns VBox status code.
1088	* @param pVM Pointer to the VM.
1089	* @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1090	* enmFlush).
1091	* @param enmFlush Type of flush.
1092	* @param GCPtr Virtual address of the page to flush (can be 0 depending
1093	* on @a enmFlush).
1094	*/
1095	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr)
1096	{
1097	AssertPtr(pVM);
1098	Assert(pVM->hm.s.vmx.fVpid);
1099
1100	uint64_t descriptor[2];
1101	if (enmFlush == VMX_FLUSH_VPID_ALL_CONTEXTS)
1102	{
1103	descriptor[0] = 0;
1104	descriptor[1] = 0;
1105	}
1106	else
1107	{
1108	AssertPtr(pVCpu);
1109	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1110	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1111	descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1112	descriptor[1] = GCPtr;
1113	}
1114
1115	int rc = VMXR0InvVPID(enmFlush, &descriptor[0]); NOREF(rc);
1116	AssertMsg(rc == VINF_SUCCESS,
1117	("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1118	if ( RT_SUCCESS(rc)
1119	&& pVCpu)
1120	{
1121	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1122	}
1123	}
1124
1125
1126	/**
1127	* Invalidates a guest page by guest virtual address. Only relevant for
1128	* EPT/VPID, otherwise there is nothing really to invalidate.
1129	*
1130	* @returns VBox status code.
1131	* @param pVM Pointer to the VM.
1132	* @param pVCpu Pointer to the VMCPU.
1133	* @param GCVirt Guest virtual address of the page to invalidate.
1134	*/
1135	VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1136	{
1137	AssertPtr(pVM);
1138	AssertPtr(pVCpu);
1139	LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1140
1141	bool fFlushPending = VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1142	if (!fFlushPending)
1143	{
1144	/*
1145	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1146	* See @bugref{6043} and @bugref{6177}.
1147	*
1148	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1149	* function maybe called in a loop with individual addresses.
1150	*/
1151	if (pVM->hm.s.vmx.fVpid)
1152	{
1153	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1154	{
1155	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, GCVirt);
1156	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1157	}
1158	else
1159	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1160	}
1161	else if (pVM->hm.s.fNestedPaging)
1162	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1163	}
1164
1165	return VINF_SUCCESS;
1166	}
1167
1168
1169	/**
1170	* Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1171	* otherwise there is nothing really to invalidate.
1172	*
1173	* @returns VBox status code.
1174	* @param pVM Pointer to the VM.
1175	* @param pVCpu Pointer to the VMCPU.
1176	* @param GCPhys Guest physical address of the page to invalidate.
1177	*/
1178	VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1179	{
1180	LogFlowFunc(("%RGp\n", GCPhys));
1181
1182	/*
1183	* We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1184	* by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1185	* This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1186	*/
1187	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1188	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1189	return VINF_SUCCESS;
1190	}
1191
1192
1193	/**
1194	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1195	* case where neither EPT nor VPID is supported by the CPU.
1196	*
1197	* @param pVM Pointer to the VM.
1198	* @param pVCpu Pointer to the VMCPU.
1199	*
1200	* @remarks Called with interrupts disabled.
1201	*/
1202	static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu)
1203	{
1204	NOREF(pVM);
1205	AssertPtr(pVCpu);
1206	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1207	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1208
1209	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1210	AssertPtr(pCpu);
1211
1212	pVCpu->hm.s.TlbShootdown.cPages = 0;
1213	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1214	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1215	pVCpu->hm.s.fForceTLBFlush = false;
1216	return;
1217	}
1218
1219
1220	/**
1221	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1222	*
1223	* @param pVM Pointer to the VM.
1224	* @param pVCpu Pointer to the VMCPU.
1225	* @remarks All references to "ASID" in this function pertains to "VPID" in
1226	* Intel's nomenclature. The reason is, to avoid confusion in compare
1227	* statements since the host-CPU copies are named "ASID".
1228	*
1229	* @remarks Called with interrupts disabled.
1230	*/
1231	static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu)
1232	{
1233	#ifdef VBOX_WITH_STATISTICS
1234	bool fTlbFlushed = false;
1235	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1236	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1237	if (!fTlbFlushed) \
1238	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1239	} while (0)
1240	#else
1241	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1242	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1243	#endif
1244
1245	AssertPtr(pVM);
1246	AssertPtr(pVCpu);
1247	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1248	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1249	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1250
1251	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1252	AssertPtr(pCpu);
1253
1254	/*
1255	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1256	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1257	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1258	*/
1259	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1260	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1261	{
1262	++pCpu->uCurrentAsid;
1263	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1264	{
1265	pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1266	pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1267	pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1268	}
1269
1270	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1271	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1272	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1273
1274	/*
1275	* Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1276	* invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1277	*/
1278	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1279	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1280	HMVMX_SET_TAGGED_TLB_FLUSHED();
1281	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1282	}
1283
1284	/* Check for explicit TLB shootdowns. */
1285	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1286	{
1287	/*
1288	* Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
1289	* guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
1290	* Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
1291	* but not guest-physical mappings.
1292	* See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
1293	*/
1294	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1295	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1296	HMVMX_SET_TAGGED_TLB_FLUSHED();
1297	}
1298
1299	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1300	* not be executed. See hmQueueInvlPage() where it is commented
1301	* out. Support individual entry flushing someday. */
1302	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1303	{
1304	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1305
1306	/*
1307	* Flush individual guest entries using VPID from the TLB or as little as possible with EPT
1308	* as supported by the CPU.
1309	*/
1310	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1311	{
1312	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1313	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1314	}
1315	else
1316	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1317
1318	HMVMX_SET_TAGGED_TLB_FLUSHED();
1319	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1320	}
1321
1322	pVCpu->hm.s.TlbShootdown.cPages = 0;
1323	pVCpu->hm.s.fForceTLBFlush = false;
1324
1325	HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
1326
1327	Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
1328	Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
1329	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1330	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1331	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1332	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1333	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1334	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1335
1336	/* Update VMCS with the VPID. */
1337	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1338	AssertRC(rc);
1339
1340	#undef HMVMX_SET_TAGGED_TLB_FLUSHED
1341	}
1342
1343
1344	/**
1345	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
1346	*
1347	* @returns VBox status code.
1348	* @param pVM Pointer to the VM.
1349	* @param pVCpu Pointer to the VMCPU.
1350	*
1351	* @remarks Called with interrupts disabled.
1352	*/
1353	static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu)
1354	{
1355	AssertPtr(pVM);
1356	AssertPtr(pVCpu);
1357	AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
1358	AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
1359
1360	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1361	AssertPtr(pCpu);
1362
1363	/*
1364	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1365	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
1366	*/
1367	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1368	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1369	{
1370	pVCpu->hm.s.fForceTLBFlush = true;
1371	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1372	}
1373
1374	/* Check for explicit TLB shootdown flushes. */
1375	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1376	{
1377	pVCpu->hm.s.fForceTLBFlush = true;
1378	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1379	}
1380
1381	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1382	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1383
1384	if (pVCpu->hm.s.fForceTLBFlush)
1385	{
1386	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1387	pVCpu->hm.s.fForceTLBFlush = false;
1388	}
1389	else
1390	{
1391	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1392	* not be executed. See hmQueueInvlPage() where it is commented
1393	* out. Support individual entry flushing someday. */
1394	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1395	{
1396	/* We cannot flush individual entries without VPID support. Flush using EPT. */
1397	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1398	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1399	}
1400	else
1401	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1402	}
1403
1404	pVCpu->hm.s.TlbShootdown.cPages = 0;
1405	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1406	}
1407
1408
1409	/**
1410	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
1411	*
1412	* @returns VBox status code.
1413	* @param pVM Pointer to the VM.
1414	* @param pVCpu Pointer to the VMCPU.
1415	*
1416	* @remarks Called with interrupts disabled.
1417	*/
1418	static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu)
1419	{
1420	AssertPtr(pVM);
1421	AssertPtr(pVCpu);
1422	AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
1423	AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
1424
1425	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1426
1427	/*
1428	* Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
1429	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1430	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1431	*/
1432	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1433	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1434	{
1435	pVCpu->hm.s.fForceTLBFlush = true;
1436	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1437	}
1438
1439	/* Check for explicit TLB shootdown flushes. */
1440	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1441	{
1442	/*
1443	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
1444	* we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
1445	* pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
1446	*/
1447	pVCpu->hm.s.fForceTLBFlush = true;
1448	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1449	}
1450
1451	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1452	if (pVCpu->hm.s.fForceTLBFlush)
1453	{
1454	++pCpu->uCurrentAsid;
1455	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1456	{
1457	pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
1458	pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1459	pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1460	}
1461
1462	pVCpu->hm.s.fForceTLBFlush = false;
1463	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1464	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1465	if (pCpu->fFlushAsidBeforeUse)
1466	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1467	}
1468	else
1469	{
1470	AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
1471	("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
1472	pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
1473	pCpu->uCurrentAsid, pCpu->cTlbFlushes));
1474
1475	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1476	* not be executed. See hmQueueInvlPage() where it is commented
1477	* out. Support individual entry flushing someday. */
1478	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1479	{
1480	/* Flush individual guest entries using VPID or as little as possible with EPT as supported by the CPU. */
1481	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1482	{
1483	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1484	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1485	}
1486	else
1487	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1488	}
1489	else
1490	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1491	}
1492
1493	pVCpu->hm.s.TlbShootdown.cPages = 0;
1494	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1495
1496	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1497	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1498	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1499	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1500	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1501	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1502
1503	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1504	AssertRC(rc);
1505	}
1506
1507
1508	/**
1509	* Flushes the guest TLB entry based on CPU capabilities.
1510	*
1511	* @param pVCpu Pointer to the VMCPU.
1512	*/
1513	DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu)
1514	{
1515	PVM pVM = pVCpu->CTX_SUFF(pVM);
1516	switch (pVM->hm.s.vmx.uFlushTaggedTlb)
1517	{
1518	case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu); break;
1519	case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu); break;
1520	case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu); break;
1521	case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu); break;
1522	default:
1523	AssertMsgFailed(("Invalid flush-tag function identifier\n"));
1524	break;
1525	}
1526	}
1527
1528
1529	/**
1530	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
1531	* TLB entries from the host TLB before VM-entry.
1532	*
1533	* @returns VBox status code.
1534	* @param pVM Pointer to the VM.
1535	*/
1536	static int hmR0VmxSetupTaggedTlb(PVM pVM)
1537	{
1538	/*
1539	* Determine optimal flush type for Nested Paging.
1540	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
1541	* guest execution (see hmR3InitFinalizeR0()).
1542	*/
1543	if (pVM->hm.s.fNestedPaging)
1544	{
1545	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
1546	{
1547	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
1548	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_SINGLE_CONTEXT;
1549	else if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1550	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_ALL_CONTEXTS;
1551	else
1552	{
1553	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
1554	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1555	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1556	}
1557
1558	/* Make sure the write-back cacheable memory type for EPT is supported. */
1559	if (!(pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB))
1560	{
1561	LogRel(("hmR0VmxSetupTaggedTlb: Unsupported EPTP memory type %#x.\n", pVM->hm.s.vmx.msr.vmx_ept_vpid_caps));
1562	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1563	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1564	}
1565	}
1566	else
1567	{
1568	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
1569	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1570	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1571	}
1572	}
1573
1574	/*
1575	* Determine optimal flush type for VPID.
1576	*/
1577	if (pVM->hm.s.vmx.fVpid)
1578	{
1579	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
1580	{
1581	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
1582	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_SINGLE_CONTEXT;
1583	else if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
1584	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_ALL_CONTEXTS;
1585	else
1586	{
1587	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
1588	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1589	LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
1590	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
1591	LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
1592	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1593	pVM->hm.s.vmx.fVpid = false;
1594	}
1595	}
1596	else
1597	{
1598	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
1599	Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
1600	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1601	pVM->hm.s.vmx.fVpid = false;
1602	}
1603	}
1604
1605	/*
1606	* Setup the handler for flushing tagged-TLBs.
1607	*/
1608	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
1609	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
1610	else if (pVM->hm.s.fNestedPaging)
1611	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
1612	else if (pVM->hm.s.vmx.fVpid)
1613	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
1614	else
1615	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
1616	return VINF_SUCCESS;
1617	}
1618
1619
1620	/**
1621	* Sets up pin-based VM-execution controls in the VMCS.
1622	*
1623	* @returns VBox status code.
1624	* @param pVM Pointer to the VM.
1625	* @param pVCpu Pointer to the VMCPU.
1626	*/
1627	static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
1628	{
1629	AssertPtr(pVM);
1630	AssertPtr(pVCpu);
1631
1632	uint32_t val = pVM->hm.s.vmx.msr.vmx_pin_ctls.n.disallowed0; /* Bits set here must always be set. */
1633	uint32_t zap = pVM->hm.s.vmx.msr.vmx_pin_ctls.n.allowed1; /* Bits cleared here must always be cleared. */
1634
1635	val \|= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts causes a VM-exits. */
1636	\| VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts causes a VM-exit. */
1637	Assert(!(val & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI));
1638
1639	/* Enable the VMX preemption timer. */
1640	if (pVM->hm.s.vmx.fUsePreemptTimer)
1641	{
1642	Assert(pVM->hm.s.vmx.msr.vmx_pin_ctls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
1643	val \|= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
1644	}
1645
1646	if ((val & zap) != val)
1647	{
1648	LogRel(("hmR0VmxSetupPinCtls: invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1649	pVM->hm.s.vmx.msr.vmx_pin_ctls.n.disallowed0, val, zap));
1650	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1651	}
1652
1653	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
1654	AssertRCReturn(rc, rc);
1655
1656	/* Update VCPU with the currently set pin-based VM-execution controls. */
1657	pVCpu->hm.s.vmx.u32PinCtls = val;
1658	return rc;
1659	}
1660
1661
1662	/**
1663	* Sets up processor-based VM-execution controls in the VMCS.
1664	*
1665	* @returns VBox status code.
1666	* @param pVM Pointer to the VM.
1667	* @param pVMCPU Pointer to the VMCPU.
1668	*/
1669	static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
1670	{
1671	AssertPtr(pVM);
1672	AssertPtr(pVCpu);
1673
1674	int rc = VERR_INTERNAL_ERROR_5;
1675	uint32_t val = pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0; /* Bits set here must be set in the VMCS. */
1676	uint32_t zap = pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1677
1678	val \|= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
1679	\| VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
1680	\| VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
1681	\| VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
1682	\| VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
1683	\| VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
1684	\| VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
1685
1686	/* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
1687	if ( !(pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
1688	\|\| (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
1689	{
1690	LogRel(("hmR0VmxSetupProcCtls: unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
1691	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1692	}
1693
1694	/* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
1695	if (!pVM->hm.s.fNestedPaging)
1696	{
1697	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
1698	val \|= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
1699	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
1700	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
1701	}
1702
1703	/* Use TPR shadowing if supported by the CPU. */
1704	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
1705	{
1706	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
1707	Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
1708	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
1709	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
1710	AssertRCReturn(rc, rc);
1711
1712	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
1713	/* CR8 writes causes a VM-exit based on TPR threshold. */
1714	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
1715	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
1716	}
1717	else
1718	{
1719	val \|= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads causes a VM-exit. */
1720	\| VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes causes a VM-exit. */
1721	}
1722
1723	/* Use MSR-bitmaps if supported by the CPU. */
1724	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1725	{
1726	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
1727
1728	Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1729	Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
1730	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1731	AssertRCReturn(rc, rc);
1732
1733	/*
1734	* The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
1735	* automatically (either as part of the MSR-load/store areas or dedicated fields in the VMCS).
1736	*/
1737	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1738	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1739	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1740	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1741	hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1742	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1743	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1744	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1745	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1746	}
1747
1748	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
1749	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1750	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
1751
1752	if ((val & zap) != val)
1753	{
1754	LogRel(("hmR0VmxSetupProcCtls: invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1755	pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0, val, zap));
1756	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1757	}
1758
1759	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
1760	AssertRCReturn(rc, rc);
1761
1762	/* Update VCPU with the currently set processor-based VM-execution controls. */
1763	pVCpu->hm.s.vmx.u32ProcCtls = val;
1764
1765	/*
1766	* Secondary processor-based VM-execution controls.
1767	*/
1768	if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
1769	{
1770	val = pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
1771	zap = pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1772
1773	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
1774	val \|= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
1775
1776	if (pVM->hm.s.fNestedPaging)
1777	val \|= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
1778	else
1779	{
1780	/*
1781	* Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
1782	* VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT when INVPCID is executed by the guest.
1783	* See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
1784	*/
1785	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
1786	val \|= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
1787	}
1788
1789	if (pVM->hm.s.vmx.fVpid)
1790	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
1791
1792	if (pVM->hm.s.vmx.fUnrestrictedGuest)
1793	val \|= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
1794
1795	/* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
1796	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
1797	* done dynamically. */
1798	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
1799	{
1800	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
1801	Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
1802	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
1803	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
1804	AssertRCReturn(rc, rc);
1805	}
1806
1807	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
1808	{
1809	val \|= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
1810	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1811	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_TSC_AUX, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1812	}
1813
1814	if ((val & zap) != val)
1815	{
1816	LogRel(("hmR0VmxSetupProcCtls: invalid secondary processor-based VM-execution controls combo! "
1817	"cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0, val, zap));
1818	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1819	}
1820
1821	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
1822	AssertRCReturn(rc, rc);
1823
1824	/* Update VCPU with the currently set secondary processor-based VM-execution controls. */
1825	pVCpu->hm.s.vmx.u32ProcCtls2 = val;
1826	}
1827
1828	return VINF_SUCCESS;
1829	}
1830
1831
1832	/**
1833	* Sets up miscellaneous (everything other than Pin & Processor-based
1834	* VM-execution) control fields in the VMCS.
1835	*
1836	* @returns VBox status code.
1837	* @param pVM Pointer to the VM.
1838	* @param pVCpu Pointer to the VMCPU.
1839	*/
1840	static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
1841	{
1842	AssertPtr(pVM);
1843	AssertPtr(pVCpu);
1844
1845	int rc = VERR_GENERAL_FAILURE;
1846
1847	/* All fields are zero-initialized during allocation; but don't remove the commented block below. */
1848	#if 0
1849	/* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestControlRegs())*/
1850	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0); AssertRCReturn(rc, rc);
1851	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0); AssertRCReturn(rc, rc);
1852
1853	/*
1854	* Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
1855	* and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
1856	* We thus use the exception bitmap to control it rather than use both.
1857	*/
1858	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0); AssertRCReturn(rc, rc);
1859	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0); AssertRCReturn(rc, rc);
1860
1861	/** @todo Explore possibility of using IO-bitmaps. */
1862	/* All IO & IOIO instructions cause VM-exits. */
1863	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0); AssertRCReturn(rc, rc);
1864	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0); AssertRCReturn(rc, rc);
1865
1866	/* Initialize the MSR-bitmap area. */
1867	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
1868	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0); AssertRCReturn(rc, rc);
1869	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
1870	#endif
1871
1872	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
1873	/* Setup MSR autoloading/storing. */
1874	Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
1875	Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
1876	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1877	AssertRCReturn(rc, rc);
1878	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1879	AssertRCReturn(rc, rc);
1880
1881	Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
1882	Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
1883	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
1884	AssertRCReturn(rc, rc);
1885	#endif
1886
1887	/* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
1888	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
1889	AssertRCReturn(rc, rc);
1890
1891	/* All fields are zero-initialized during allocation; but don't remove the commented block below. */
1892	#if 0
1893	/* Setup debug controls */
1894	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
1895	AssertRCReturn(rc, rc);
1896	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
1897	AssertRCReturn(rc, rc);
1898	#endif
1899
1900	return rc;
1901	}
1902
1903
1904	/**
1905	* Sets up the initial exception bitmap in the VMCS based on static conditions
1906	* (i.e. conditions that cannot ever change at runtime).
1907	*
1908	* @returns VBox status code.
1909	* @param pVM Pointer to the VM.
1910	* @param pVCpu Pointer to the VMCPU.
1911	*/
1912	static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
1913	{
1914	AssertPtr(pVM);
1915	AssertPtr(pVCpu);
1916
1917	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
1918
1919	uint32_t u32XcptBitmap = 0;
1920
1921	/* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
1922	if (!pVM->hm.s.fNestedPaging)
1923	u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
1924
1925	pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
1926	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
1927	AssertRCReturn(rc, rc);
1928	return rc;
1929	}
1930
1931
1932	/**
1933	* Sets up the initial guest-state mask. The guest-state mask is consulted
1934	* before reading guest-state fields from the VMCS as VMREADs can be expensive
1935	* for the nested virtualization case (as it would cause a VM-exit).
1936	*
1937	* @param pVCpu Pointer to the VMCPU.
1938	*/
1939	static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
1940	{
1941	/* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
1942	pVCpu->hm.s.vmx.fUpdatedGuestState = HMVMX_UPDATED_GUEST_ALL;
1943	return VINF_SUCCESS;
1944	}
1945
1946
1947	/**
1948	* Does per-VM VT-x initialization.
1949	*
1950	* @returns VBox status code.
1951	* @param pVM Pointer to the VM.
1952	*/
1953	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
1954	{
1955	LogFlowFunc(("pVM=%p\n", pVM));
1956
1957	int rc = hmR0VmxStructsAlloc(pVM);
1958	if (RT_FAILURE(rc))
1959	{
1960	LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
1961	return rc;
1962	}
1963
1964	return VINF_SUCCESS;
1965	}
1966
1967
1968	/**
1969	* Does per-VM VT-x termination.
1970	*
1971	* @returns VBox status code.
1972	* @param pVM Pointer to the VM.
1973	*/
1974	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
1975	{
1976	LogFlowFunc(("pVM=%p\n", pVM));
1977
1978	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1979	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
1980	ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
1981	#endif
1982	hmR0VmxStructsFree(pVM);
1983	return VINF_SUCCESS;
1984	}
1985
1986
1987	/**
1988	* Sets up the VM for execution under VT-x.
1989	* This function is only called once per-VM during initalization.
1990	*
1991	* @returns VBox status code.
1992	* @param pVM Pointer to the VM.
1993	*/
1994	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
1995	{
1996	AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
1997	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1998
1999	LogFlowFunc(("pVM=%p\n", pVM));
2000
2001	/*
2002	* Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable must always be allocated.
2003	* We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0().
2004	*/
2005	/* -XXX- change hmR3InitFinalizeR0Intel() to fail if pRealModeTSS alloc fails. */
2006	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2007	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2008	\|\| !pVM->hm.s.vmx.pRealModeTSS))
2009	{
2010	LogRel(("VMXR0SetupVM: invalid real-on-v86 state.\n"));
2011	return VERR_INTERNAL_ERROR;
2012	}
2013
2014	/* Initialize these always, see hmR3InitFinalizeR0().*/
2015	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NONE;
2016	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NONE;
2017
2018	/* Setup the tagged-TLB flush handlers. */
2019	int rc = hmR0VmxSetupTaggedTlb(pVM);
2020	if (RT_FAILURE(rc))
2021	{
2022	LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2023	return rc;
2024	}
2025
2026	for (VMCPUID i = 0; i < pVM->cCpus; i++)
2027	{
2028	PVMCPU pVCpu = &pVM->aCpus[i];
2029	AssertPtr(pVCpu);
2030	AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2031
2032	/* Set revision dword at the beginning of the VMCS structure. */
2033	(uint32_t )pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.msr.vmx_basic_info);
2034
2035	/* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2036	rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
2037	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVMCS failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2038	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2039
2040	/* Load this VMCS as the current VMCS. */
2041	rc = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
2042	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVMCS failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2043	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2044
2045	rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2046	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2047	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2048
2049	rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2050	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2051	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2052
2053	rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2054	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2055	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2056
2057	rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2058	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2059	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2060
2061	rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2062	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2063	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2064
2065	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2066	rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2067	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2068	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2069	#endif
2070
2071	/* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2072	rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
2073	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVMCS(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2074	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2075
2076	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2077	}
2078
2079	return VINF_SUCCESS;
2080	}
2081
2082
2083	/**
2084	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2085	* the VMCS.
2086	*
2087	* @returns VBox status code.
2088	* @param pVM Pointer to the VM.
2089	* @param pVCpu Pointer to the VMCPU.
2090	*/
2091	DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2092	{
2093	RTCCUINTREG uReg = ASMGetCR0();
2094	int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2095	AssertRCReturn(rc, rc);
2096
2097	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2098	/* For the darwin 32-bit hybrid kernel, we need the 64-bit CR3 as it uses 64-bit paging. */
2099	if (HMVMX_IS_64BIT_HOST_MODE())
2100	{
2101	uint64_t uRegCR3 = HMR0Get64bitCR3();
2102	rc = VMXWriteVmcs64(VMX_VMCS_HOST_CR3, uRegCR3);
2103	}
2104	else
2105	#endif
2106	{
2107	uReg = ASMGetCR3();
2108	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2109	}
2110	AssertRCReturn(rc, rc);
2111
2112	uReg = ASMGetCR4();
2113	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2114	AssertRCReturn(rc, rc);
2115	return rc;
2116	}
2117
2118
2119	/**
2120	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2121	* the host-state area in the VMCS.
2122	*
2123	* @returns VBox status code.
2124	* @param pVM Pointer to the VM.
2125	* @param pVCpu Pointer to the VMCPU.
2126	*/
2127	DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2128	{
2129	int rc = VERR_INTERNAL_ERROR_5;
2130	RTSEL uSelDS = 0;
2131	RTSEL uSelES = 0;
2132	RTSEL uSelFS = 0;
2133	RTSEL uSelGS = 0;
2134	RTSEL uSelTR = 0;
2135
2136	/*
2137	* Host DS, ES, FS and GS segment registers.
2138	*/
2139	#if HC_ARCH_BITS == 64
2140	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2141	uSelDS = ASMGetDS();
2142	uSelES = ASMGetES();
2143	uSelFS = ASMGetFS();
2144	uSelGS = ASMGetGS();
2145	#endif
2146
2147	/*
2148	* Host CS and SS segment registers.
2149	*/
2150	RTSEL uSelCS;
2151	RTSEL uSelSS;
2152	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2153	if (HMVMX_IS_64BIT_HOST_MODE())
2154	{
2155	uSelCS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
2156	uSelSS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
2157	}
2158	else
2159	{
2160	/* Seems darwin uses the LDT (TI flag is set) in the CS & SS selectors which VT-x doesn't like. */
2161	uSelCS = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
2162	uSelSS = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
2163	}
2164	#else
2165	uSelCS = ASMGetCS();
2166	uSelSS = ASMGetSS();
2167	#endif
2168
2169	/*
2170	* Host TR segment register.
2171	*/
2172	uSelTR = ASMGetTR();
2173
2174	#if HC_ARCH_BITS == 64
2175	/*
2176	* Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
2177	* before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2178	*/
2179	if (uSelDS & (X86_SEL_RPL \| X86_SEL_LDT))
2180	{
2181	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_DS;
2182	pVCpu->hm.s.vmx.RestoreHost.uHostSelDS = uSelDS;
2183	uSelDS = 0;
2184	}
2185	if (uSelES & (X86_SEL_RPL \| X86_SEL_LDT))
2186	{
2187	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_ES;
2188	pVCpu->hm.s.vmx.RestoreHost.uHostSelES = uSelES;
2189	uSelES = 0;
2190	}
2191	if (uSelFS & (X86_SEL_RPL \| X86_SEL_LDT))
2192	{
2193	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_FS;
2194	pVCpu->hm.s.vmx.RestoreHost.uHostSelFS = uSelFS;
2195	uSelFS = 0;
2196	}
2197	if (uSelGS & (X86_SEL_RPL \| X86_SEL_LDT))
2198	{
2199	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_GS;
2200	pVCpu->hm.s.vmx.RestoreHost.uHostSelGS = uSelGS;
2201	uSelGS = 0;
2202	}
2203	#endif
2204
2205	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2206	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2207	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2208	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2209	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2210	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2211	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2212	Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2213	Assert(uSelCS);
2214	Assert(uSelTR);
2215
2216	/* Assertion is right but we would not have updated u32ExitCtls yet. */
2217	#if 0
2218	if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2219	Assert(uSelSS != 0);
2220	#endif
2221
2222	/* Write these host selector fields into the host-state area in the VMCS. */
2223	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_CS, uSelCS); AssertRCReturn(rc, rc);
2224	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_SS, uSelSS); AssertRCReturn(rc, rc);
2225	#if HC_ARCH_BITS == 64
2226	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_DS, uSelDS); AssertRCReturn(rc, rc);
2227	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_ES, uSelES); AssertRCReturn(rc, rc);
2228	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_FS, uSelFS); AssertRCReturn(rc, rc);
2229	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_GS, uSelGS); AssertRCReturn(rc, rc);
2230	#endif
2231	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_TR, uSelTR); AssertRCReturn(rc, rc);
2232
2233	/*
2234	* Host GDTR and IDTR.
2235	*/
2236	RTGDTR Gdtr;
2237	RT_ZERO(Gdtr);
2238	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2239	if (HMVMX_IS_64BIT_HOST_MODE())
2240	{
2241	X86XDTR64 Gdtr64;
2242	X86XDTR64 Idtr64;
2243	HMR0Get64bitGdtrAndIdtr(&Gdtr64, &Idtr64);
2244	rc = VMXWriteVmcs64(VMX_VMCS_HOST_GDTR_BASE, Gdtr64.uAddr); AssertRCReturn(rc, rc);
2245	rc = VMXWriteVmcs64(VMX_VMCS_HOST_IDTR_BASE, Idtr64.uAddr); AssertRCReturn(rc, rc);
2246
2247	Gdtr.cbGdt = Gdtr64.cb;
2248	Gdtr.pGdt = (uintptr_t)Gdtr64.uAddr;
2249	}
2250	else
2251	#endif
2252	{
2253	RTIDTR Idtr;
2254	ASMGetGDTR(&Gdtr);
2255	ASMGetIDTR(&Idtr);
2256	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt); AssertRCReturn(rc, rc);
2257	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt); AssertRCReturn(rc, rc);
2258
2259	#if HC_ARCH_BITS == 64
2260	/*
2261	* Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
2262	* maximum limit (0xffff) on every VM-exit.
2263	*/
2264	if (Gdtr.cbGdt != 0xffff)
2265	{
2266	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDTR;
2267	AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
2268	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
2269	}
2270
2271	/*
2272	* IDT limit is practically 0xfff. Therefore if the host has the limit as 0xfff, VT-x bloating the limit to 0xffff
2273	* is not a problem as it's not possible to get at them anyway. See Intel spec. 6.14.1 "64-Bit Mode IDT" and
2274	* Intel spec. 6.2 "Exception and Interrupt Vectors".
2275	*/
2276	if (Idtr.cbIdt < 0x0fff)
2277	{
2278	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_IDTR;
2279	AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
2280	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
2281	}
2282	#endif
2283	}
2284
2285	/*
2286	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
2287	* is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
2288	*/
2289	if ((uSelTR & X86_SEL_MASK) > Gdtr.cbGdt)
2290	{
2291	AssertMsgFailed(("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt));
2292	return VERR_VMX_INVALID_HOST_STATE;
2293	}
2294
2295	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
2296	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2297	if (HMVMX_IS_64BIT_HOST_MODE())
2298	{
2299	/* We need the 64-bit TR base for hybrid darwin. */
2300	uint64_t u64TRBase = X86DESC64_BASE((PX86DESC64)pDesc);
2301	rc = VMXWriteVmcs64(VMX_VMCS_HOST_TR_BASE, u64TRBase);
2302	}
2303	else
2304	#endif
2305	{
2306	uintptr_t uTRBase;
2307	#if HC_ARCH_BITS == 64
2308	uTRBase = X86DESC64_BASE(pDesc);
2309	#else
2310	uTRBase = X86DESC_BASE(pDesc);
2311	#endif
2312	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
2313	}
2314	AssertRCReturn(rc, rc);
2315
2316	/*
2317	* Host FS base and GS base.
2318	*/
2319	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2320	if (HMVMX_IS_64BIT_HOST_MODE())
2321	{
2322	uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
2323	uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
2324	rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase); AssertRCReturn(rc, rc);
2325	rc = VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase); AssertRCReturn(rc, rc);
2326
2327	# if HC_ARCH_BITS == 64
2328	/* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
2329	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
2330	pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
2331	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
2332	pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
2333	# endif
2334	}
2335	#endif
2336	return rc;
2337	}
2338
2339
2340	/**
2341	* Saves certain host MSRs in the VM-Exit MSR-load area and some in the
2342	* host-state area of the VMCS. Theses MSRs will be automatically restored on
2343	* the host after every successful VM exit.
2344	*
2345	* @returns VBox status code.
2346	* @param pVM Pointer to the VM.
2347	* @param pVCpu Pointer to the VMCPU.
2348	*/
2349	DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
2350	{
2351	AssertPtr(pVCpu);
2352	AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
2353
2354	int rc = VINF_SUCCESS;
2355	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
2356	PVMXMSR pHostMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvHostMsr;
2357	uint32_t cHostMsrs = 0;
2358	uint32_t u32HostExtFeatures = pVM->hm.s.cpuid.u32AMDFeatureEDX;
2359
2360	if (u32HostExtFeatures & (X86_CPUID_EXT_FEATURE_EDX_NX \| X86_CPUID_EXT_FEATURE_EDX_LONG_MODE))
2361	{
2362	uint64_t u64HostEfer = ASMRdMsr(MSR_K6_EFER);
2363
2364	# if HC_ARCH_BITS == 64
2365	/* Paranoia. 64-bit code requires these bits to be set always. */
2366	Assert((u64HostEfer & (MSR_K6_EFER_LMA \| MSR_K6_EFER_LME)) == (MSR_K6_EFER_LMA \| MSR_K6_EFER_LME));
2367
2368	/*
2369	* We currently do not save/restore host EFER, we just make sure it doesn't get modified by VT-x operation.
2370	* All guest accesses (read, write) on EFER cause VM-exits. If we are to conditionally load the guest EFER for
2371	* some reason (e.g. allow transparent reads) we would activate the code below.
2372	*/
2373	# if 0
2374	/* All our supported 64-bit host platforms must have NXE bit set. Otherwise we can change the below code to save EFER. */
2375	Assert(u64HostEfer & (MSR_K6_EFER_NXE));
2376	/* The SCE bit is only applicable in 64-bit mode. Save EFER if it doesn't match what the guest has.
2377	See Intel spec. 30.10.4.3 "Handling the SYSCALL and SYSRET Instructions". */
2378	if (CPUMIsGuestInLongMode(pVCpu))
2379	{
2380	uint64_t u64GuestEfer;
2381	rc = CPUMQueryGuestMsr(pVCpu, MSR_K6_EFER, &u64GuestEfer);
2382	AssertRC(rc);
2383
2384	if ((u64HostEfer & MSR_K6_EFER_SCE) != (u64GuestEfer & MSR_K6_EFER_SCE))
2385	{
2386	pHostMsr->u32IndexMSR = MSR_K6_EFER;
2387	pHostMsr->u32Reserved = 0;
2388	pHostMsr->u64Value = u64HostEfer;
2389	pHostMsr++; cHostMsrs++;
2390	}
2391	}
2392	# endif
2393	# else /* HC_ARCH_BITS != 64 */
2394	pHostMsr->u32IndexMSR = MSR_K6_EFER;
2395	pHostMsr->u32Reserved = 0;
2396	# if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2397	if (CPUMIsGuestInLongMode(pVCpu))
2398	{
2399	/* Must match the EFER value in our 64 bits switcher. */
2400	pHostMsr->u64Value = u64HostEfer \| MSR_K6_EFER_LME \| MSR_K6_EFER_SCE \| MSR_K6_EFER_NXE;
2401	}
2402	else
2403	# endif
2404	pHostMsr->u64Value = u64HostEfer;
2405	pHostMsr++; cHostMsrs++;
2406	# endif /* HC_ARCH_BITS == 64 */
2407	}
2408
2409	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2410	if (HMVMX_IS_64BIT_HOST_MODE())
2411	{
2412	pHostMsr->u32IndexMSR = MSR_K6_STAR;
2413	pHostMsr->u32Reserved = 0;
2414	pHostMsr->u64Value = ASMRdMsr(MSR_K6_STAR); /* legacy syscall eip, cs & ss */
2415	pHostMsr++; cHostMsrs++;
2416	pHostMsr->u32IndexMSR = MSR_K8_LSTAR;
2417	pHostMsr->u32Reserved = 0;
2418	pHostMsr->u64Value = ASMRdMsr(MSR_K8_LSTAR); /* 64-bit mode syscall rip */
2419	pHostMsr++; cHostMsrs++;
2420	pHostMsr->u32IndexMSR = MSR_K8_SF_MASK;
2421	pHostMsr->u32Reserved = 0;
2422	pHostMsr->u64Value = ASMRdMsr(MSR_K8_SF_MASK); /* syscall flag mask */
2423	pHostMsr++; cHostMsrs++;
2424	pHostMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
2425	pHostMsr->u32Reserved = 0;
2426	pHostMsr->u64Value = ASMRdMsr(MSR_K8_KERNEL_GS_BASE); /* swapgs exchange value */
2427	pHostMsr++; cHostMsrs++;
2428	}
2429	# endif
2430
2431	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
2432	if (RT_UNLIKELY(cHostMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc)))
2433	{
2434	LogRel(("cHostMsrs=%u Cpu=%u\n", cHostMsrs, (unsigned)MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc)));
2435	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2436	}
2437
2438	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cHostMsrs);
2439	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
2440
2441	/*
2442	* Host Sysenter MSRs.
2443	*/
2444	rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
2445	AssertRCReturn(rc, rc);
2446	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2447	if (HMVMX_IS_64BIT_HOST_MODE())
2448	{
2449	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2450	AssertRCReturn(rc, rc);
2451	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2452	}
2453	else
2454	{
2455	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2456	AssertRCReturn(rc, rc);
2457	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2458	}
2459	#elif HC_ARCH_BITS == 32
2460	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2461	AssertRCReturn(rc, rc);
2462	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2463	#else
2464	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2465	AssertRCReturn(rc, rc);
2466	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2467	#endif
2468	AssertRCReturn(rc, rc);
2469
2470	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT, IA32_EFER, also see
2471	* hmR0VmxSetupExitCtls() !! */
2472	return rc;
2473	}
2474
2475
2476	/**
2477	* Sets up VM-entry controls in the VMCS. These controls can affect things done
2478	* on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
2479	* controls".
2480	*
2481	* @returns VBox status code.
2482	* @param pVCpu Pointer to the VMCPU.
2483	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2484	* out-of-sync. Make sure to update the required fields
2485	* before using them.
2486	*
2487	* @remarks No-long-jump zone!!!
2488	*/
2489	DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2490	{
2491	int rc = VINF_SUCCESS;
2492	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_ENTRY_CTLS)
2493	{
2494	PVM pVM = pVCpu->CTX_SUFF(pVM);
2495	uint32_t val = pVM->hm.s.vmx.msr.vmx_entry.n.disallowed0; /* Bits set here must be set in the VMCS. */
2496	uint32_t zap = pVM->hm.s.vmx.msr.vmx_entry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2497
2498	/* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
2499	val \|= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
2500
2501	/* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
2502	if (CPUMIsGuestInLongModeEx(pMixedCtx))
2503	val \|= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
2504	else
2505	Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
2506
2507	/*
2508	* The following should not be set (since we're not in SMM mode):
2509	* - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
2510	* - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
2511	*/
2512
2513	/** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
2514	* VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR,
2515	* VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR */
2516
2517	if ((val & zap) != val)
2518	{
2519	LogRel(("hmR0VmxLoadGuestEntryCtls: invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2520	pVM->hm.s.vmx.msr.vmx_entry.n.disallowed0, val, zap));
2521	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2522	}
2523
2524	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
2525	AssertRCReturn(rc, rc);
2526
2527	/* Update VCPU with the currently set VM-exit controls. */
2528	pVCpu->hm.s.vmx.u32EntryCtls = val;
2529	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_ENTRY_CTLS;
2530	}
2531	return rc;
2532	}
2533
2534
2535	/**
2536	* Sets up the VM-exit controls in the VMCS.
2537	*
2538	* @returns VBox status code.
2539	* @param pVM Pointer to the VM.
2540	* @param pVCpu Pointer to the VMCPU.
2541	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2542	* out-of-sync. Make sure to update the required fields
2543	* before using them.
2544	*
2545	* @remarks requires EFER.
2546	*/
2547	DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2548	{
2549	int rc = VINF_SUCCESS;
2550	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_EXIT_CTLS)
2551	{
2552	PVM pVM = pVCpu->CTX_SUFF(pVM);
2553	uint32_t val = pVM->hm.s.vmx.msr.vmx_exit.n.disallowed0; /* Bits set here must be set in the VMCS. */
2554	uint32_t zap = pVM->hm.s.vmx.msr.vmx_exit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2555
2556	/* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
2557	val \|= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
2558
2559	/*
2560	* Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
2561	* On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
2562	*/
2563	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2564	if (HMVMX_IS_64BIT_HOST_MODE())
2565	val \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
2566	else
2567	Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
2568	#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
2569	if (CPUMIsGuestInLongModeEx(pMixedCtx))
2570	val \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE; /* The switcher goes to long mode. */
2571	else
2572	Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
2573	#endif
2574
2575	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
2576	Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
2577
2578	/** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
2579	* VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
2580	* VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR,
2581	* VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR,
2582	* VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR. */
2583
2584	if (pVM->hm.s.vmx.msr.vmx_exit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER)
2585	val \|= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
2586
2587	if ((val & zap) != val)
2588	{
2589	LogRel(("hmR0VmxSetupProcCtls: invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2590	pVM->hm.s.vmx.msr.vmx_exit.n.disallowed0, val, zap));
2591	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2592	}
2593
2594	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
2595	AssertRCReturn(rc, rc);
2596
2597	/* Update VCPU with the currently set VM-exit controls. */
2598	pVCpu->hm.s.vmx.u32ExitCtls = val;
2599	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_EXIT_CTLS;
2600	}
2601	return rc;
2602	}
2603
2604
2605	/**
2606	* Loads the guest APIC and related state.
2607	*
2608	* @returns VBox status code.
2609	* @param pVM Pointer to the VM.
2610	* @param pVCpu Pointer to the VMCPU.
2611	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2612	* out-of-sync. Make sure to update the required fields
2613	* before using them.
2614	*/
2615	DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2616	{
2617	int rc = VINF_SUCCESS;
2618	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_APIC_STATE)
2619	{
2620	/* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
2621	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2622	{
2623	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2624
2625	bool fPendingIntr = false;
2626	uint8_t u8Tpr = 0;
2627	uint8_t u8PendingIntr = 0;
2628	rc = PDMApicGetTPR(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
2629	AssertRCReturn(rc, rc);
2630
2631	/*
2632	* If there are external interrupts pending but masked by the TPR value, instruct VT-x to cause a VM-exit when
2633	* the guest lowers its TPR below the highest-priority pending interrupt and we can deliver the interrupt.
2634	* If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
2635	* the interrupt when we VM-exit for other reasons.
2636	*/
2637	pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8Tpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
2638	uint32_t u32TprThreshold = 0;
2639	if (fPendingIntr)
2640	{
2641	/* Bits 3-0 of the TPR threshold field correspond to bits 7-4 of the TPR (which is the Task-Priority Class). */
2642	const uint8_t u8PendingPriority = (u8PendingIntr >> 4);
2643	const uint8_t u8TprPriority = (u8Tpr >> 4) & 7;
2644	if (u8PendingPriority <= u8TprPriority)
2645	u32TprThreshold = u8PendingPriority;
2646	else
2647	u32TprThreshold = u8TprPriority; /* Required for Vista 64-bit guest, see @bugref{6398}. */
2648	}
2649	Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
2650
2651	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
2652	AssertRCReturn(rc, rc);
2653	}
2654
2655	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_APIC_STATE;
2656	}
2657	return rc;
2658	}
2659
2660
2661	/**
2662	* Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
2663	*
2664	* @returns Guest's interruptibility-state.
2665	* @param pVCpu Pointer to the VMCPU.
2666	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2667	* out-of-sync. Make sure to update the required fields
2668	* before using them.
2669	*
2670	* @remarks No-long-jump zone!!!
2671	* @remarks Has side-effects with VMCPU_FF_INHIBIT_INTERRUPTS force-flag.
2672	*/
2673	DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2674	{
2675	/*
2676	* Instructions like STI and MOV SS inhibit interrupts till the next instruction completes. Check if we should
2677	* inhibit interrupts or clear any existing interrupt-inhibition.
2678	*/
2679	uint32_t uIntrState = 0;
2680	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
2681	{
2682	/* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
2683	AssertMsg((pVCpu->hm.s.vmx.fUpdatedGuestState & (HMVMX_UPDATED_GUEST_RIP \| HMVMX_UPDATED_GUEST_RFLAGS))
2684	== (HMVMX_UPDATED_GUEST_RIP \| HMVMX_UPDATED_GUEST_RFLAGS), ("%#x\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
2685	if (pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
2686	{
2687	/*
2688	* We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
2689	* VT-x, the flag's condition to be cleared is met and thus the cleared state is correct.
2690	*/
2691	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2692	}
2693	else if (pMixedCtx->eflags.Bits.u1IF)
2694	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
2695	else
2696	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
2697	}
2698	return uIntrState;
2699	}
2700
2701
2702	/**
2703	* Loads the guest's interruptibility-state into the guest-state area in the
2704	* VMCS.
2705	*
2706	* @returns VBox status code.
2707	* @param pVCpu Pointer to the VMCPU.
2708	* @param uIntrState The interruptibility-state to set.
2709	*/
2710	static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
2711	{
2712	AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
2713	Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
2714	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
2715	AssertRCReturn(rc, rc);
2716	return rc;
2717	}
2718
2719
2720	/**
2721	* Loads the guest's RIP into the guest-state area in the VMCS.
2722	*
2723	* @returns VBox status code.
2724	* @param pVCpu Pointer to the VMCPU.
2725	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2726	* out-of-sync. Make sure to update the required fields
2727	* before using them.
2728	*
2729	* @remarks No-long-jump zone!!!
2730	*/
2731	static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2732	{
2733	int rc = VINF_SUCCESS;
2734	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RIP)
2735	{
2736	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
2737	AssertRCReturn(rc, rc);
2738	Log4(("Load: VMX_VMCS_GUEST_RIP=%#RX64\n", pMixedCtx->rip));
2739	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RIP;
2740	}
2741	return rc;
2742	}
2743
2744
2745	/**
2746	* Loads the guest's RSP into the guest-state area in the VMCS.
2747	*
2748	* @returns VBox status code.
2749	* @param pVCpu Pointer to the VMCPU.
2750	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2751	* out-of-sync. Make sure to update the required fields
2752	* before using them.
2753	*
2754	* @remarks No-long-jump zone!!!
2755	*/
2756	static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2757	{
2758	int rc = VINF_SUCCESS;
2759	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RSP)
2760	{
2761	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
2762	AssertRCReturn(rc, rc);
2763	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RSP;
2764	}
2765	return rc;
2766	}
2767
2768
2769	/**
2770	* Loads the guest's RFLAGS into the guest-state area in the VMCS.
2771	*
2772	* @returns VBox status code.
2773	* @param pVCpu Pointer to the VMCPU.
2774	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2775	* out-of-sync. Make sure to update the required fields
2776	* before using them.
2777	*
2778	* @remarks No-long-jump zone!!!
2779	*/
2780	static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2781	{
2782	int rc = VINF_SUCCESS;
2783	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RFLAGS)
2784	{
2785	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
2786	Let us assert it as such and use 32-bit VMWRITE. */
2787	Assert(!(pMixedCtx->rflags.u64 >> 32));
2788	X86EFLAGS uEFlags = pMixedCtx->eflags;
2789	uEFlags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
2790	uEFlags.u32 \|= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
2791
2792	/*
2793	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM exit.
2794	* Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
2795	*/
2796	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2797	{
2798	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
2799	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
2800	pVCpu->hm.s.vmx.RealMode.eflags.u32 = uEFlags.u32; /* Save the original eflags of the real-mode guest. */
2801	uEFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
2802	uEFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
2803	}
2804
2805	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, uEFlags.u32);
2806	AssertRCReturn(rc, rc);
2807
2808	Log4(("Load: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", uEFlags.u32));
2809	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RFLAGS;
2810	}
2811	return rc;
2812	}
2813
2814
2815	/**
2816	* Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
2817	*
2818	* @returns VBox status code.
2819	* @param pVCpu Pointer to the VMCPU.
2820	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2821	* out-of-sync. Make sure to update the required fields
2822	* before using them.
2823	*
2824	* @remarks No-long-jump zone!!!
2825	*/
2826	DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2827	{
2828	int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
2829	AssertRCReturn(rc, rc);
2830	rc = hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
2831	AssertRCReturn(rc, rc);
2832	rc = hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
2833	AssertRCReturn(rc, rc);
2834	return rc;
2835	}
2836
2837
2838	/**
2839	* Loads the guest control registers (CR0, CR3, CR4) into the guest-state area
2840	* in the VMCS.
2841	*
2842	* @returns VBox status code.
2843	* @param pVM Pointer to the VM.
2844	* @param pVCpu Pointer to the VMCPU.
2845	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2846	* out-of-sync. Make sure to update the required fields
2847	* before using them.
2848	*
2849	* @remarks No-long-jump zone!!!
2850	*/
2851	static int hmR0VmxLoadGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pCtx)
2852	{
2853	int rc = VINF_SUCCESS;
2854	PVM pVM = pVCpu->CTX_SUFF(pVM);
2855
2856	/*
2857	* Guest CR0.
2858	* Guest FPU.
2859	*/
2860	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR0)
2861	{
2862	Assert(!(pCtx->cr0 >> 32));
2863	uint32_t u32GuestCR0 = pCtx->cr0;
2864
2865	/* The guest's view (read access) of its CR0 is unblemished. */
2866	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
2867	AssertRCReturn(rc, rc);
2868	Log4(("Load: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", u32GuestCR0));
2869
2870	/* Setup VT-x's view of the guest CR0. */
2871	/* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
2872	if (pVM->hm.s.fNestedPaging)
2873	{
2874	if (CPUMIsGuestPagingEnabledEx(pCtx))
2875	{
2876	/* The guest has paging enabled, let it access CR3 without causing a VM exit if supported. */
2877	pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2878	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
2879	}
2880	else
2881	{
2882	/* The guest doesn't have paging enabled, make CR3 access to cause VM exits to update our shadow. */
2883	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2884	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2885	}
2886
2887	/* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
2888	if (pVM->hm.s.vmx.fUnrestrictedGuest)
2889	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2890
2891	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
2892	AssertRCReturn(rc, rc);
2893	}
2894	else
2895	u32GuestCR0 \|= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
2896
2897	/*
2898	* Guest FPU bits.
2899	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
2900	* CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
2901	*/
2902	u32GuestCR0 \|= X86_CR0_NE;
2903	bool fInterceptNM = false;
2904	if (CPUMIsGuestFPUStateActive(pVCpu))
2905	{
2906	fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
2907	/* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
2908	We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
2909	}
2910	else
2911	{
2912	fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
2913	u32GuestCR0 \|= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
2914	\| X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
2915	}
2916
2917	/* Catch floating point exceptions if we need to report them to the guest in a different way. */
2918	bool fInterceptMF = false;
2919	if (!(pCtx->cr0 & X86_CR0_NE))
2920	fInterceptMF = true;
2921
2922	/* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
2923	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2924	{
2925	Assert(PDMVmmDevHeapIsEnabled(pVM));
2926	Assert(pVM->hm.s.vmx.pRealModeTSS);
2927	pVCpu->hm.s.vmx.u32XcptBitmap \|= HMVMX_REAL_MODE_XCPT_MASK;
2928	fInterceptNM = true;
2929	fInterceptMF = true;
2930	}
2931	else
2932	pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
2933
2934	if (fInterceptNM)
2935	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_NM);
2936	else
2937	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
2938
2939	if (fInterceptMF)
2940	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_MF);
2941	else
2942	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
2943
2944	/* Additional intercepts for debugging, define these yourself explicitly. */
2945	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
2946	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_BP)
2947	\| RT_BIT(X86_XCPT_DB)
2948	\| RT_BIT(X86_XCPT_DE)
2949	\| RT_BIT(X86_XCPT_NM)
2950	\| RT_BIT(X86_XCPT_UD)
2951	\| RT_BIT(X86_XCPT_NP)
2952	\| RT_BIT(X86_XCPT_SS)
2953	\| RT_BIT(X86_XCPT_GP)
2954	\| RT_BIT(X86_XCPT_PF)
2955	\| RT_BIT(X86_XCPT_MF);
2956	#elif defined(HMVMX_ALWAYS_TRAP_PF)
2957	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
2958	#endif
2959
2960	Assert(pVM->hm.s.fNestedPaging \|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
2961
2962	/* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
2963	uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.msr.vmx_cr0_fixed0 & pVM->hm.s.vmx.msr.vmx_cr0_fixed1);
2964	uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.msr.vmx_cr0_fixed0 \| pVM->hm.s.vmx.msr.vmx_cr0_fixed1);
2965	if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
2966	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
2967	else
2968	Assert((uSetCR0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
2969
2970	u32GuestCR0 \|= uSetCR0;
2971	u32GuestCR0 &= uZapCR0;
2972	u32GuestCR0 &= ~(X86_CR0_CD \| X86_CR0_NW); /* Always enable caching. */
2973
2974	/* Write VT-x's view of the guest CR0 into the VMCS and update the exception bitmap. */
2975	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
2976	AssertRCReturn(rc, rc);
2977	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
2978	AssertRCReturn(rc, rc);
2979	Log4(("Load: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", u32GuestCR0, uSetCR0, uZapCR0));
2980
2981	/*
2982	* CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
2983	* by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
2984	* we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
2985	*/
2986	uint32_t u32CR0Mask = 0;
2987	u32CR0Mask = X86_CR0_PE
2988	\| X86_CR0_NE
2989	\| X86_CR0_WP
2990	\| X86_CR0_PG
2991	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
2992	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
2993	\| X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
2994	if (pVM->hm.s.vmx.fUnrestrictedGuest)
2995	u32CR0Mask &= ~X86_CR0_PE;
2996	if (pVM->hm.s.fNestedPaging)
2997	u32CR0Mask &= ~X86_CR0_WP;
2998
2999	/* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
3000	if (fInterceptNM)
3001	u32CR0Mask \|= (X86_CR0_TS \| X86_CR0_MP);
3002	else
3003	u32CR0Mask &= ~(X86_CR0_TS \| X86_CR0_MP);
3004
3005	/* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3006	pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3007	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3008	AssertRCReturn(rc, rc);
3009
3010	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR0;
3011	}
3012
3013	/*
3014	* Guest CR2.
3015	* It's always loaded in the assembler code. Nothing to do here.
3016	*/
3017
3018	/*
3019	* Guest CR3.
3020	*/
3021	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR3)
3022	{
3023	RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3024	if (pVM->hm.s.fNestedPaging)
3025	{
3026	pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3027
3028	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3029	Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3030	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3031	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3032
3033	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3034	pVCpu->hm.s.vmx.HCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
3035	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3036
3037	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3038	AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3039	&& ((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x3f) == 0, /* Bits 6:11 MBZ. */
3040	("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3041
3042	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3043	AssertRCReturn(rc, rc);
3044	Log4(("Load: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3045
3046	if ( pVM->hm.s.vmx.fUnrestrictedGuest
3047	\|\| CPUMIsGuestPagingEnabledEx(pCtx))
3048	{
3049	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3050	if (CPUMIsGuestInPAEModeEx(pCtx))
3051	{
3052	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]); AssertRCReturn(rc, rc);
3053	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
3054	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
3055	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
3056	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
3057	}
3058
3059	/* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
3060	have Unrestricted Execution to handle the guest when it's not using paging. */
3061	GCPhysGuestCR3 = pCtx->cr3;
3062	}
3063	else
3064	{
3065	/*
3066	* The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
3067	* directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
3068	* EPT takes care of translating it to host-physical addresses.
3069	*/
3070	RTGCPHYS GCPhys;
3071	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3072	Assert(PDMVmmDevHeapIsEnabled(pVM));
3073
3074	/* We obtain it here every time as the guest could have relocated this PCI region. */
3075	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3076	AssertRCReturn(rc, rc);
3077
3078	GCPhysGuestCR3 = GCPhys;
3079	}
3080
3081	Log4(("Load: VMX_VMCS_GUEST_CR3=%#RGv (GstN)\n", GCPhysGuestCR3));
3082	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3083	}
3084	else
3085	{
3086	/* Non-nested paging case, just use the hypervisor's CR3. */
3087	RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3088
3089	Log4(("Load: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", HCPhysGuestCR3));
3090	rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3091	}
3092	AssertRCReturn(rc, rc);
3093
3094	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR3;
3095	}
3096
3097	/*
3098	* Guest CR4.
3099	*/
3100	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR4)
3101	{
3102	Assert(!(pCtx->cr4 >> 32));
3103	uint32_t u32GuestCR4 = pCtx->cr4;
3104
3105	/* The guest's view of its CR4 is unblemished. */
3106	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
3107	AssertRCReturn(rc, rc);
3108	Log4(("Load: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", u32GuestCR4));
3109
3110	/* Setup VT-x's view of the guest CR4. */
3111	/*
3112	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
3113	* interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3114	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3115	*/
3116	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3117	{
3118	Assert(pVM->hm.s.vmx.pRealModeTSS);
3119	Assert(PDMVmmDevHeapIsEnabled(pVM));
3120	u32GuestCR4 &= ~X86_CR4_VME;
3121	}
3122
3123	if (pVM->hm.s.fNestedPaging)
3124	{
3125	if ( !CPUMIsGuestPagingEnabledEx(pCtx)
3126	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
3127	{
3128	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
3129	u32GuestCR4 \|= X86_CR4_PSE;
3130	/* Our identity mapping is a 32 bits page directory. */
3131	u32GuestCR4 &= ~X86_CR4_PAE;
3132	}
3133	/* else use guest CR4.*/
3134	}
3135	else
3136	{
3137	/*
3138	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
3139	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
3140	*/
3141	switch (pVCpu->hm.s.enmShadowMode)
3142	{
3143	case PGMMODE_REAL: /* Real-mode. */
3144	case PGMMODE_PROTECTED: /* Protected mode without paging. */
3145	case PGMMODE_32_BIT: /* 32-bit paging. */
3146	{
3147	u32GuestCR4 &= ~X86_CR4_PAE;
3148	break;
3149	}
3150
3151	case PGMMODE_PAE: /* PAE paging. */
3152	case PGMMODE_PAE_NX: /* PAE paging with NX. */
3153	{
3154	u32GuestCR4 \|= X86_CR4_PAE;
3155	break;
3156	}
3157
3158	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
3159	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
3160	#ifdef VBOX_ENABLE_64_BITS_GUESTS
3161	break;
3162	#endif
3163	default:
3164	AssertFailed();
3165	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3166	}
3167	}
3168
3169	/* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
3170	uint64_t uSetCR4 = (pVM->hm.s.vmx.msr.vmx_cr4_fixed0 & pVM->hm.s.vmx.msr.vmx_cr4_fixed1);
3171	uint64_t uZapCR4 = (pVM->hm.s.vmx.msr.vmx_cr4_fixed0 \| pVM->hm.s.vmx.msr.vmx_cr4_fixed1);
3172	u32GuestCR4 \|= uSetCR4;
3173	u32GuestCR4 &= uZapCR4;
3174
3175	/* Write VT-x's view of the guest CR4 into the VMCS. */
3176	Log4(("Load: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", u32GuestCR4, uSetCR4, uZapCR4));
3177	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
3178	AssertRCReturn(rc, rc);
3179
3180	/* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM exit. */
3181	uint32_t u32CR4Mask = 0;
3182	u32CR4Mask = X86_CR4_VME
3183	\| X86_CR4_PAE
3184	\| X86_CR4_PGE
3185	\| X86_CR4_PSE
3186	\| X86_CR4_VMXE;
3187	pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
3188	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
3189	AssertRCReturn(rc, rc);
3190
3191	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR4;
3192	}
3193	return rc;
3194	}
3195
3196
3197	/**
3198	* Loads the guest debug registers into the guest-state area in the VMCS.
3199	* This also sets up whether #DB and MOV DRx accesses cause VM exits.
3200	*
3201	* @returns VBox status code.
3202	* @param pVCpu Pointer to the VMCPU.
3203	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3204	* out-of-sync. Make sure to update the required fields
3205	* before using them.
3206	*
3207	* @remarks No-long-jump zone!!!
3208	*/
3209	static int hmR0VmxLoadGuestDebugRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3210	{
3211	if (!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_DEBUG))
3212	return VINF_SUCCESS;
3213
3214	#ifdef VBOX_STRICT
3215	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
3216	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
3217	{
3218	Assert(!(pMixedCtx->dr[7] >> 32)); /* upper 32 bits are reserved (MBZ). */
3219	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
3220	Assert((pMixedCtx->dr[7] & 0xd800) == 0); /* bits 15, 14, 12, 11 are reserved (MBZ). */
3221	Assert((pMixedCtx->dr[7] & 0x400) == 0x400); /* bit 10 is reserved (MB1). */
3222	}
3223	#endif
3224
3225	int rc = VERR_INTERNAL_ERROR_5;
3226	PVM pVM = pVCpu->CTX_SUFF(pVM);
3227	bool fInterceptDB = false;
3228	bool fInterceptMovDRx = false;
3229	if (DBGFIsStepping(pVCpu))
3230	{
3231	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
3232	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
3233	{
3234	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
3235	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3236	AssertRCReturn(rc, rc);
3237	Assert(fInterceptDB == false);
3238	}
3239	else
3240	{
3241	fInterceptDB = true;
3242	pMixedCtx->eflags.u32 \|= X86_EFL_TF;
3243	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RFLAGS;
3244	}
3245	}
3246
3247	if (CPUMGetHyperDR7(pVCpu) & (X86_DR7_ENABLED_MASK \| X86_DR7_GD))
3248	{
3249	if (!CPUMIsHyperDebugStateActive(pVCpu))
3250	{
3251	rc = CPUMR0LoadHyperDebugState(pVM, pVCpu, pMixedCtx, true /* include DR6 */);
3252	AssertRC(rc);
3253	}
3254	Assert(CPUMIsHyperDebugStateActive(pVCpu));
3255	fInterceptMovDRx = true;
3256	}
3257	else if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD))
3258	{
3259	if (!CPUMIsGuestDebugStateActive(pVCpu))
3260	{
3261	rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pMixedCtx, true /* include DR6 */);
3262	AssertRC(rc);
3263	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
3264	}
3265	Assert(CPUMIsGuestDebugStateActive(pVCpu));
3266	Assert(fInterceptMovDRx == false);
3267	}
3268	else if (!CPUMIsGuestDebugStateActive(pVCpu))
3269	{
3270	/* For the first time we would need to intercept MOV DRx accesses even when the guest debug registers aren't loaded. */
3271	fInterceptMovDRx = true;
3272	}
3273
3274	/* Update the exception bitmap regarding intercepting #DB generated by the guest. */
3275	if (fInterceptDB)
3276	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_DB);
3277	else if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3278	{
3279	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3280	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
3281	#endif
3282	}
3283
3284	/* Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions. */
3285	if (fInterceptMovDRx)
3286	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
3287	else
3288	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
3289
3290	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3291	AssertRCReturn(rc, rc);
3292	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3293	AssertRCReturn(rc, rc);
3294
3295	/* The guest's view of its DR7 is unblemished. Use 32-bit write as upper 32-bits MBZ as asserted above. */
3296	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
3297	AssertRCReturn(rc, rc);
3298
3299	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_DEBUG;
3300	return rc;
3301	}
3302
3303
3304	#ifdef VBOX_STRICT
3305	/**
3306	* Strict function to validate segment registers.
3307	*
3308	* @remarks Requires CR0.
3309	*/
3310	static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3311	{
3312	/* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
3313	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0);
3314	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS);
3315
3316	/* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
3317	* only updates the VMCS bits with the unusable bit and doesn't change the guest-context value. */
3318	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
3319	&& ( !CPUMIsGuestInRealModeEx(pCtx)
3320	&& !CPUMIsGuestInV86ModeEx(pCtx)))
3321	{
3322	/* Protected mode checks */
3323	/* CS */
3324	Assert(pCtx->cs.Attr.n.u1Present);
3325	Assert(!(pCtx->cs.Attr.u & 0xf00));
3326	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
3327	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
3328	\|\| !(pCtx->cs.Attr.n.u1Granularity));
3329	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
3330	\|\| (pCtx->cs.Attr.n.u1Granularity));
3331	/* CS cannot be loaded with NULL in protected mode. */
3332	Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & HMVMX_SEL_UNUSABLE)); /** @todo is this really true even for 64-bit CS?!? */
3333	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
3334	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
3335	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
3336	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
3337	else
3338	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
3339	/* SS */
3340	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3341	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
3342	Assert(!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR0));
3343	if ( !(pCtx->cr0 & X86_CR0_PE)
3344	\|\| pCtx->cs.Attr.n.u4Type == 3)
3345	{
3346	Assert(!pCtx->ss.Attr.n.u2Dpl);
3347	}
3348	if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & HMVMX_SEL_UNUSABLE))
3349	{
3350	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3351	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
3352	Assert(pCtx->ss.Attr.n.u1Present);
3353	Assert(!(pCtx->ss.Attr.u & 0xf00));
3354	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
3355	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
3356	\|\| !(pCtx->ss.Attr.n.u1Granularity));
3357	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
3358	\|\| (pCtx->ss.Attr.n.u1Granularity));
3359	}
3360	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
3361	if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & HMVMX_SEL_UNUSABLE))
3362	{
3363	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3364	Assert(pCtx->ds.Attr.n.u1Present);
3365	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
3366	Assert(!(pCtx->ds.Attr.u & 0xf00));
3367	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
3368	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
3369	\|\| !(pCtx->ds.Attr.n.u1Granularity));
3370	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
3371	\|\| (pCtx->ds.Attr.n.u1Granularity));
3372	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3373	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
3374	}
3375	if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & HMVMX_SEL_UNUSABLE))
3376	{
3377	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3378	Assert(pCtx->es.Attr.n.u1Present);
3379	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
3380	Assert(!(pCtx->es.Attr.u & 0xf00));
3381	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
3382	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
3383	\|\| !(pCtx->es.Attr.n.u1Granularity));
3384	Assert( !(pCtx->es.u32Limit & 0xfff00000)
3385	\|\| (pCtx->es.Attr.n.u1Granularity));
3386	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3387	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
3388	}
3389	if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & HMVMX_SEL_UNUSABLE))
3390	{
3391	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3392	Assert(pCtx->fs.Attr.n.u1Present);
3393	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
3394	Assert(!(pCtx->fs.Attr.u & 0xf00));
3395	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
3396	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
3397	\|\| !(pCtx->fs.Attr.n.u1Granularity));
3398	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
3399	\|\| (pCtx->fs.Attr.n.u1Granularity));
3400	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3401	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3402	}
3403	if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & HMVMX_SEL_UNUSABLE))
3404	{
3405	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3406	Assert(pCtx->gs.Attr.n.u1Present);
3407	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
3408	Assert(!(pCtx->gs.Attr.u & 0xf00));
3409	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
3410	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
3411	\|\| !(pCtx->gs.Attr.n.u1Granularity));
3412	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
3413	\|\| (pCtx->gs.Attr.n.u1Granularity));
3414	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3415	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3416	}
3417	/* 64-bit capable CPUs. */
3418	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3419	Assert(!(pCtx->cs.u64Base >> 32));
3420	Assert(!pCtx->ss.Attr.u \|\| !(pCtx->ss.u64Base >> 32));
3421	Assert(!pCtx->ds.Attr.u \|\| !(pCtx->ds.u64Base >> 32));
3422	Assert(!pCtx->es.Attr.u \|\| !(pCtx->es.u64Base >> 32));
3423	# endif
3424	}
3425	else if ( CPUMIsGuestInV86ModeEx(pCtx)
3426	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
3427	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
3428	{
3429	/* Real and v86 mode checks. */
3430	/* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
3431	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
3432	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3433	{
3434	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
3435	}
3436	else
3437	{
3438	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
3439	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
3440	}
3441
3442	/* CS */
3443	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
3444	Assert(pCtx->cs.u32Limit == 0xffff);
3445	Assert(u32CSAttr == 0xf3);
3446	/* SS */
3447	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
3448	Assert(pCtx->ss.u32Limit == 0xffff);
3449	Assert(u32SSAttr == 0xf3);
3450	/* DS */
3451	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
3452	Assert(pCtx->ds.u32Limit == 0xffff);
3453	Assert(u32DSAttr == 0xf3);
3454	/* ES */
3455	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
3456	Assert(pCtx->es.u32Limit == 0xffff);
3457	Assert(u32ESAttr == 0xf3);
3458	/* FS */
3459	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
3460	Assert(pCtx->fs.u32Limit == 0xffff);
3461	Assert(u32FSAttr == 0xf3);
3462	/* GS */
3463	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
3464	Assert(pCtx->gs.u32Limit == 0xffff);
3465	Assert(u32GSAttr == 0xf3);
3466	/* 64-bit capable CPUs. */
3467	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3468	Assert(!(pCtx->cs.u64Base >> 32));
3469	Assert(!u32SSAttr \|\| !(pCtx->ss.u64Base >> 32));
3470	Assert(!u32DSAttr \|\| !(pCtx->ds.u64Base >> 32));
3471	Assert(!u32ESAttr \|\| !(pCtx->es.u64Base >> 32));
3472	# endif
3473	}
3474	}
3475	#endif /* VBOX_STRICT */
3476
3477
3478	/**
3479	* Writes a guest segment register into the guest-state area in the VMCS.
3480	*
3481	* @returns VBox status code.
3482	* @param pVCpu Pointer to the VMCPU.
3483	* @param idxSel Index of the selector in the VMCS.
3484	* @param idxLimit Index of the segment limit in the VMCS.
3485	* @param idxBase Index of the segment base in the VMCS.
3486	* @param idxAccess Index of the access rights of the segment in the VMCS.
3487	* @param pSelReg Pointer to the segment selector.
3488	* @param pCtx Pointer to the guest-CPU context.
3489	*
3490	* @remarks No-long-jump zone!!!
3491	*/
3492	static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
3493	uint32_t idxAccess, PCPUMSELREG pSelReg, PCPUMCTX pCtx)
3494	{
3495	int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
3496	AssertRCReturn(rc, rc);
3497	rc = VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
3498	AssertRCReturn(rc, rc);
3499	rc = VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
3500	AssertRCReturn(rc, rc);
3501
3502	uint32_t u32Access = pSelReg->Attr.u;
3503	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3504	{
3505	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
3506	u32Access = 0xf3;
3507	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3508	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3509	}
3510	else
3511	{
3512	/*
3513	* The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
3514	* real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
3515	* protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
3516	* loaded in protected-mode have their attribute as 0.
3517	*/
3518	if (!u32Access)
3519	u32Access = HMVMX_SEL_UNUSABLE;
3520	}
3521
3522	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
3523	AssertMsg((u32Access & HMVMX_SEL_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
3524	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
3525
3526	rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
3527	AssertRCReturn(rc, rc);
3528	return rc;
3529	}
3530
3531
3532	/**
3533	* Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
3534	* into the guest-state area in the VMCS.
3535	*
3536	* @returns VBox status code.
3537	* @param pVM Pointer to the VM.
3538	* @param pVCPU Pointer to the VMCPU.
3539	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3540	* out-of-sync. Make sure to update the required fields
3541	* before using them.
3542	*
3543	* @remarks Requires CR0 (strict builds validation).
3544	* @remarks No-long-jump zone!!!
3545	*/
3546	static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3547	{
3548	int rc = VERR_INTERNAL_ERROR_5;
3549	PVM pVM = pVCpu->CTX_SUFF(pVM);
3550
3551	/*
3552	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
3553	*/
3554	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SEGMENT_REGS)
3555	{
3556	/* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
3557	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3558	{
3559	pVCpu->hm.s.vmx.RealMode.uAttrCS.u = pMixedCtx->cs.Attr.u;
3560	pVCpu->hm.s.vmx.RealMode.uAttrSS.u = pMixedCtx->ss.Attr.u;
3561	pVCpu->hm.s.vmx.RealMode.uAttrDS.u = pMixedCtx->ds.Attr.u;
3562	pVCpu->hm.s.vmx.RealMode.uAttrES.u = pMixedCtx->es.Attr.u;
3563	pVCpu->hm.s.vmx.RealMode.uAttrFS.u = pMixedCtx->fs.Attr.u;
3564	pVCpu->hm.s.vmx.RealMode.uAttrGS.u = pMixedCtx->gs.Attr.u;
3565	}
3566
3567	#ifdef VBOX_WITH_REM
3568	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
3569	{
3570	Assert(pVM->hm.s.vmx.pRealModeTSS);
3571	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
3572	if ( pVCpu->hm.s.vmx.fWasInRealMode
3573	&& PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
3574	{
3575	/* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
3576	in real-mode (e.g. OpenBSD 4.0) */
3577	REMFlushTBs(pVM);
3578	Log4(("Load: Switch to protected mode detected!\n"));
3579	pVCpu->hm.s.vmx.fWasInRealMode = false;
3580	}
3581	}
3582	#endif
3583	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
3584	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs, pMixedCtx);
3585	AssertRCReturn(rc, rc);
3586	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
3587	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss, pMixedCtx);
3588	AssertRCReturn(rc, rc);
3589	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
3590	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds, pMixedCtx);
3591	AssertRCReturn(rc, rc);
3592	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
3593	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es, pMixedCtx);
3594	AssertRCReturn(rc, rc);
3595	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
3596	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs, pMixedCtx);
3597	AssertRCReturn(rc, rc);
3598	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
3599	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs, pMixedCtx);
3600	AssertRCReturn(rc, rc);
3601
3602	Log4(("Load: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pMixedCtx->cs.Sel, pMixedCtx->cs.u64Base,
3603	pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
3604	#ifdef VBOX_STRICT
3605	hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
3606	#endif
3607	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SEGMENT_REGS;
3608	}
3609
3610	/*
3611	* Guest TR.
3612	*/
3613	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_TR)
3614	{
3615	/*
3616	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
3617	* using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
3618	* See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
3619	*/
3620	uint16_t u16Sel = 0;
3621	uint32_t u32Limit = 0;
3622	uint64_t u64Base = 0;
3623	uint32_t u32AccessRights = 0;
3624
3625	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3626	{
3627	u16Sel = pMixedCtx->tr.Sel;
3628	u32Limit = pMixedCtx->tr.u32Limit;
3629	u64Base = pMixedCtx->tr.u64Base;
3630	u32AccessRights = pMixedCtx->tr.Attr.u;
3631	}
3632	else
3633	{
3634	Assert(pVM->hm.s.vmx.pRealModeTSS);
3635	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
3636
3637	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
3638	RTGCPHYS GCPhys;
3639	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
3640	AssertRCReturn(rc, rc);
3641
3642	X86DESCATTR DescAttr;
3643	DescAttr.u = 0;
3644	DescAttr.n.u1Present = 1;
3645	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
3646
3647	u16Sel = 0;
3648	u32Limit = HM_VTX_TSS_SIZE;
3649	u64Base = GCPhys; /* in real-mode phys = virt. */
3650	u32AccessRights = DescAttr.u;
3651	}
3652
3653	/* Validate. */
3654	Assert(!(u16Sel & RT_BIT(2)));
3655	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
3656	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
3657	AssertMsg(!(u32AccessRights & HMVMX_SEL_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
3658	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
3659	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
3660	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
3661	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
3662	Assert( (u32Limit & 0xfff) == 0xfff
3663	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
3664	Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
3665	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
3666
3667	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_TR, u16Sel); AssertRCReturn(rc, rc);
3668	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit); AssertRCReturn(rc, rc);
3669	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base); AssertRCReturn(rc, rc);
3670	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights); AssertRCReturn(rc, rc);
3671
3672	Log4(("Load: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", u64Base));
3673	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_TR;
3674	}
3675
3676	/*
3677	* Guest GDTR.
3678	*/
3679	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_GDTR)
3680	{
3681	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt); AssertRCReturn(rc, rc);
3682	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt); AssertRCReturn(rc, rc);
3683
3684	Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
3685	Log4(("Load: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pMixedCtx->gdtr.pGdt));
3686	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_GDTR;
3687	}
3688
3689	/*
3690	* Guest LDTR.
3691	*/
3692	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_LDTR)
3693	{
3694	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
3695	uint32_t u32Access = 0;
3696	if (!pMixedCtx->ldtr.Attr.u)
3697	u32Access = HMVMX_SEL_UNUSABLE;
3698	else
3699	u32Access = pMixedCtx->ldtr.Attr.u;
3700
3701	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_LDTR, pMixedCtx->ldtr.Sel); AssertRCReturn(rc, rc);
3702	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit); AssertRCReturn(rc, rc);
3703	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base); AssertRCReturn(rc, rc);
3704	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access); AssertRCReturn(rc, rc);
3705
3706	/* Validate. */
3707	if (!(u32Access & HMVMX_SEL_UNUSABLE))
3708	{
3709	Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
3710	Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
3711	Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
3712	Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
3713	Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
3714	Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
3715	Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
3716	\|\| !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
3717	Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
3718	\|\| pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
3719	}
3720
3721	Log4(("Load: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pMixedCtx->ldtr.u64Base));
3722	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_LDTR;
3723	}
3724
3725	/*
3726	* Guest IDTR.
3727	*/
3728	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_IDTR)
3729	{
3730	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt); AssertRCReturn(rc, rc);
3731	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt); AssertRCReturn(rc, rc);
3732
3733	Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
3734	Log4(("Load: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pMixedCtx->idtr.pIdt));
3735	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_IDTR;
3736	}
3737
3738	return VINF_SUCCESS;
3739	}
3740
3741
3742	/**
3743	* Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
3744	* areas. These MSRs will automatically be loaded to the host CPU on every
3745	* successful VM entry and stored from the host CPU on every successful VM exit.
3746	* Also loads the sysenter MSRs into the guest-state area in the VMCS.
3747	*
3748	* @returns VBox status code.
3749	* @param pVCpu Pointer to the VMCPU.
3750	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3751	* out-of-sync. Make sure to update the required fields
3752	* before using them.
3753	*
3754	* @remarks No-long-jump zone!!!
3755	*/
3756	static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3757	{
3758	AssertPtr(pVCpu);
3759	AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
3760
3761	/*
3762	* MSRs covered by Auto-load/store: EFER, LSTAR, STAR, SF_MASK, TSC_AUX (RDTSCP).
3763	*/
3764	int rc = VINF_SUCCESS;
3765	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
3766	{
3767	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
3768	PVM pVM = pVCpu->CTX_SUFF(pVM);
3769	PVMXMSR pGuestMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvGuestMsr;
3770	uint32_t cGuestMsrs = 0;
3771
3772	/* See Intel spec. 4.1.4 "Enumeration of Paging Features by CPUID". */
3773	/** @todo r=ramshankar: Optimize this further to do lazy restoration and only
3774	* when the guest really is in 64-bit mode. */
3775	bool fSupportsLongMode = CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_LONG_MODE);
3776	if (fSupportsLongMode)
3777	{
3778	pGuestMsr->u32IndexMSR = MSR_K8_LSTAR;
3779	pGuestMsr->u32Reserved = 0;
3780	pGuestMsr->u64Value = pMixedCtx->msrLSTAR; /* 64 bits mode syscall rip */
3781	pGuestMsr++; cGuestMsrs++;
3782	pGuestMsr->u32IndexMSR = MSR_K6_STAR;
3783	pGuestMsr->u32Reserved = 0;
3784	pGuestMsr->u64Value = pMixedCtx->msrSTAR; /* legacy syscall eip, cs & ss */
3785	pGuestMsr++; cGuestMsrs++;
3786	pGuestMsr->u32IndexMSR = MSR_K8_SF_MASK;
3787	pGuestMsr->u32Reserved = 0;
3788	pGuestMsr->u64Value = pMixedCtx->msrSFMASK; /* syscall flag mask */
3789	pGuestMsr++; cGuestMsrs++;
3790	pGuestMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
3791	pGuestMsr->u32Reserved = 0;
3792	pGuestMsr->u64Value = pMixedCtx->msrKERNELGSBASE; /* swapgs exchange value */
3793	pGuestMsr++; cGuestMsrs++;
3794	}
3795
3796	/*
3797	* RDTSCP requires the TSC_AUX MSR. Host and guest share the physical MSR. So we have to
3798	* load the guest's copy if the guest can execute RDTSCP without causing VM-exits.
3799	*/
3800	if ( CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_RDTSCP)
3801	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP))
3802	{
3803	pGuestMsr->u32IndexMSR = MSR_K8_TSC_AUX;
3804	pGuestMsr->u32Reserved = 0;
3805	rc = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &pGuestMsr->u64Value);
3806	AssertRCReturn(rc, rc);
3807	pGuestMsr++; cGuestMsrs++;
3808	}
3809
3810	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
3811	if (cGuestMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc))
3812	{
3813	LogRel(("CPU autoload/store MSR count in VMCS exceeded cGuestMsrs=%u.\n", cGuestMsrs));
3814	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3815	}
3816
3817	/* Update the VCPU's copy of the guest MSR count. */
3818	pVCpu->hm.s.vmx.cGuestMsrs = cGuestMsrs;
3819	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cGuestMsrs); AssertRCReturn(rc, rc);
3820	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cGuestMsrs); AssertRCReturn(rc, rc);
3821	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
3822
3823	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_AUTO_MSRS;
3824	}
3825
3826	/*
3827	* Guest Sysenter MSRs.
3828	* These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
3829	* VM-exits on WRMSRs for these MSRs.
3830	*/
3831	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
3832	{
3833	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
3834	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_CS_MSR;
3835	}
3836	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
3837	{
3838	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
3839	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR;
3840	}
3841	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
3842	{
3843	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
3844	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR;
3845	}
3846
3847	return rc;
3848	}
3849
3850
3851	/**
3852	* Loads the guest activity state into the guest-state area in the VMCS.
3853	*
3854	* @returns VBox status code.
3855	* @param pVCpu Pointer to the VMCPU.
3856	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3857	* out-of-sync. Make sure to update the required fields
3858	* before using them.
3859	*
3860	* @remarks No-long-jump zone!!!
3861	*/
3862	static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pCtx)
3863	{
3864	/** @todo See if we can make use of other states, e.g.
3865	* VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
3866	int rc = VINF_SUCCESS;
3867	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_ACTIVITY_STATE)
3868	{
3869	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
3870	AssertRCReturn(rc, rc);
3871	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_ACTIVITY_STATE;
3872	}
3873	return rc;
3874	}
3875
3876
3877	/**
3878	* Sets up the appropriate function to run guest code.
3879	*
3880	* @returns VBox status code.
3881	* @param pVCpu Pointer to the VMCPU.
3882	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3883	* out-of-sync. Make sure to update the required fields
3884	* before using them.
3885	*
3886	* @remarks No-long-jump zone!!!
3887	*/
3888	static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3889	{
3890	if (CPUMIsGuestInLongModeEx(pMixedCtx))
3891	{
3892	#ifndef VBOX_ENABLE_64_BITS_GUESTS
3893	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3894	#endif
3895	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
3896	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3897	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
3898	pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
3899	#else
3900	/* 64-bit host or hybrid host. */
3901	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
3902	#endif
3903	}
3904	else
3905	{
3906	/* Guest is not in long mode, use the 32-bit handler. */
3907	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
3908	}
3909	Assert(pVCpu->hm.s.vmx.pfnStartVM);
3910	return VINF_SUCCESS;
3911	}
3912
3913
3914	/**
3915	* Wrapper for running the guest code in VT-x.
3916	*
3917	* @returns VBox strict status code.
3918	* @param pVM Pointer to the VM.
3919	* @param pVCpu Pointer to the VMCPU.
3920	* @param pCtx Pointer to the guest-CPU context.
3921	*
3922	* @remarks No-long-jump zone!!!
3923	*/
3924	DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3925	{
3926	/*
3927	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
3928	* using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
3929	* Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
3930	*/
3931	#ifdef VBOX_WITH_KERNEL_USING_XMM
3932	return HMR0VMXStartVMWrapXMM(pVCpu->hm.s.fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
3933	#else
3934	return pVCpu->hm.s.vmx.pfnStartVM(pVCpu->hm.s.fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
3935	#endif
3936	}
3937
3938
3939	/**
3940	* Reports world-switch error and dumps some useful debug info.
3941	*
3942	* @param pVM Pointer to the VM.
3943	* @param pVCpu Pointer to the VMCPU.
3944	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
3945	* @param pCtx Pointer to the guest-CPU context.
3946	* @param pVmxTransient Pointer to the VMX transient structure (only
3947	* exitReason updated).
3948	*/
3949	static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
3950	{
3951	Assert(pVM);
3952	Assert(pVCpu);
3953	Assert(pCtx);
3954	Assert(pVmxTransient);
3955	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3956
3957	Log4(("VM-entry failure: %Rrc\n", rcVMRun));
3958	switch (rcVMRun)
3959	{
3960	case VERR_VMX_INVALID_VMXON_PTR:
3961	AssertFailed();
3962	break;
3963	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
3964	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
3965	{
3966	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.lasterror.u32ExitReason);
3967	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.lasterror.u32InstrError);
3968	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
3969	AssertRC(rc);
3970
3971	#ifdef VBOX_STRICT
3972	Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.lasterror.u32ExitReason,
3973	pVmxTransient->uExitReason));
3974	Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
3975	Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.lasterror.u32InstrError));
3976	if (pVCpu->hm.s.vmx.lasterror.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
3977	Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.lasterror.u32InstrError]));
3978	else
3979	Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
3980
3981	/* VMX control bits. */
3982	uint32_t u32Val;
3983	uint64_t u64Val;
3984	HMVMXHCUINTREG uHCReg;
3985	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
3986	Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
3987	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
3988	Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
3989	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
3990	Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
3991	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
3992	Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
3993	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
3994	Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
3995	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
3996	Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
3997	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
3998	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
3999	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
4000	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
4001	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
4002	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
4003	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
4004	Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
4005	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
4006	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
4007	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4008	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
4009	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4010	Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
4011	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
4012	Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
4013	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
4014	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
4015	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
4016	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
4017	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
4018	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
4019	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
4020	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4021	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
4022	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
4023	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
4024	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4025	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
4026	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
4027
4028	/* Guest bits. */
4029	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
4030	Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
4031	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
4032	Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
4033	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
4034	Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
4035	rc = VMXReadVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, &u32Val); AssertRC(rc);
4036	Log4(("VMX_VMCS16_GUEST_FIELD_VPID %u\n", u32Val));
4037
4038	/* Host bits. */
4039	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
4040	Log4(("Host CR0 %#RHr\n", uHCReg));
4041	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
4042	Log4(("Host CR3 %#RHr\n", uHCReg));
4043	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
4044	Log4(("Host CR4 %#RHr\n", uHCReg));
4045
4046	RTGDTR HostGdtr;
4047	PCX86DESCHC pDesc;
4048	ASMGetGDTR(&HostGdtr);
4049	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_CS, &u32Val); AssertRC(rc);
4050	Log4(("Host CS %#08x\n", u32Val));
4051	if (u32Val < HostGdtr.cbGdt)
4052	{
4053	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4054	HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
4055	}
4056
4057	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_DS, &u32Val); AssertRC(rc);
4058	Log4(("Host DS %#08x\n", u32Val));
4059	if (u32Val < HostGdtr.cbGdt)
4060	{
4061	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4062	HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
4063	}
4064
4065	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_ES, &u32Val); AssertRC(rc);
4066	Log4(("Host ES %#08x\n", u32Val));
4067	if (u32Val < HostGdtr.cbGdt)
4068	{
4069	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4070	HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
4071	}
4072
4073	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_FS, &u32Val); AssertRC(rc);
4074	Log4(("Host FS %#08x\n", u32Val));
4075	if (u32Val < HostGdtr.cbGdt)
4076	{
4077	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4078	HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
4079	}
4080
4081	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_GS, &u32Val); AssertRC(rc);
4082	Log4(("Host GS %#08x\n", u32Val));
4083	if (u32Val < HostGdtr.cbGdt)
4084	{
4085	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4086	HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
4087	}
4088
4089	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_SS, &u32Val); AssertRC(rc);
4090	Log4(("Host SS %#08x\n", u32Val));
4091	if (u32Val < HostGdtr.cbGdt)
4092	{
4093	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4094	HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
4095	}
4096
4097	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_TR, &u32Val); AssertRC(rc);
4098	Log4(("Host TR %#08x\n", u32Val));
4099	if (u32Val < HostGdtr.cbGdt)
4100	{
4101	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4102	HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
4103	}
4104
4105	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
4106	Log4(("Host TR Base %#RHv\n", uHCReg));
4107	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
4108	Log4(("Host GDTR Base %#RHv\n", uHCReg));
4109	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
4110	Log4(("Host IDTR Base %#RHv\n", uHCReg));
4111	rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
4112	Log4(("Host SYSENTER CS %#08x\n", u32Val));
4113	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
4114	Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
4115	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
4116	Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
4117	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
4118	Log4(("Host RSP %#RHv\n", uHCReg));
4119	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
4120	Log4(("Host RIP %#RHv\n", uHCReg));
4121	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4122	if (HMVMX_IS_64BIT_HOST_MODE())
4123	{
4124	Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
4125	Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
4126	Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
4127	Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
4128	Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
4129	Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
4130	}
4131	# endif
4132	#endif /* VBOX_STRICT */
4133	break;
4134	}
4135
4136	default:
4137	/* Impossible */
4138	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
4139	break;
4140	}
4141	NOREF(pVM);
4142	}
4143
4144
4145	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4146	#ifndef VMX_USE_CACHED_VMCS_ACCESSES
4147	# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
4148	#endif
4149	#ifdef VBOX_STRICT
4150	static bool hmR0VmxIsValidWriteField(uint32_t idxField)
4151	{
4152	switch (idxField)
4153	{
4154	case VMX_VMCS_GUEST_RIP:
4155	case VMX_VMCS_GUEST_RSP:
4156	case VMX_VMCS_GUEST_SYSENTER_EIP:
4157	case VMX_VMCS_GUEST_SYSENTER_ESP:
4158	case VMX_VMCS_GUEST_GDTR_BASE:
4159	case VMX_VMCS_GUEST_IDTR_BASE:
4160	case VMX_VMCS_GUEST_CS_BASE:
4161	case VMX_VMCS_GUEST_DS_BASE:
4162	case VMX_VMCS_GUEST_ES_BASE:
4163	case VMX_VMCS_GUEST_FS_BASE:
4164	case VMX_VMCS_GUEST_GS_BASE:
4165	case VMX_VMCS_GUEST_SS_BASE:
4166	case VMX_VMCS_GUEST_LDTR_BASE:
4167	case VMX_VMCS_GUEST_TR_BASE:
4168	case VMX_VMCS_GUEST_CR3:
4169	return true;
4170	}
4171	return false;
4172	}
4173
4174	static bool hmR0VmxIsValidReadField(uint32_t idxField)
4175	{
4176	switch (idxField)
4177	{
4178	/* Read-only fields. */
4179	case VMX_VMCS_RO_EXIT_QUALIFICATION:
4180	return true;
4181	}
4182	/* Remaining readable fields should also be writable. */
4183	return hmR0VmxIsValidWriteField(idxField);
4184	}
4185	#endif /* VBOX_STRICT */
4186
4187
4188	/**
4189	* Executes the specified handler in 64-bit mode.
4190	*
4191	* @returns VBox status code.
4192	* @param pVM Pointer to the VM.
4193	* @param pVCpu Pointer to the VMCPU.
4194	* @param pCtx Pointer to the guest CPU context.
4195	* @param enmOp The operation to perform.
4196	* @param cbParam Number of parameters.
4197	* @param paParam Array of 32-bit parameters.
4198	*/
4199	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp, uint32_t cbParam,
4200	uint32_t *paParam)
4201	{
4202	int rc, rc2;
4203	PHMGLOBLCPUINFO pCpu;
4204	RTHCPHYS HCPhysCpuPage;
4205	RTCCUINTREG uOldEFlags;
4206
4207	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
4208	Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
4209	Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
4210	Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
4211
4212	#ifdef VBOX_STRICT
4213	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
4214	Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
4215
4216	for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
4217	Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
4218	#endif
4219
4220	/* Disable interrupts. */
4221	uOldEFlags = ASMIntDisableFlags();
4222
4223	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
4224	RTCPUID idHostCpu = RTMpCpuId();
4225	CPUMR0SetLApic(pVM, idHostCpu);
4226	#endif
4227
4228	pCpu = HMR0GetCurrentCpu();
4229	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4230
4231	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
4232	VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
4233
4234	/* Leave VMX Root Mode. */
4235	VMXDisable();
4236
4237	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4238
4239	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
4240	CPUMSetHyperEIP(pVCpu, enmOp);
4241	for (int i = (int)cbParam - 1; i >= 0; i--)
4242	CPUMPushHyper(pVCpu, paParam[i]);
4243
4244	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
4245
4246	/* Call the switcher. */
4247	rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
4248	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
4249
4250	/** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
4251	/* Make sure the VMX instructions don't cause #UD faults. */
4252	ASMSetCR4(ASMGetCR4() \| X86_CR4_VMXE);
4253
4254	/* Re-enter VMX Root Mode */
4255	rc2 = VMXEnable(HCPhysCpuPage);
4256	if (RT_FAILURE(rc2))
4257	{
4258	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4259	ASMSetFlags(uOldEFlags);
4260	return rc2;
4261	}
4262
4263	rc2 = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
4264	AssertRC(rc2);
4265	Assert(!(ASMGetFlags() & X86_EFL_IF));
4266	ASMSetFlags(uOldEFlags);
4267	return rc;
4268	}
4269
4270
4271	/**
4272	* Prepares for and executes VMLAUNCH (64 bits guests) for 32-bit hosts
4273	* supporting 64-bit guests.
4274	*
4275	* @returns VBox status code.
4276	* @param fResume Whether to VMLAUNCH or VMRESUME.
4277	* @param pCtx Pointer to the guest-CPU context.
4278	* @param pCache Pointer to the VMCS cache.
4279	* @param pVM Pointer to the VM.
4280	* @param pVCpu Pointer to the VMCPU.
4281	*/
4282	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
4283	{
4284	uint32_t aParam[6];
4285	PHMGLOBLCPUINFO pCpu = NULL;
4286	RTHCPHYS HCPhysCpuPage = 0;
4287	int rc = VERR_INTERNAL_ERROR_5;
4288
4289	pCpu = HMR0GetCurrentCpu();
4290	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4291
4292	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4293	pCache->uPos = 1;
4294	pCache->interPD = PGMGetInterPaeCR3(pVM);
4295	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
4296	#endif
4297
4298	#ifdef VBOX_STRICT
4299	pCache->TestIn.HCPhysCpuPage = 0;
4300	pCache->TestIn.HCPhysVmcs = 0;
4301	pCache->TestIn.pCache = 0;
4302	pCache->TestOut.HCPhysVmcs = 0;
4303	pCache->TestOut.pCache = 0;
4304	pCache->TestOut.pCtx = 0;
4305	pCache->TestOut.eflags = 0;
4306	#endif
4307
4308	aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
4309	aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
4310	aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
4311	aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
4312	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
4313	aParam[5] = 0;
4314
4315	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4316	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
4317	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
4318	#endif
4319	rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, 6, &aParam[0]);
4320
4321	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4322	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
4323	Assert(pCtx->dr[4] == 10);
4324	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
4325	#endif
4326
4327	#ifdef VBOX_STRICT
4328	AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
4329	AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4330	pVCpu->hm.s.vmx.HCPhysVmcs));
4331	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4332	pCache->TestOut.HCPhysVmcs));
4333	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
4334	pCache->TestOut.pCache));
4335	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
4336	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
4337	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
4338	pCache->TestOut.pCtx));
4339	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
4340	#endif
4341	return rc;
4342	}
4343
4344
4345	/**
4346	* Initialize the VMCS-Read cache. The VMCS cache is used for 32-bit hosts
4347	* running 64-bit guests (except 32-bit Darwin which runs with 64-bit paging in
4348	* 32-bit mode) for 64-bit fields that cannot be accessed in 32-bit mode. Some
4349	* 64-bit fields -can- be accessed (those that have a 32-bit FULL & HIGH part).
4350	*
4351	* @returns VBox status code.
4352	* @param pVM Pointer to the VM.
4353	* @param pVCpu Pointer to the VMCPU.
4354	*/
4355	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
4356	{
4357	#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
4358	{ \
4359	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
4360	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
4361	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
4362	++cReadFields; \
4363	}
4364
4365	AssertPtr(pVM);
4366	AssertPtr(pVCpu);
4367	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4368	uint32_t cReadFields = 0;
4369
4370	/*
4371	* Don't remove the #if 0'd fields in this code. They're listed here for consistency
4372	* and serve to indicate exceptions to the rules.
4373	*/
4374
4375	/* Guest-natural selector base fields. */
4376	#if 0
4377	/* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
4378	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
4379	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
4380	#endif
4381	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
4382	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
4383	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
4384	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
4385	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
4386	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
4387	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
4388	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
4389	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
4390	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
4391	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
4392	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
4393	#if 0
4394	/* Unused natural width guest-state fields. */
4395	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
4396	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
4397	#endif
4398	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
4399	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
4400
4401	/* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
4402	#if 0
4403	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
4404	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
4405	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
4406	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
4407	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
4408	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
4409	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
4410	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
4411	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
4412	#endif
4413
4414	/* Natural width guest-state fields. */
4415	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
4416	#if 0
4417	/* Currently unused field. */
4418	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
4419	#endif
4420
4421	if (pVM->hm.s.fNestedPaging)
4422	{
4423	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
4424	AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
4425	VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
4426	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
4427	}
4428	else
4429	{
4430	AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
4431	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
4432	}
4433
4434	#undef VMXLOCAL_INIT_READ_CACHE_FIELD
4435	return VINF_SUCCESS;
4436	}
4437
4438
4439	/**
4440	* Writes a field into the VMCS. This can either directly invoke a VMWRITE or
4441	* queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
4442	* darwin, running 64-bit guests).
4443	*
4444	* @returns VBox status code.
4445	* @param pVCpu Pointer to the VMCPU.
4446	* @param idxField The VMCS field encoding.
4447	* @param u64Val 16, 32 or 64 bits value.
4448	*/
4449	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4450	{
4451	int rc;
4452	switch (idxField)
4453	{
4454	/*
4455	* These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
4456	*/
4457	/* 64-bit Control fields. */
4458	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
4459	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
4460	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
4461	case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
4462	case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
4463	case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
4464	case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
4465	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
4466	case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
4467	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
4468	case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
4469	case VMX_VMCS64_CTRL_EPTP_FULL:
4470	case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
4471	/* 64-bit Guest-state fields. */
4472	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
4473	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
4474	case VMX_VMCS64_GUEST_PAT_FULL:
4475	case VMX_VMCS64_GUEST_EFER_FULL:
4476	case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
4477	case VMX_VMCS64_GUEST_PDPTE0_FULL:
4478	case VMX_VMCS64_GUEST_PDPTE1_FULL:
4479	case VMX_VMCS64_GUEST_PDPTE2_FULL:
4480	case VMX_VMCS64_GUEST_PDPTE3_FULL:
4481	/* 64-bit Host-state fields. */
4482	case VMX_VMCS64_HOST_FIELD_PAT_FULL:
4483	case VMX_VMCS64_HOST_FIELD_EFER_FULL:
4484	case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
4485	{
4486	rc = VMXWriteVmcs32(idxField, u64Val);
4487	rc \|= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
4488	break;
4489	}
4490
4491	/*
4492	* These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
4493	* values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
4494	*/
4495	/* Natural-width Guest-state fields. */
4496	case VMX_VMCS_GUEST_CR3:
4497	case VMX_VMCS_GUEST_ES_BASE:
4498	case VMX_VMCS_GUEST_CS_BASE:
4499	case VMX_VMCS_GUEST_SS_BASE:
4500	case VMX_VMCS_GUEST_DS_BASE:
4501	case VMX_VMCS_GUEST_FS_BASE:
4502	case VMX_VMCS_GUEST_GS_BASE:
4503	case VMX_VMCS_GUEST_LDTR_BASE:
4504	case VMX_VMCS_GUEST_TR_BASE:
4505	case VMX_VMCS_GUEST_GDTR_BASE:
4506	case VMX_VMCS_GUEST_IDTR_BASE:
4507	case VMX_VMCS_GUEST_RSP:
4508	case VMX_VMCS_GUEST_RIP:
4509	case VMX_VMCS_GUEST_SYSENTER_ESP:
4510	case VMX_VMCS_GUEST_SYSENTER_EIP:
4511	{
4512	if (!(u64Val >> 32))
4513	{
4514	/* If this field is 64-bit, VT-x will zero out the top bits. */
4515	rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
4516	}
4517	else
4518	{
4519	/* Assert that only the 32->64 switcher case should ever come here. */
4520	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
4521	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
4522	}
4523	break;
4524	}
4525
4526	default:
4527	{
4528	AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
4529	rc = VERR_INVALID_PARAMETER;
4530	break;
4531	}
4532	}
4533	AssertRCReturn(rc, rc);
4534	return rc;
4535	}
4536
4537
4538	/**
4539	* Queue up a VMWRITE by using the VMCS write cache. This is only used on 32-bit
4540	* hosts (except darwin) for 64-bit guests.
4541	*
4542	* @param pVCpu Pointer to the VMCPU.
4543	* @param idxField The VMCS field encoding.
4544	* @param u64Val 16, 32 or 64 bits value.
4545	*/
4546	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4547	{
4548	AssertPtr(pVCpu);
4549	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4550
4551	AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
4552	("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
4553
4554	/* Make sure there are no duplicates. */
4555	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
4556	{
4557	if (pCache->Write.aField[i] == idxField)
4558	{
4559	pCache->Write.aFieldVal[i] = u64Val;
4560	return VINF_SUCCESS;
4561	}
4562	}
4563
4564	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
4565	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
4566	pCache->Write.cValidEntries++;
4567	return VINF_SUCCESS;
4568	}
4569
4570	/* Enable later when the assembly code uses these as callbacks. */
4571	#if 0
4572	/*
4573	* Loads the VMCS write-cache into the CPU (by executing VMWRITEs).
4574	*
4575	* @param pVCpu Pointer to the VMCPU.
4576	* @param pCache Pointer to the VMCS cache.
4577	*
4578	* @remarks No-long-jump zone!!!
4579	*/
4580	VMMR0DECL(void) VMXWriteCachedVmcsLoad(PVMCPU pVCpu, PVMCSCACHE pCache)
4581	{
4582	AssertPtr(pCache);
4583	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
4584	{
4585	int rc = VMXWriteVmcs64(pCache->Write.aField[i], pCache->Write.aFieldVal[i]);
4586	AssertRC(rc);
4587	}
4588	pCache->Write.cValidEntries = 0;
4589	}
4590
4591
4592	/**
4593	* Stores the VMCS read-cache from the CPU (by executing VMREADs).
4594	*
4595	* @param pVCpu Pointer to the VMCPU.
4596	* @param pCache Pointer to the VMCS cache.
4597	*
4598	* @remarks No-long-jump zone!!!
4599	*/
4600	VMMR0DECL(void) VMXReadCachedVmcsStore(PVMCPU pVCpu, PVMCSCACHE pCache)
4601	{
4602	AssertPtr(pCache);
4603	for (uint32_t i = 0; i < pCache->Read.cValidEntries; i++)
4604	{
4605	int rc = VMXReadVmcs64(pCache->Read.aField[i], &pCache->Read.aFieldVal[i]);
4606	AssertRC(rc);
4607	}
4608	}
4609	#endif
4610	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
4611
4612
4613	/**
4614	* Sets up the usage of TSC-offsetting and updates the VMCS. If offsetting is
4615	* not possible, cause VM-exits on RDTSC(P)s. Also sets up the VMX preemption
4616	* timer.
4617	*
4618	* @returns VBox status code.
4619	* @param pVCpu Pointer to the VMCPU.
4620	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4621	* out-of-sync. Make sure to update the required fields
4622	* before using them.
4623	* @remarks No-long-jump zone!!!
4624	*/
4625	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4626	{
4627	int rc = VERR_INTERNAL_ERROR_5;
4628	bool fOffsettedTsc = false;
4629	PVM pVM = pVCpu->CTX_SUFF(pVM);
4630	if (pVM->hm.s.vmx.fUsePreemptTimer)
4631	{
4632	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &pVCpu->hm.s.vmx.u64TSCOffset);
4633
4634	/* Make sure the returned values have sane upper and lower boundaries. */
4635	uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
4636	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
4637	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
4638	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
4639
4640	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
4641	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
4642	}
4643	else
4644	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset);
4645
4646	if (fOffsettedTsc)
4647	{
4648	uint64_t u64CurTSC = ASMReadTSC();
4649	if (u64CurTSC + pVCpu->hm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
4650	{
4651	/* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
4652	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
4653
4654	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
4655	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4656	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
4657	}
4658	else
4659	{
4660	/* VM-exit on RDTSC(P) as we would otherwise pass decreasing TSC values to the guest. */
4661	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
4662	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4663	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscInterceptOverFlow);
4664	}
4665	}
4666	else
4667	{
4668	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
4669	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
4670	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4671	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
4672	}
4673	}
4674
4675
4676	/**
4677	* Determines if an exception is a contributory exception. Contributory
4678	* exceptions are ones which can cause double-faults. Page-fault is
4679	* intentionally not included here as it's a conditional contributory exception.
4680	*
4681	* @returns true if the exception is contributory, false otherwise.
4682	* @param uVector The exception vector.
4683	*/
4684	DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
4685	{
4686	switch (uVector)
4687	{
4688	case X86_XCPT_GP:
4689	case X86_XCPT_SS:
4690	case X86_XCPT_NP:
4691	case X86_XCPT_TS:
4692	case X86_XCPT_DE:
4693	return true;
4694	default:
4695	break;
4696	}
4697	return false;
4698	}
4699
4700
4701	/**
4702	* Sets an event as a pending event to be injected into the guest.
4703	*
4704	* @param pVCpu Pointer to the VMCPU.
4705	* @param u32IntrInfo The VM-entry interruption-information field.
4706	* @param cbInstr The VM-entry instruction length in bytes (for software
4707	* interrupts, exceptions and privileged software
4708	* exceptions).
4709	* @param u32ErrCode The VM-entry exception error code.
4710	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
4711	* page-fault.
4712	*/
4713	DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntrInfo, uint32_t cbInstr, uint32_t u32ErrCode,
4714	RTGCUINTPTR GCPtrFaultAddress)
4715	{
4716	Assert(!pVCpu->hm.s.Event.fPending);
4717	pVCpu->hm.s.Event.fPending = true;
4718	pVCpu->hm.s.Event.u64IntrInfo = u32IntrInfo;
4719	pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
4720	pVCpu->hm.s.Event.cbInstr = cbInstr;
4721	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
4722	}
4723
4724
4725	/**
4726	* Sets a double-fault (#DF) exception as pending-for-injection into the VM.
4727	*
4728	* @param pVCpu Pointer to the VMCPU.
4729	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4730	* out-of-sync. Make sure to update the required fields
4731	* before using them.
4732	*/
4733	DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4734	{
4735	uint32_t u32IntrInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
4736	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
4737	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
4738	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
4739	}
4740
4741
4742	/**
4743	* Handle a condition that occurred while delivering an event through the guest
4744	* IDT.
4745	*
4746	* @returns VBox status code (informational error codes included).
4747	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
4748	* @retval VINF_VMX_DOUBLE_FAULT if a #DF condition was detected and we ought to
4749	* continue execution of the guest which will delivery the #DF.
4750	* @retval VINF_EM_RESET if we detected a triple-fault condition.
4751	*
4752	* @param pVCpu Pointer to the VMCPU.
4753	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4754	* out-of-sync. Make sure to update the required fields
4755	* before using them.
4756	* @param pVmxTransient Pointer to the VMX transient structure.
4757	*
4758	* @remarks No-long-jump zone!!!
4759	*/
4760	static int hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
4761	{
4762	int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
4763	AssertRC(rc);
4764	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
4765	{
4766	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
4767	AssertRCReturn(rc, rc);
4768
4769	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
4770	uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntrInfo);
4771	uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
4772
4773	typedef enum
4774	{
4775	VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
4776	VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
4777	VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
4778	VMXREFLECTXCPT_NONE /* Nothing to reflect. */
4779	} VMXREFLECTXCPT;
4780
4781	/* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
4782	VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
4783	if (uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
4784	{
4785	enmReflect = VMXREFLECTXCPT_XCPT;
4786	#ifdef VBOX_STRICT
4787	if ( hmR0VmxIsContributoryXcpt(uIdtVector)
4788	&& uExitVector == X86_XCPT_PF)
4789	{
4790	Log4(("IDT: Contributory #PF uCR2=%#RX64\n", pMixedCtx->cr2));
4791	}
4792	#endif
4793	if ( uExitVector == X86_XCPT_PF
4794	&& uIdtVector == X86_XCPT_PF)
4795	{
4796	pVmxTransient->fVectoringPF = true;
4797	Log4(("IDT: Vectoring #PF uCR2=%#RX64\n", pMixedCtx->cr2));
4798	}
4799	else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
4800	&& hmR0VmxIsContributoryXcpt(uExitVector)
4801	&& ( hmR0VmxIsContributoryXcpt(uIdtVector)
4802	\|\| uIdtVector == X86_XCPT_PF))
4803	{
4804	enmReflect = VMXREFLECTXCPT_DF;
4805	}
4806	else if (uIdtVector == X86_XCPT_DF)
4807	enmReflect = VMXREFLECTXCPT_TF;
4808	}
4809	else if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
4810	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
4811	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
4812	{
4813	/*
4814	* Ignore software interrupts (INT n), software exceptions (#BP, #OF) and privileged software exception
4815	* (whatever they are) as they reoccur when restarting the instruction.
4816	*/
4817	enmReflect = VMXREFLECTXCPT_XCPT;
4818	}
4819
4820	switch (enmReflect)
4821	{
4822	case VMXREFLECTXCPT_XCPT:
4823	{
4824	uint32_t u32ErrCode = 0;
4825	if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntrInfo))
4826	{
4827	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
4828	AssertRCReturn(rc, rc);
4829	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
4830	}
4831
4832	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
4833	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INTR_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
4834	0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
4835	rc = VINF_SUCCESS;
4836	Log4(("IDT: Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->hm.s.Event.u64IntrInfo,
4837	pVCpu->hm.s.Event.u32ErrCode));
4838	break;
4839	}
4840
4841	case VMXREFLECTXCPT_DF:
4842	{
4843	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
4844	rc = VINF_VMX_DOUBLE_FAULT;
4845	Log4(("IDT: Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->hm.s.Event.u64IntrInfo,
4846	uIdtVector, uExitVector));
4847	break;
4848	}
4849
4850	case VMXREFLECTXCPT_TF:
4851	{
4852	Log4(("IDT: Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", uIdtVector, uExitVector));
4853	rc = VINF_EM_RESET;
4854	break;
4855	}
4856
4857	default:
4858	Assert(rc == VINF_SUCCESS);
4859	break;
4860	}
4861	}
4862	Assert(rc == VINF_SUCCESS \|\| rc == VINF_VMX_DOUBLE_FAULT \|\| rc == VINF_EM_RESET);
4863	return rc;
4864	}
4865
4866
4867	/**
4868	* Saves the guest's CR0 register from the VMCS into the guest-CPU context.
4869	*
4870	* @returns VBox status code.
4871	* @param pVCpu Pointer to the VMCPU.
4872	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4873	* out-of-sync. Make sure to update the required fields
4874	* before using them.
4875	*
4876	* @remarks No-long-jump zone!!!
4877	*/
4878	static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4879	{
4880	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0))
4881	{
4882	uint32_t uVal = 0;
4883	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
4884	AssertRCReturn(rc, rc);
4885	uint32_t uShadow = 0;
4886	rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
4887	AssertRCReturn(rc, rc);
4888
4889	uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) \| (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
4890	CPUMSetGuestCR0(pVCpu, uVal);
4891	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR0;
4892	}
4893	return VINF_SUCCESS;
4894	}
4895
4896
4897	/**
4898	* Saves the guest's CR4 register from the VMCS into the guest-CPU context.
4899	*
4900	* @returns VBox status code.
4901	* @param pVCpu Pointer to the VMCPU.
4902	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4903	* out-of-sync. Make sure to update the required fields
4904	* before using them.
4905	*
4906	* @remarks No-long-jump zone!!!
4907	*/
4908	static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4909	{
4910	int rc = VINF_SUCCESS;
4911	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR4))
4912	{
4913	uint32_t uVal = 0;
4914	uint32_t uShadow = 0;
4915	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
4916	AssertRCReturn(rc, rc);
4917	rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
4918	AssertRCReturn(rc, rc);
4919
4920	uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) \| (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
4921	CPUMSetGuestCR4(pVCpu, uVal);
4922	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR4;
4923	}
4924	return rc;
4925	}
4926
4927
4928	/**
4929	* Saves the guest's RIP register from the VMCS into the guest-CPU context.
4930	*
4931	* @returns VBox status code.
4932	* @param pVCpu Pointer to the VMCPU.
4933	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4934	* out-of-sync. Make sure to update the required fields
4935	* before using them.
4936	*
4937	* @remarks No-long-jump zone!!!
4938	*/
4939	static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4940	{
4941	int rc = VINF_SUCCESS;
4942	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RIP))
4943	{
4944	uint64_t u64Val = 0;
4945	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
4946	AssertRCReturn(rc, rc);
4947
4948	pMixedCtx->rip = u64Val;
4949	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RIP;
4950	}
4951	return rc;
4952	}
4953
4954
4955	/**
4956	* Saves the guest's RSP register from the VMCS into the guest-CPU context.
4957	*
4958	* @returns VBox status code.
4959	* @param pVCpu Pointer to the VMCPU.
4960	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4961	* out-of-sync. Make sure to update the required fields
4962	* before using them.
4963	*
4964	* @remarks No-long-jump zone!!!
4965	*/
4966	static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4967	{
4968	int rc = VINF_SUCCESS;
4969	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RSP))
4970	{
4971	uint64_t u64Val = 0;
4972	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
4973	AssertRCReturn(rc, rc);
4974
4975	pMixedCtx->rsp = u64Val;
4976	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RSP;
4977	}
4978	return rc;
4979	}
4980
4981
4982	/**
4983	* Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
4984	*
4985	* @returns VBox status code.
4986	* @param pVCpu Pointer to the VMCPU.
4987	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4988	* out-of-sync. Make sure to update the required fields
4989	* before using them.
4990	*
4991	* @remarks No-long-jump zone!!!
4992	*/
4993	static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4994	{
4995	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS))
4996	{
4997	uint32_t uVal = 0;
4998	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
4999	AssertRCReturn(rc, rc);
5000
5001	pMixedCtx->eflags.u32 = uVal;
5002	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
5003	{
5004	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
5005	Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
5006
5007	pMixedCtx->eflags.Bits.u1VM = 0;
5008	pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.eflags.Bits.u2IOPL;
5009	}
5010
5011	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RFLAGS;
5012	}
5013	return VINF_SUCCESS;
5014	}
5015
5016
5017	/**
5018	* Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
5019	* guest-CPU context.
5020	*/
5021	DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5022	{
5023	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5024	rc \|= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
5025	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
5026	return rc;
5027	}
5028
5029
5030	/**
5031	* Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
5032	* from the guest-state area in the VMCS.
5033	*
5034	* @param pVCpu Pointer to the VMCPU.
5035	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5036	* out-of-sync. Make sure to update the required fields
5037	* before using them.
5038	*
5039	* @remarks No-long-jump zone!!!
5040	*/
5041	static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5042	{
5043	uint32_t uIntrState = 0;
5044	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
5045	AssertRC(rc);
5046
5047	if (!uIntrState)
5048	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
5049	else
5050	{
5051	Assert( uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
5052	\|\| uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
5053	rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5054	AssertRC(rc);
5055	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
5056	AssertRC(rc);
5057
5058	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
5059	Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
5060	}
5061	}
5062
5063
5064	/**
5065	* Saves the guest's activity state.
5066	*
5067	* @returns VBox status code.
5068	* @param pVCpu Pointer to the VMCPU.
5069	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5070	* out-of-sync. Make sure to update the required fields
5071	* before using them.
5072	*
5073	* @remarks No-long-jump zone!!!
5074	*/
5075	static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5076	{
5077	/* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
5078	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_ACTIVITY_STATE;
5079	return VINF_SUCCESS;
5080	}
5081
5082
5083	/**
5084	* Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
5085	* the current VMCS into the guest-CPU context.
5086	*
5087	* @returns VBox status code.
5088	* @param pVCpu Pointer to the VMCPU.
5089	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5090	* out-of-sync. Make sure to update the required fields
5091	* before using them.
5092	*
5093	* @remarks No-long-jump zone!!!
5094	*/
5095	static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5096	{
5097	int rc = VINF_SUCCESS;
5098	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
5099	{
5100	uint32_t u32Val = 0;
5101	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
5102	pMixedCtx->SysEnter.cs = u32Val;
5103	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR;
5104	}
5105
5106	uint64_t u64Val = 0;
5107	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
5108	{
5109	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
5110	pMixedCtx->SysEnter.eip = u64Val;
5111	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR;
5112	}
5113	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
5114	{
5115	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
5116	pMixedCtx->SysEnter.esp = u64Val;
5117	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR;
5118	}
5119	return rc;
5120	}
5121
5122
5123	/**
5124	* Saves the guest FS_BASE MSRs from the current VMCS into the guest-CPU
5125	* context.
5126	*
5127	* @returns VBox status code.
5128	* @param pVCpu Pointer to the VMCPU.
5129	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5130	* out-of-sync. Make sure to update the required fields
5131	* before using them.
5132	*
5133	* @remarks No-long-jump zone!!!
5134	*/
5135	static int hmR0VmxSaveGuestFSBaseMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5136	{
5137	int rc = VINF_SUCCESS;
5138	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_FS_BASE_MSR))
5139	{
5140	uint64_t u64Val = 0;
5141	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_FS_BASE, &u64Val); AssertRCReturn(rc, rc);
5142	pMixedCtx->fs.u64Base = u64Val;
5143	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_FS_BASE_MSR;
5144	}
5145	return rc;
5146	}
5147
5148
5149	/**
5150	* Saves the guest GS_BASE MSRs from the current VMCS into the guest-CPU
5151	* context.
5152	*
5153	* @returns VBox status code.
5154	* @param pVCpu Pointer to the VMCPU.
5155	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5156	* out-of-sync. Make sure to update the required fields
5157	* before using them.
5158	*
5159	* @remarks No-long-jump zone!!!
5160	*/
5161	static int hmR0VmxSaveGuestGSBaseMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5162	{
5163	int rc = VINF_SUCCESS;
5164	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_GS_BASE_MSR))
5165	{
5166	uint64_t u64Val = 0;
5167	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GS_BASE, &u64Val); AssertRCReturn(rc, rc);
5168	pMixedCtx->gs.u64Base = u64Val;
5169	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_GS_BASE_MSR;
5170	}
5171	return rc;
5172	}
5173
5174
5175	/**
5176	* Saves the auto load/store'd guest MSRs from the current VMCS into the
5177	* guest-CPU context. Currently these are LSTAR, STAR, SFMASK, KERNEL-GS BASE
5178	* and TSC_AUX.
5179	*
5180	* @returns VBox status code.
5181	* @param pVCpu Pointer to the VMCPU.
5182	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5183	* out-of-sync. Make sure to update the required fields
5184	* before using them.
5185	*
5186	* @remarks No-long-jump zone!!!
5187	*/
5188	static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5189	{
5190	if (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS)
5191	return VINF_SUCCESS;
5192
5193	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
5194	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cGuestMsrs; i++)
5195	{
5196	PVMXMSR pMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvGuestMsr;
5197	pMsr += i;
5198	switch (pMsr->u32IndexMSR)
5199	{
5200	case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
5201	case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
5202	case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
5203	case MSR_K8_TSC_AUX: CPUMSetGuestMsr(pVCpu, MSR_K8_TSC_AUX, pMsr->u64Value); break;
5204	case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
5205	case MSR_K6_EFER: /* EFER can't be changed without causing a VM-exit. */ break;
5206	default:
5207	{
5208	AssertFailed();
5209	return VERR_HM_UNEXPECTED_LD_ST_MSR;
5210	}
5211	}
5212	}
5213	#endif
5214
5215	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS;
5216	return VINF_SUCCESS;
5217	}
5218
5219
5220	/**
5221	* Saves the guest control registers from the current VMCS into the guest-CPU
5222	* context.
5223	*
5224	* @returns VBox status code.
5225	* @param pVCpu Pointer to the VMCPU.
5226	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5227	* out-of-sync. Make sure to update the required fields
5228	* before using them.
5229	*
5230	* @remarks No-long-jump zone!!!
5231	*/
5232	static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5233	{
5234	/* Guest CR0. Guest FPU. */
5235	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5236	AssertRCReturn(rc, rc);
5237
5238	/* Guest CR4. */
5239	rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
5240	AssertRCReturn(rc, rc);
5241
5242	/* Guest CR2 - updated always during the world-switch or in #PF. */
5243	/* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
5244	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR3))
5245	{
5246	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0);
5247	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR4);
5248
5249	PVM pVM = pVCpu->CTX_SUFF(pVM);
5250	if ( pVM->hm.s.vmx.fUnrestrictedGuest
5251	\|\| ( pVM->hm.s.fNestedPaging
5252	&& CPUMIsGuestPagingEnabledEx(pMixedCtx)))
5253	{
5254	uint64_t u64Val = 0;
5255	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
5256	if (pMixedCtx->cr3 != u64Val)
5257	{
5258	CPUMSetGuestCR3(pVCpu, u64Val);
5259	if (VMMRZCallRing3IsEnabled(pVCpu))
5260	{
5261	PGMUpdateCR3(pVCpu, u64Val);
5262	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5263	}
5264	else
5265	{
5266	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
5267	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
5268	}
5269	}
5270
5271	/* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
5272	if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
5273	{
5274	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
5275	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
5276	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
5277	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
5278
5279	if (VMMRZCallRing3IsEnabled(pVCpu))
5280	{
5281	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5282	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5283	}
5284	else
5285	{
5286	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
5287	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
5288	}
5289	}
5290	}
5291
5292	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR3;
5293	}
5294
5295	/*
5296	* Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
5297	* -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
5298	* -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
5299	*
5300	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
5301	*/
5302	if (VMMRZCallRing3IsEnabled(pVCpu))
5303	{
5304	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
5305	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
5306
5307	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
5308	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5309
5310	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5311	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5312	}
5313
5314	return rc;
5315	}
5316
5317
5318	/**
5319	* Reads a guest segment register from the current VMCS into the guest-CPU
5320	* context.
5321	*
5322	* @returns VBox status code.
5323	* @param pVCpu Pointer to the VMCPU.
5324	* @param idxSel Index of the selector in the VMCS.
5325	* @param idxLimit Index of the segment limit in the VMCS.
5326	* @param idxBase Index of the segment base in the VMCS.
5327	* @param idxAccess Index of the access rights of the segment in the VMCS.
5328	* @param pSelReg Pointer to the segment selector.
5329	*
5330	* @remarks No-long-jump zone!!!
5331	* @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
5332	* macro as that takes care of whether to read from the VMCS cache or
5333	* not.
5334	*/
5335	DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
5336	PCPUMSELREG pSelReg)
5337	{
5338	uint32_t u32Val = 0;
5339	int rc = VMXReadVmcs32(idxSel, &u32Val);
5340	AssertRCReturn(rc, rc);
5341	pSelReg->Sel = (uint16_t)u32Val;
5342	pSelReg->ValidSel = (uint16_t)u32Val;
5343	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
5344
5345	rc = VMXReadVmcs32(idxLimit, &u32Val);
5346	AssertRCReturn(rc, rc);
5347	pSelReg->u32Limit = u32Val;
5348
5349	uint64_t u64Val = 0;
5350	rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
5351	AssertRCReturn(rc, rc);
5352	pSelReg->u64Base = u64Val;
5353
5354	rc = VMXReadVmcs32(idxAccess, &u32Val);
5355	AssertRCReturn(rc, rc);
5356	pSelReg->Attr.u = u32Val;
5357
5358	/*
5359	* If VT-x marks the segment as unusable, the rest of the attributes are undefined with certain exceptions (some bits in
5360	* CS, SS). Regardless, we have to clear the bits here and only retain the unusable bit because the unusable bit is specific
5361	* to VT-x, everyone else relies on the attribute being zero and have no clue what the unusable bit is.
5362	*
5363	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
5364	*/
5365	if (pSelReg->Attr.u & HMVMX_SEL_UNUSABLE)
5366	{
5367	Assert(idxSel != VMX_VMCS16_GUEST_FIELD_TR); /* TR is the only selector that can never be unusable. */
5368	pSelReg->Attr.u = HMVMX_SEL_UNUSABLE;
5369	}
5370	return VINF_SUCCESS;
5371	}
5372
5373
5374	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
5375	#define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5376	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5377	VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5378	#else
5379	#define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5380	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5381	VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5382	#endif
5383
5384
5385	/**
5386	* Saves the guest segment registers from the current VMCS into the guest-CPU
5387	* context.
5388	*
5389	* @returns VBox status code.
5390	* @param pVCpu Pointer to the VMCPU.
5391	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5392	* out-of-sync. Make sure to update the required fields
5393	* before using them.
5394	*
5395	* @remarks No-long-jump zone!!!
5396	*/
5397	static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5398	{
5399	/* Guest segment registers. */
5400	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SEGMENT_REGS))
5401	{
5402	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); AssertRCReturn(rc, rc);
5403	rc = VMXLOCAL_READ_SEG(CS, cs); AssertRCReturn(rc, rc);
5404	rc = VMXLOCAL_READ_SEG(SS, ss); AssertRCReturn(rc, rc);
5405	rc = VMXLOCAL_READ_SEG(DS, ds); AssertRCReturn(rc, rc);
5406	rc = VMXLOCAL_READ_SEG(ES, es); AssertRCReturn(rc, rc);
5407	rc = VMXLOCAL_READ_SEG(FS, fs); AssertRCReturn(rc, rc);
5408	rc = VMXLOCAL_READ_SEG(GS, gs); AssertRCReturn(rc, rc);
5409
5410	/* Restore segment attributes for real-on-v86 mode hack. */
5411	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5412	{
5413	pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrCS.u;
5414	pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrSS.u;
5415	pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrDS.u;
5416	pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrES.u;
5417	pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrFS.u;
5418	pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrGS.u;
5419	}
5420	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SEGMENT_REGS;
5421	}
5422
5423	return VINF_SUCCESS;
5424	}
5425
5426
5427	/**
5428	* Saves the guest descriptor table registers and task register from the current
5429	* VMCS into the guest-CPU context.
5430	*
5431	* @returns VBox status code.
5432	* @param pVCpu Pointer to the VMCPU.
5433	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5434	* out-of-sync. Make sure to update the required fields
5435	* before using them.
5436	*
5437	* @remarks No-long-jump zone!!!
5438	*/
5439	static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5440	{
5441	int rc = VINF_SUCCESS;
5442
5443	/* Guest LDTR. */
5444	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_LDTR))
5445	{
5446	rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
5447	AssertRCReturn(rc, rc);
5448	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_LDTR;
5449	}
5450
5451	/* Guest GDTR. */
5452	uint64_t u64Val = 0;
5453	uint32_t u32Val = 0;
5454	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_GDTR))
5455	{
5456	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
5457	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5458	pMixedCtx->gdtr.pGdt = u64Val;
5459	pMixedCtx->gdtr.cbGdt = u32Val;
5460	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_GDTR;
5461	}
5462
5463	/* Guest IDTR. */
5464	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_IDTR))
5465	{
5466	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
5467	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5468	pMixedCtx->idtr.pIdt = u64Val;
5469	pMixedCtx->idtr.cbIdt = u32Val;
5470	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_IDTR;
5471	}
5472
5473	/* Guest TR. */
5474	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_TR))
5475	{
5476	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5477	AssertRCReturn(rc, rc);
5478
5479	/* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
5480	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5481	{
5482	rc = VMXLOCAL_READ_SEG(TR, tr);
5483	AssertRCReturn(rc, rc);
5484	}
5485	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_TR;
5486	}
5487	return rc;
5488	}
5489
5490	#undef VMXLOCAL_READ_SEG
5491
5492
5493	/**
5494	* Saves the guest debug registers from the current VMCS into the guest-CPU
5495	* context.
5496	*
5497	* @returns VBox status code.
5498	* @param pVCpu Pointer to the VMCPU.
5499	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5500	* out-of-sync. Make sure to update the required fields
5501	* before using them.
5502	*
5503	* @remarks No-long-jump zone!!!
5504	*/
5505	static int hmR0VmxSaveGuestDebugRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5506	{
5507	int rc = VINF_SUCCESS;
5508	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_DEBUG))
5509	{
5510	/* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
5511	uint32_t u32Val;
5512	rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
5513	pMixedCtx->dr[7] = u32Val;
5514
5515	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_DEBUG;
5516	}
5517	return rc;
5518	}
5519
5520
5521	/**
5522	* Saves the guest APIC state from the currentl VMCS into the guest-CPU context.
5523	*
5524	* @returns VBox status code.
5525	* @param pVCpu Pointer to the VMCPU.
5526	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5527	* out-of-sync. Make sure to update the required fields
5528	* before using them.
5529	*
5530	* @remarks No-long-jump zone!!!
5531	*/
5532	static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5533	{
5534	/* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
5535	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_APIC_STATE;
5536	return VINF_SUCCESS;
5537	}
5538
5539
5540	/**
5541	* Saves the entire guest state from the currently active VMCS into the
5542	* guest-CPU context. This essentially VMREADs all guest-data.
5543	*
5544	* @returns VBox status code.
5545	* @param pVCpu Pointer to the VMCPU.
5546	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5547	* out-of-sync. Make sure to update the required fields
5548	* before using them.
5549	*/
5550	static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5551	{
5552	Assert(pVCpu);
5553	Assert(pMixedCtx);
5554
5555	if (pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL)
5556	return VINF_SUCCESS;
5557
5558	/* Though we can longjmp to ring-3 due to log-flushes here and get recalled again on the ring-3 callback path,
5559	there is no real need to. */
5560	if (VMMRZCallRing3IsEnabled(pVCpu))
5561	VMMR0LogFlushDisable(pVCpu);
5562	else
5563	Assert(VMMR0IsLogFlushDisabled(pVCpu));
5564	Log4Func(("\n"));
5565
5566	int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
5567	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5568
5569	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
5570	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5571
5572	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
5573	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5574
5575	rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
5576	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5577
5578	rc = hmR0VmxSaveGuestDebugRegs(pVCpu, pMixedCtx);
5579	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDebugRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5580
5581	rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
5582	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5583
5584	rc = hmR0VmxSaveGuestFSBaseMsr(pVCpu, pMixedCtx);
5585	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestFSBaseMsr failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5586
5587	rc = hmR0VmxSaveGuestGSBaseMsr(pVCpu, pMixedCtx);
5588	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestGSBaseMsr failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5589
5590	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
5591	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5592
5593	rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
5594	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5595
5596	rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
5597	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDebugRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5598
5599	AssertMsg(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL,
5600	("Missed guest state bits while saving state; residue %RX32\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
5601
5602	if (VMMRZCallRing3IsEnabled(pVCpu))
5603	VMMR0LogFlushEnable(pVCpu);
5604
5605	return rc;
5606	}
5607
5608
5609	/**
5610	* Check per-VM and per-VCPU force flag actions that require us to go back to
5611	* ring-3 for one reason or another.
5612	*
5613	* @returns VBox status code (information status code included).
5614	* @retval VINF_SUCCESS if we don't have any actions that require going back to
5615	* ring-3.
5616	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
5617	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
5618	* interrupts)
5619	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
5620	* all EMTs to be in ring-3.
5621	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
5622	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
5623	* to the EM loop.
5624	*
5625	* @param pVM Pointer to the VM.
5626	* @param pVCpu Pointer to the VMCPU.
5627	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5628	* out-of-sync. Make sure to update the required fields
5629	* before using them.
5630	*/
5631	static int hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5632	{
5633	Assert(VMMRZCallRing3IsEnabled(pVCpu));
5634
5635	int rc = VERR_INTERNAL_ERROR_5;
5636	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK \| VM_FF_REQUEST \| VM_FF_PGM_POOL_FLUSH_PENDING \| VM_FF_PDM_DMA)
5637	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK \| VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL
5638	\| VMCPU_FF_REQUEST \| VMCPU_FF_HM_UPDATE_CR3 \| VMCPU_FF_HM_UPDATE_PAE_PDPES))
5639	{
5640	/* We need the control registers now, make sure the guest-CPU context is updated. */
5641	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
5642	AssertRCReturn(rc, rc);
5643
5644	/* Pending HM CR3 sync. */
5645	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
5646	{
5647	rc = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
5648	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3);
5649	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5650	}
5651
5652	/* Pending HM PAE PDPEs. */
5653	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
5654	{
5655	rc = PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5656	AssertRC(rc);
5657	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5658	}
5659
5660	/* Pending PGM C3 sync. */
5661	if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
5662	{
5663	rc = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4, VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
5664	if (rc != VINF_SUCCESS)
5665	{
5666	AssertRC(rc);
5667	Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc=%d\n", rc));
5668	return rc;
5669	}
5670	}
5671
5672	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
5673	/* -XXX- what was that about single stepping? */
5674	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
5675	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
5676	{
5677	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
5678	rc = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
5679	Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc));
5680	return rc;
5681	}
5682
5683	/* Pending VM request packets, such as hardware interrupts. */
5684	if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
5685	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
5686	{
5687	Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
5688	return VINF_EM_PENDING_REQUEST;
5689	}
5690
5691	/* Pending PGM pool flushes. */
5692	if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
5693	{
5694	Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
5695	return VINF_PGM_POOL_FLUSH_PENDING;
5696	}
5697
5698	/* Pending DMA requests. */
5699	if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
5700	{
5701	Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
5702	return VINF_EM_RAW_TO_R3;
5703	}
5704	}
5705
5706	/* Paranoia. */
5707	Assert(rc != VERR_EM_INTERPRETER);
5708	return VINF_SUCCESS;
5709	}
5710
5711
5712	/**
5713	* Converts any TRPM trap into a pending VMX event. This is typically used when
5714	* entering from ring-3 (not longjmp returns).
5715	*
5716	* @param pVCpu Pointer to the VMCPU.
5717	*/
5718	static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
5719	{
5720	Assert(TRPMHasTrap(pVCpu));
5721	Assert(!pVCpu->hm.s.Event.fPending);
5722
5723	uint8_t uVector;
5724	TRPMEVENT enmTrpmEvent;
5725	RTGCUINT uErrCode;
5726	RTGCUINTPTR GCPtrFaultAddress;
5727	uint8_t cbInstr;
5728
5729	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
5730	AssertRC(rc);
5731
5732	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntrInfo. */
5733	uint32_t u32IntrInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
5734	if (enmTrpmEvent == TRPM_TRAP)
5735	{
5736	switch (uVector)
5737	{
5738	case X86_XCPT_BP:
5739	case X86_XCPT_OF:
5740	{
5741	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5742	break;
5743	}
5744
5745	case X86_XCPT_PF:
5746	case X86_XCPT_DF:
5747	case X86_XCPT_TS:
5748	case X86_XCPT_NP:
5749	case X86_XCPT_SS:
5750	case X86_XCPT_GP:
5751	case X86_XCPT_AC:
5752	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5753	/* no break! */
5754	default:
5755	{
5756	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5757	break;
5758	}
5759	}
5760	}
5761	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
5762	{
5763	if (uVector == X86_XCPT_NMI)
5764	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5765	else
5766	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5767	}
5768	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
5769	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5770	else
5771	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
5772
5773	rc = TRPMResetTrap(pVCpu);
5774	AssertRC(rc);
5775	Log4(("TRPM->HM event: u32IntrInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
5776	u32IntrInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
5777	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, uErrCode, GCPtrFaultAddress);
5778	}
5779
5780
5781	/**
5782	* Converts any pending VMX event into a TRPM trap. Typically used when leaving
5783	* VT-x to execute any instruction.
5784	*
5785	* @param pvCpu Pointer to the VMCPU.
5786	*/
5787	static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
5788	{
5789	Assert(pVCpu->hm.s.Event.fPending);
5790
5791	uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo);
5792	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntrInfo);
5793	bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntrInfo);
5794	uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
5795
5796	/* If a trap was already pending, we did something wrong! */
5797	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
5798
5799	TRPMEVENT enmTrapType;
5800	switch (uVectorType)
5801	{
5802	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
5803	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
5804	enmTrapType = TRPM_HARDWARE_INT;
5805	break;
5806	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
5807	enmTrapType = TRPM_SOFTWARE_INT;
5808	break;
5809	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
5810	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
5811	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
5812	enmTrapType = TRPM_TRAP;
5813	break;
5814	default:
5815	AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
5816	enmTrapType = TRPM_32BIT_HACK;
5817	break;
5818	}
5819
5820	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
5821
5822	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
5823	AssertRC(rc);
5824
5825	if (fErrorCodeValid)
5826	TRPMSetErrorCode(pVCpu, uErrorCode);
5827
5828	if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5829	&& uVector == X86_XCPT_PF)
5830	{
5831	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
5832	}
5833	else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5834	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5835	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
5836	{
5837	AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5838	\|\| (uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF),
5839	("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
5840	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
5841	}
5842	pVCpu->hm.s.Event.fPending = false;
5843	}
5844
5845
5846	/**
5847	* Does the necessary state syncing before doing a longjmp to ring-3.
5848	*
5849	* @param pVM Pointer to the VM.
5850	* @param pVCpu Pointer to the VMCPU.
5851	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5852	* out-of-sync. Make sure to update the required fields
5853	* before using them.
5854	* @param rcExit The reason for exiting to ring-3. Can be
5855	* VINF_VMM_UNKNOWN_RING3_CALL.
5856	*
5857	* @remarks No-long-jmp zone!!!
5858	*/
5859	static void hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
5860	{
5861	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
5862	Assert(VMMR0IsLogFlushDisabled(pVCpu));
5863
5864	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
5865	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL);
5866	AssertRC(rc);
5867
5868	/* Restore host FPU state if necessary and resync on next R0 reentry .*/
5869	if (CPUMIsGuestFPUStateActive(pVCpu))
5870	{
5871	CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
5872	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
5873	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
5874	}
5875
5876	/* Restore host debug registers if necessary and resync on next R0 reentry. */
5877	if (CPUMIsGuestDebugStateActive(pVCpu))
5878	{
5879	CPUMR0SaveGuestDebugState(pVM, pVCpu, pMixedCtx, true /* save DR6 */);
5880	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
5881	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
5882	}
5883	else if (CPUMIsHyperDebugStateActive(pVCpu))
5884	{
5885	CPUMR0LoadHostDebugState(pVM, pVCpu);
5886	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
5887	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
5888	}
5889
5890	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
5891	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
5892	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
5893	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
5894	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
5895	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
5896	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
5897	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
5898	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
5899	}
5900
5901
5902	/**
5903	* An action requires us to go back to ring-3. This function does the necessary
5904	* steps before we can safely return to ring-3. This is not the same as longjmps
5905	* to ring-3, this is voluntary.
5906	*
5907	* @param pVM Pointer to the VM.
5908	* @param pVCpu Pointer to the VMCPU.
5909	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5910	* out-of-sync. Make sure to update the required fields
5911	* before using them.
5912	* @param rcExit The reason for exiting to ring-3. Can be
5913	* VINF_VMM_UNKNOWN_RING3_CALL.
5914	*/
5915	static void hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
5916	{
5917	Assert(pVM);
5918	Assert(pVCpu);
5919	Assert(pMixedCtx);
5920	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5921
5922	if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_GUEST_STATE))
5923	{
5924	/* We want to see what the guest-state was before VM-entry, don't resync here, as we won't continue guest execution. */
5925	return;
5926	}
5927	else if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
5928	{
5929	VMXGetActivateVMCS(&pVCpu->hm.s.vmx.lasterror.u64VMCSPhys);
5930	pVCpu->hm.s.vmx.lasterror.u32VMCSRevision = (uint32_t )pVCpu->hm.s.vmx.pvVmcs;
5931	pVCpu->hm.s.vmx.lasterror.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
5932	pVCpu->hm.s.vmx.lasterror.idCurrentCpu = RTMpCpuId();
5933	return;
5934	}
5935
5936	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
5937	VMMRZCallRing3Disable(pVCpu);
5938	Log4(("hmR0VmxExitToRing3: rcExit=%d\n", rcExit));
5939
5940	/* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
5941	if (pVCpu->hm.s.Event.fPending)
5942	{
5943	hmR0VmxPendingEventToTrpmTrap(pVCpu);
5944	Assert(!pVCpu->hm.s.Event.fPending);
5945	}
5946
5947	/* Sync. the guest state. */
5948	hmR0VmxLongJmpToRing3(pVM, pVCpu, pMixedCtx, rcExit);
5949	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
5950
5951	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
5952	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
5953	\| CPUM_CHANGED_LDTR
5954	\| CPUM_CHANGED_GDTR
5955	\| CPUM_CHANGED_IDTR
5956	\| CPUM_CHANGED_TR
5957	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
5958
5959	/* On our way back from ring-3 the following needs to be done. */
5960	/** @todo This can change with preemption hooks. */
5961	if (rcExit == VINF_EM_RAW_INTERRUPT)
5962	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_HOST_CONTEXT;
5963	else
5964	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_ALL_GUEST;
5965
5966	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
5967	VMMRZCallRing3Enable(pVCpu);
5968	}
5969
5970
5971	/**
5972	* VMMRZCallRing3() callback wrapper which saves the guest state before we
5973	* longjump to ring-3 and possibly get preempted.
5974	*
5975	* @param pVCpu Pointer to the VMCPU.
5976	* @param enmOperation The operation causing the ring-3 longjump.
5977	* @param pvUser The user argument (pointer to the possibly
5978	* out-of-date guest-CPU context).
5979	*
5980	* @remarks Must never be called with @a enmOperation ==
5981	* VMMCALLRING3_VM_R0_ASSERTION.
5982	*/
5983	DECLCALLBACK(void) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
5984	{
5985	/* VMMRZCallRing3() already makes sure we never get called as a result of an longjmp due to an assertion. */
5986	Assert(pVCpu);
5987	Assert(pvUser);
5988	Assert(VMMRZCallRing3IsEnabled(pVCpu));
5989	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5990
5991	VMMRZCallRing3Disable(pVCpu);
5992	Assert(VMMR0IsLogFlushDisabled(pVCpu));
5993	Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3\n"));
5994	hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser, VINF_VMM_UNKNOWN_RING3_CALL);
5995	VMMRZCallRing3Enable(pVCpu);
5996	}
5997
5998
5999	/**
6000	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
6001	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
6002	*
6003	* @param pVCpu Pointer to the VMCPU.
6004	*/
6005	DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
6006	{
6007	if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
6008	{
6009	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
6010	{
6011	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
6012	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
6013	AssertRC(rc);
6014	}
6015	} /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
6016	}
6017
6018
6019	/**
6020	* Injects any pending events into the guest if the guest is in a state to
6021	* receive them.
6022	*
6023	* @returns VBox status code (informational status codes included).
6024	* @param pVCpu Pointer to the VMCPU.
6025	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6026	* out-of-sync. Make sure to update the required fields
6027	* before using them.
6028	*/
6029	static int hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6030	{
6031	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
6032	uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
6033	bool fBlockMovSS = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6034	bool fBlockSti = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
6035
6036	Assert(!fBlockSti \|\| (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS));
6037	Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
6038	&& !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
6039	Assert(!fBlockSti \|\| pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
6040	Assert(!TRPMHasTrap(pVCpu));
6041
6042	int rc = VINF_SUCCESS;
6043	if (pVCpu->hm.s.Event.fPending) /* First, inject any pending HM events. */
6044	{
6045	uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo);
6046	bool fInject = true;
6047	if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
6048	{
6049	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6050	AssertRCReturn(rc, rc);
6051	const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
6052	if ( fBlockInt
6053	\|\| fBlockSti
6054	\|\| fBlockMovSS)
6055	{
6056	fInject = false;
6057	}
6058	}
6059	else if ( uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
6060	&& ( fBlockMovSS
6061	\|\| fBlockSti))
6062	{
6063	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
6064	fInject = false;
6065	}
6066
6067	if (fInject)
6068	{
6069	Log4(("Injecting pending event\n"));
6070	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntrInfo, pVCpu->hm.s.Event.cbInstr,
6071	pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, &uIntrState);
6072	AssertRCReturn(rc, rc);
6073	pVCpu->hm.s.Event.fPending = false;
6074	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntReinject);
6075	}
6076	else
6077	hmR0VmxSetIntWindowExitVmcs(pVCpu);
6078	} /** @todo SMI. SMIs take priority over NMIs. */
6079	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts . */
6080	{
6081	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
6082	if ( !fBlockMovSS
6083	&& !fBlockSti)
6084	{
6085	Log4(("Injecting NMI\n"));
6086	uint32_t u32IntrInfo = X86_XCPT_NMI \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6087	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6088	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */,
6089	0 /* GCPtrFaultAddress */, &uIntrState);
6090	AssertRCReturn(rc, rc);
6091	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
6092	}
6093	else
6094	hmR0VmxSetIntWindowExitVmcs(pVCpu);
6095	}
6096	else if (VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)))
6097	{
6098	/* Check if there are guest external interrupts (PIC/APIC) pending and inject them if the guest can receive them. */
6099	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6100	AssertRCReturn(rc, rc);
6101	const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
6102	if ( !fBlockInt
6103	&& !fBlockSti
6104	&& !fBlockMovSS)
6105	{
6106	uint8_t u8Interrupt;
6107	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
6108	if (RT_SUCCESS(rc))
6109	{
6110	Log4(("Injecting interrupt u8Interrupt=%#x\n", u8Interrupt));
6111	uint32_t u32IntrInfo = u8Interrupt \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6112	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6113	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */,
6114	0 /* GCPtrFaultAddress */, &uIntrState);
6115	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
6116	}
6117	else
6118	{
6119	/** @todo Does this actually happen? If not turn it into an assertion. */
6120	Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)));
6121	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
6122	rc = VINF_SUCCESS;
6123	}
6124	}
6125	else
6126	hmR0VmxSetIntWindowExitVmcs(pVCpu);
6127	}
6128
6129	/*
6130	* Delivery pending debug exception if the guest is single-stepping. The interruptibility-state could have been changed by
6131	* hmR0VmxInjectEventVmcs() (e.g. real-on-v86 injecting software interrupts), re-evaluate it and set the BS bit.
6132	*/
6133	fBlockMovSS = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6134	fBlockSti = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
6135	int rc2 = VINF_SUCCESS;
6136	if ( fBlockSti
6137	\|\| fBlockMovSS)
6138	{
6139	if (!DBGFIsStepping(pVCpu))
6140	{
6141	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS);
6142	if (pMixedCtx->eflags.Bits.u1TF) /* We don't have any IA32_DEBUGCTL MSR for guests. Treat as all bits 0. */
6143	{
6144	/*
6145	* The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD, VMX_EXIT_MTF
6146	* VMX_EXIT_APIC_WRITE, VMX_EXIT_VIRTUALIZED_EOI. See Intel spec. 27.3.4 "Saving Non-Register State".
6147	*/
6148	rc2 = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
6149	AssertRCReturn(rc, rc);
6150	}
6151	}
6152	else
6153	{
6154	/* We are single-stepping in the hypervisor debugger, clear interrupt inhibition as setting the BS bit would mean
6155	delivering a #DB to the guest upon VM-entry when it shouldn't be. */
6156	uIntrState = 0;
6157	}
6158	}
6159
6160	/*
6161	* There's no need to clear the VM entry-interruption information field here if we're not injecting anything.
6162	* VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
6163	*/
6164	rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
6165	AssertRC(rc2);
6166
6167	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
6168	return rc;
6169	}
6170
6171
6172	/**
6173	* Sets an invalid-opcode (#UD) exception as pending-for-injection into the VM.
6174	*
6175	* @param pVCpu Pointer to the VMCPU.
6176	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6177	* out-of-sync. Make sure to update the required fields
6178	* before using them.
6179	*/
6180	DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6181	{
6182	uint32_t u32IntrInfo = X86_XCPT_UD \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6183	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6184	}
6185
6186
6187	/**
6188	* Injects a double-fault (#DF) exception into the VM.
6189	*
6190	* @returns VBox status code (informational status code included).
6191	* @param pVCpu Pointer to the VMCPU.
6192	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6193	* out-of-sync. Make sure to update the required fields
6194	* before using them.
6195	*/
6196	DECLINLINE(int) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t *puIntrState)
6197	{
6198	uint32_t u32IntrInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6199	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6200	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6201	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */,
6202	puIntrState);
6203	}
6204
6205
6206	/**
6207	* Sets a debug (#DB) exception as pending-for-injection into the VM.
6208	*
6209	* @param pVCpu Pointer to the VMCPU.
6210	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6211	* out-of-sync. Make sure to update the required fields
6212	* before using them.
6213	*/
6214	DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6215	{
6216	uint32_t u32IntrInfo = X86_XCPT_DB \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6217	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6218	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6219	}
6220
6221
6222	/**
6223	* Sets an overflow (#OF) exception as pending-for-injection into the VM.
6224	*
6225	* @param pVCpu Pointer to the VMCPU.
6226	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6227	* out-of-sync. Make sure to update the required fields
6228	* before using them.
6229	* @param cbInstr The value of RIP that is to be pushed on the guest
6230	* stack.
6231	*/
6232	DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
6233	{
6234	uint32_t u32IntrInfo = X86_XCPT_OF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6235	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6236	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
6237	}
6238
6239
6240	/**
6241	* Injects a general-protection (#GP) fault into the VM.
6242	*
6243	* @returns VBox status code (informational status code included).
6244	* @param pVCpu Pointer to the VMCPU.
6245	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6246	* out-of-sync. Make sure to update the required fields
6247	* before using them.
6248	* @param u32ErrorCode The error code associated with the #GP.
6249	*/
6250	DECLINLINE(int) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
6251	uint32_t *puIntrState)
6252	{
6253	uint32_t u32IntrInfo = X86_XCPT_GP \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6254	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6255	if (fErrorCodeValid)
6256	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6257	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, u32ErrorCode, 0 / GCPtrFaultAddress */,
6258	puIntrState);
6259	}
6260
6261
6262	/**
6263	* Sets a software interrupt (INTn) as pending-for-injection into the VM.
6264	*
6265	* @param pVCpu Pointer to the VMCPU.
6266	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6267	* out-of-sync. Make sure to update the required fields
6268	* before using them.
6269	* @param uVector The software interrupt vector number.
6270	* @param cbInstr The value of RIP that is to be pushed on the guest
6271	* stack.
6272	*/
6273	DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
6274	{
6275	uint32_t u32IntrInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6276	if ( uVector == X86_XCPT_BP
6277	\|\| uVector == X86_XCPT_OF)
6278	{
6279	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6280	}
6281	else
6282	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6283	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
6284	}
6285
6286
6287	/**
6288	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
6289	* stack.
6290	*
6291	* @returns VBox status code (information status code included).
6292	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
6293	* @param pVM Pointer to the VM.
6294	* @param pMixedCtx Pointer to the guest-CPU context.
6295	* @param uValue The value to push to the guest stack.
6296	*/
6297	DECLINLINE(int) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
6298	{
6299	/*
6300	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
6301	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
6302	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
6303	*/
6304	if (pMixedCtx->sp == 1)
6305	return VINF_EM_RESET;
6306	pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
6307	int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
6308	AssertRCReturn(rc, rc);
6309	return rc;
6310	}
6311
6312
6313	/**
6314	* Injects an event into the guest upon VM-entry by updating the relevant fields
6315	* in the VM-entry area in the VMCS.
6316	*
6317	* @returns VBox status code (informational error codes included).
6318	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
6319	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
6320	*
6321	* @param pVCpu Pointer to the VMCPU.
6322	* @param pMixedCtx Pointer to the guest-CPU context. The data may
6323	* be out-of-sync. Make sure to update the required
6324	* fields before using them.
6325	* @param u64IntrInfo The VM-entry interruption-information field.
6326	* @param cbInstr The VM-entry instruction length in bytes (for
6327	* software interrupts, exceptions and privileged
6328	* software exceptions).
6329	* @param u32ErrCode The VM-entry exception error code.
6330	* @param GCPtrFaultAddress The page-fault address for #PF exceptions.
6331	* @param puIntrState Pointer to the current guest interruptibility-state.
6332	* This interruptibility-state will be updated if
6333	* necessary. This cannot not be NULL.
6334	*
6335	* @remarks No-long-jump zone!!!
6336	* @remarks Requires CR0!
6337	*/
6338	static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntrInfo, uint32_t cbInstr,
6339	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState)
6340	{
6341	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
6342	AssertMsg(u64IntrInfo >> 32 == 0, ("%#RX64\n", u64IntrInfo));
6343	Assert(puIntrState);
6344	uint32_t u32IntrInfo = (uint32_t)u64IntrInfo;
6345
6346	const uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntrInfo);
6347	const uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo);
6348
6349	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
6350	Assert( uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
6351	\|\| !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
6352
6353	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
6354
6355	/* We require CR0 to check if the guest is in real-mode. */
6356	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6357	AssertRCReturn(rc, rc);
6358
6359	/*
6360	* Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
6361	* mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
6362	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
6363	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
6364	*/
6365	if (CPUMIsGuestInRealModeEx(pMixedCtx))
6366	{
6367	PVM pVM = pVCpu->CTX_SUFF(pVM);
6368	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
6369	{
6370	Assert(PDMVmmDevHeapIsEnabled(pVM));
6371	Assert(pVM->hm.s.vmx.pRealModeTSS);
6372
6373	/* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
6374	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6375	rc \|= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
6376	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6377	AssertRCReturn(rc, rc);
6378	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RIP);
6379
6380	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
6381	const size_t cbIdtEntry = 4;
6382	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
6383	{
6384	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
6385	if (uVector == X86_XCPT_DF)
6386	return VINF_EM_RESET;
6387	else if (uVector == X86_XCPT_GP)
6388	{
6389	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
6390	return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, puIntrState);
6391	}
6392
6393	/* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
6394	/* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
6395	return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid /, 0 / u32ErrCode */, puIntrState);
6396	}
6397
6398	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
6399	uint16_t uGuestIp = pMixedCtx->ip;
6400	if (VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
6401	{
6402	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
6403	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
6404	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6405	}
6406	else if (VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
6407	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6408
6409	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
6410	uint16_t offIdtEntry = 0;
6411	RTSEL selIdtEntry = 0;
6412	RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
6413	rc = PGMPhysSimpleReadGCPhys(pVM, &offIdtEntry, GCPhysIdtEntry, sizeof(offIdtEntry));
6414	rc \|= PGMPhysSimpleReadGCPhys(pVM, &selIdtEntry, GCPhysIdtEntry + 2, sizeof(selIdtEntry));
6415	AssertRCReturn(rc, rc);
6416
6417	/* Construct the stack frame for the interrupt/exception handler. */
6418	rc = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
6419	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
6420	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
6421	AssertRCReturn(rc, rc);
6422
6423	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
6424	if (rc == VINF_SUCCESS)
6425	{
6426	pMixedCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
6427	pMixedCtx->rip = offIdtEntry;
6428	pMixedCtx->cs.Sel = selIdtEntry;
6429	pMixedCtx->cs.u64Base = selIdtEntry << cbIdtEntry;
6430	if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
6431	&& uVector == X86_XCPT_PF)
6432	{
6433	pMixedCtx->cr2 = GCPtrFaultAddress;
6434	}
6435	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SEGMENT_REGS
6436	\| HM_CHANGED_GUEST_RIP
6437	\| HM_CHANGED_GUEST_RFLAGS
6438	\| HM_CHANGED_GUEST_RSP;
6439
6440	/* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
6441	if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
6442	{
6443	Assert( uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
6444	&& uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
6445	Log4(("Clearing inhibition due to STI.\n"));
6446	*puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
6447	}
6448	Log4(("Injecting real-mode: u32IntrInfo=%#x u32ErrCode=%#x instrlen=%#x\n", u32IntrInfo, u32ErrCode, cbInstr));
6449	}
6450	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
6451	return rc;
6452	}
6453	else
6454	{
6455	/*
6456	* For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
6457	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
6458	*/
6459	u32IntrInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6460	}
6461	}
6462
6463	/* Validate. */
6464	Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntrInfo)); /* Bit 31 (Valid bit) must be set by caller. */
6465	Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK(u32IntrInfo)); /* Bit 12 MBZ. */
6466	Assert(!(u32IntrInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
6467
6468	/* Inject. */
6469	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntrInfo);
6470	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntrInfo))
6471	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
6472	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
6473
6474	if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
6475	&& uVector == X86_XCPT_PF)
6476	{
6477	pMixedCtx->cr2 = GCPtrFaultAddress;
6478	}
6479
6480	Log4(("Injecting u32IntrInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", u32IntrInfo, u32ErrCode, cbInstr,
6481	pMixedCtx->cr2));
6482
6483	AssertRCReturn(rc, rc);
6484	return rc;
6485	}
6486
6487
6488	/**
6489	* Enters the VT-x session.
6490	*
6491	* @returns VBox status code.
6492	* @param pVM Pointer to the VM.
6493	* @param pVCpu Pointer to the VMCPU.
6494	* @param pCpu Pointer to the CPU info struct.
6495	*/
6496	VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBLCPUINFO pCpu)
6497	{
6498	AssertPtr(pVM);
6499	AssertPtr(pVCpu);
6500	Assert(pVM->hm.s.vmx.fSupported);
6501	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6502	NOREF(pCpu);
6503
6504	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6505
6506	/* Make sure we're in VMX root mode. */
6507	RTCCUINTREG u32HostCR4 = ASMGetCR4();
6508	if (!(u32HostCR4 & X86_CR4_VMXE))
6509	{
6510	LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
6511	return VERR_VMX_X86_CR4_VMXE_CLEARED;
6512	}
6513
6514	/* Load the active VMCS as the current one. */
6515	int rc = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
6516	if (RT_FAILURE(rc))
6517	return rc;
6518
6519	/** @todo this will change with preemption hooks where can can VMRESUME as long
6520	* as we're no preempted. */
6521	pVCpu->hm.s.fResumeVM = false;
6522	return VINF_SUCCESS;
6523	}
6524
6525
6526	/**
6527	* Leaves the VT-x session.
6528	*
6529	* @returns VBox status code.
6530	* @param pVM Pointer to the VM.
6531	* @param pVCpu Pointer to the VMCPU.
6532	* @param pCtx Pointer to the guest-CPU context.
6533	*/
6534	VMMR0DECL(int) VMXR0Leave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
6535	{
6536	AssertPtr(pVCpu);
6537	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6538	NOREF(pVM);
6539	NOREF(pCtx);
6540
6541	/** @todo this will change with preemption hooks where we only VMCLEAR when
6542	* we are actually going to be preempted, not all the time like we
6543	* currently do. */
6544
6545	/* Restore host-state bits that VT-x only restores partially. */
6546	if (pVCpu->hm.s.vmx.fRestoreHostFlags)
6547	{
6548	#ifndef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
6549	/** @todo r=ramshankar: This is broken when
6550	* VBOX_WITH_VMMR0_DISABLE_PREEMPTION is not defined. As
6551	* VMXRestoreHostState() may unconditionally enables interrupts. */
6552	#error "VMM: Fix Me! Make VMXRestoreHostState() function to skip cli/sti."
6553	#else
6554	Assert(ASMIntAreEnabled());
6555	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6556	#endif
6557	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6558	}
6559
6560	/*
6561	* Sync the current VMCS (writes back internal data back into the VMCS region in memory)
6562	* and mark the VMCS launch-state as "clear".
6563	*/
6564	int rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
6565	return rc;
6566	}
6567
6568
6569	/**
6570	* Saves the host state in the VMCS host-state.
6571	* Sets up the VM-exit MSR-load area.
6572	*
6573	* The CPU state will be loaded from these fields on every successful VM-exit.
6574	*
6575	* @returns VBox status code.
6576	* @param pVM Pointer to the VM.
6577	* @param pVCpu Pointer to the VMCPU.
6578	*
6579	* @remarks No-long-jump zone!!!
6580	*/
6581	VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
6582	{
6583	AssertPtr(pVM);
6584	AssertPtr(pVCpu);
6585	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6586
6587	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6588
6589	/* Nothing to do if the host-state-changed flag isn't set. This will later be optimized when preemption hooks are in place. */
6590	if (!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT))
6591	return VINF_SUCCESS;
6592
6593	int rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
6594	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6595
6596	rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
6597	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6598
6599	rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
6600	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6601
6602	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_HOST_CONTEXT;
6603	return rc;
6604	}
6605
6606
6607	/**
6608	* Loads the guest state into the VMCS guest-state area. The CPU state will be
6609	* loaded from these fields on every successful VM-entry.
6610	*
6611	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas.
6612	* Sets up the VM-entry controls.
6613	* Sets up the appropriate VMX non-root function to execute guest code based on
6614	* the guest CPU mode.
6615	*
6616	* @returns VBox status code.
6617	* @param pVM Pointer to the VM.
6618	* @param pVCpu Pointer to the VMCPU.
6619	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6620	* out-of-sync. Make sure to update the required fields
6621	* before using them.
6622	*
6623	* @remarks No-long-jump zone!!!
6624	*/
6625	VMMR0DECL(int) VMXR0LoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6626	{
6627	AssertPtr(pVM);
6628	AssertPtr(pVCpu);
6629	AssertPtr(pMixedCtx);
6630	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6631
6632	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6633
6634	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
6635
6636	/* Determine real-on-v86 mode. */
6637	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
6638	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
6639	&& CPUMIsGuestInRealModeEx(pMixedCtx))
6640	{
6641	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
6642	}
6643
6644	/*
6645	* Load the guest-state into the VMCS.
6646	* Any ordering dependency among the sub-functions below must be explicitly stated using comments.
6647	* Ideally, assert that the cross-dependent bits are up to date at the point of using it.
6648	*/
6649	int rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
6650	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6651
6652	rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
6653	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6654
6655	rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
6656	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6657
6658	rc = hmR0VmxLoadGuestControlRegs(pVCpu, pMixedCtx);
6659	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestControlRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6660
6661	/* Must be done after CR0 is loaded (strict builds require CR0 for segment register validation checks). */
6662	rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
6663	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6664
6665	rc = hmR0VmxLoadGuestDebugRegs(pVCpu, pMixedCtx);
6666	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestDebugRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6667
6668	rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
6669	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6670
6671	rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
6672	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6673
6674	/* Must be done after hmR0VmxLoadGuestDebugRegs() as it may update eflags.TF for debugging purposes. */
6675	rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
6676	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestGprs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6677
6678	rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
6679	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6680
6681	/* Clear any unused and reserved bits. */
6682	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR2;
6683
6684	AssertMsg(!pVCpu->hm.s.fContextUseFlags,
6685	("Missed updating flags while loading guest state. pVM=%p pVCpu=%p fContextUseFlags=%#RX32\n",
6686	pVM, pVCpu, pVCpu->hm.s.fContextUseFlags));
6687
6688	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
6689	return rc;
6690	}
6691
6692
6693	/**
6694	* Does the preparations before executing guest code in VT-x.
6695	*
6696	* This may cause longjmps to ring-3 and may even result in rescheduling to the
6697	* recompiler. We must be cautious what we do here regarding committing
6698	* guest-state information into the the VMCS assuming we assuredly execute the
6699	* guest in VT-x. If we fall back to the recompiler after updating the VMCS and
6700	* clearing the common-state (TRPM/forceflags), we must undo those changes so
6701	* that the recompiler can (and should) use them when it resumes guest
6702	* execution. Otherwise such operations must be done when we can no longer
6703	* exit to ring-3.
6704	*
6705	* @returns VBox status code (informational status codes included).
6706	* @retval VINF_SUCCESS if we can proceed with running the guest.
6707	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a double-fault
6708	* into the guest.
6709	* @retval VINF_* scheduling changes, we have to go back to ring-3.
6710	*
6711	* @param pVM Pointer to the VM.
6712	* @param pVCpu Pointer to the VMCPU.
6713	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6714	* out-of-sync. Make sure to update the required fields
6715	* before using them.
6716	* @param pVmxTransient Pointer to the VMX transient structure.
6717	*
6718	* @remarks Called with preemption disabled.
6719	*/
6720	DECLINLINE(int) hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6721	{
6722	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6723
6724	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
6725	PGMRZDynMapFlushAutoSet(pVCpu);
6726	#endif
6727
6728	/* Check force flag actions that might require us to go back to ring-3. */
6729	int rc = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx);
6730	if (rc != VINF_SUCCESS)
6731	return rc;
6732
6733	/* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
6734	if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
6735	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
6736	{
6737	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
6738	RTGCPHYS GCPhysApicBase;
6739	GCPhysApicBase = pMixedCtx->msrApicBase;
6740	GCPhysApicBase &= PAGE_BASE_GC_MASK;
6741
6742	/* Unalias any existing mapping. */
6743	rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
6744	AssertRCReturn(rc, rc);
6745
6746	/* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
6747	Log4(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGv\n", GCPhysApicBase));
6748	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
6749	AssertRCReturn(rc, rc);
6750
6751	pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
6752	}
6753
6754	#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
6755	/* We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.) */
6756	pVmxTransient->uEFlags = ASMIntDisableFlags();
6757	if (RTThreadPreemptIsPending(NIL_RTTHREAD))
6758	{
6759	ASMSetFlags(pVmxTransient->uEFlags);
6760	STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
6761	/* Don't use VINF_EM_RAW_INTERRUPT_HYPER as we can't assume the host does kernel preemption. Maybe some day? */
6762	return VINF_EM_RAW_INTERRUPT;
6763	}
6764	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
6765	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
6766	#endif
6767
6768	/*
6769	* Evaluates and injects any pending events, toggling force-flags and updating the guest-interruptibility
6770	* state (interrupt shadow) in the VMCS. This -can- potentially be reworked to be done before disabling
6771	* interrupts and handle returning to ring-3 afterwards, but requires very careful state restoration.
6772	*/
6773	/** @todo Rework event evaluation and injection to be completely separate. */
6774	if (TRPMHasTrap(pVCpu))
6775	hmR0VmxTrpmTrapToPendingEvent(pVCpu);
6776
6777	rc = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx);
6778	AssertRCReturn(rc, rc);
6779	return rc;
6780	}
6781
6782
6783	/**
6784	* Prepares to run guest code in VT-x and we've committed to doing so. This
6785	* means there is no backing out to ring-3 or anywhere else at this
6786	* point.
6787	*
6788	* @param pVM Pointer to the VM.
6789	* @param pVCpu Pointer to the VMCPU.
6790	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6791	* out-of-sync. Make sure to update the required fields
6792	* before using them.
6793	* @param pVmxTransient Pointer to the VMX transient structure.
6794	*
6795	* @remarks Called with preemption disabled.
6796	* @remarks No-long-jump zone!!!
6797	*/
6798	DECLINLINE(void) hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6799	{
6800	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6801	Assert(VMMR0IsLogFlushDisabled(pVCpu));
6802
6803	#ifndef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
6804	/** @todo I don't see the point of this, VMMR0EntryFast() already disables interrupts for the entire period. */
6805	pVmxTransient->uEFlags = ASMIntDisableFlags();
6806	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
6807	#endif
6808
6809	/* Load the required guest state bits (for guest-state changes in the inner execution loop). */
6810	Assert(!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT));
6811	Log5(("LoadFlags=%#RX32\n", pVCpu->hm.s.fContextUseFlags));
6812	#ifdef HMVMX_SYNC_FULL_GUEST_STATE
6813	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_ALL_GUEST;
6814	#endif
6815	int rc = VINF_SUCCESS;
6816	if (pVCpu->hm.s.fContextUseFlags == HM_CHANGED_GUEST_RIP)
6817	{
6818	rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
6819	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
6820	}
6821	else if (pVCpu->hm.s.fContextUseFlags)
6822	{
6823	rc = VMXR0LoadGuestState(pVM, pVCpu, pMixedCtx);
6824	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
6825	}
6826	AssertRC(rc);
6827	AssertMsg(!pVCpu->hm.s.fContextUseFlags, ("fContextUseFlags =%#x\n", pVCpu->hm.s.fContextUseFlags));
6828
6829	/* Cache the TPR-shadow for checking on every VM-exit if it might have changed. */
6830	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
6831	pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
6832
6833	if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
6834	\|\| HMR0GetCurrentCpu()->idCpu != pVCpu->hm.s.idLastCpu)
6835	{
6836	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu, pMixedCtx);
6837	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
6838	}
6839
6840	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB-shootdowns, set this across the world switch. */
6841	hmR0VmxFlushTaggedTlb(pVCpu); /* Invalidate the appropriate guest entries from the TLB. */
6842	Assert(HMR0GetCurrentCpu()->idCpu == pVCpu->hm.s.idLastCpu);
6843
6844	#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
6845	/*
6846	* Save the current Host TSC_AUX and write the guest TSC_AUX to the host, so that
6847	* RDTSCPs (that don't cause exits) reads the guest MSR. See @bugref{3324}.
6848	*/
6849	if ( (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
6850	&& !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
6851	{
6852	pVCpu->hm.s.u64HostTscAux = ASMRdMsr(MSR_K8_TSC_AUX);
6853	uint64_t u64HostTscAux = 0;
6854	int rc2 = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &u64HostTscAux);
6855	AssertRC(rc2);
6856	ASMWrMsr(MSR_K8_TSC_AUX, u64HostTscAux);
6857	}
6858	#endif
6859
6860	TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
6861	to start executing. */
6862	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
6863	}
6864
6865
6866	/**
6867	* Performs some essential restoration of state after running guest code in
6868	* VT-x.
6869	*
6870	* @param pVM Pointer to the VM.
6871	* @param pVCpu Pointer to the VMCPU.
6872	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6873	* out-of-sync. Make sure to update the required fields
6874	* before using them.
6875	* @param pVmxTransient Pointer to the VMX transient structure.
6876	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
6877	*
6878	* @remarks Called with interrupts disabled.
6879	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
6880	* unconditionally when it is safe to do so.
6881	*/
6882	DECLINLINE(void) hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
6883	{
6884	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6885	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
6886
6887	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB-shootdowns. */
6888	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for TLB-shootdowns. */
6889	pVCpu->hm.s.vmx.fUpdatedGuestState = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
6890	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
6891	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
6892
6893	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
6894	{
6895	#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
6896	/* Restore host's TSC_AUX. */
6897	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
6898	ASMWrMsr(MSR_K8_TSC_AUX, pVCpu->hm.s.u64HostTscAux);
6899	#endif
6900	/** @todo Find a way to fix hardcoding a guestimate. */
6901	TMCpuTickSetLastSeen(pVCpu, ASMReadTSC()
6902	+ pVCpu->hm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
6903	}
6904
6905	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
6906	Assert(!(ASMGetFlags() & X86_EFL_IF));
6907	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
6908
6909	ASMSetFlags(pVmxTransient->uEFlags); /* Enable interrupts. */
6910	pVCpu->hm.s.fResumeVM = true; /* Use VMRESUME instead of VMLAUNCH in the next run. */
6911
6912	/* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
6913	uint32_t uExitReason;
6914	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
6915	rc \|= hmR0VmxReadEntryIntrInfoVmcs(pVmxTransient);
6916	AssertRC(rc);
6917	pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
6918	pVmxTransient->fVMEntryFailed = !!VMX_ENTRY_INTERRUPTION_INFO_VALID(pVmxTransient->uEntryIntrInfo);
6919
6920	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pMixedCtx);
6921	VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
6922
6923	/* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_. /
6924	if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
6925	{
6926	Log4(("VM-entry failure: rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", rcVMRun, pVmxTransient->fVMEntryFailed));
6927	return;
6928	}
6929
6930	if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
6931	{
6932	/* Update the guest interruptibility-state from the VMCS. */
6933	hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
6934	#if defined(HMVMX_SYNC_FULL_GUEST_STATE) \|\| defined(HMVMX_SAVE_FULL_GUEST_STATE)
6935	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6936	AssertRC(rc);
6937	#endif
6938	/*
6939	* If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
6940	* we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3 and which is
6941	* why it's done here as it's easier and no less efficient to deal with it here than making hmR0VmxSaveGuestState()
6942	* cope with longjmps safely (see VMCPU_FF_HM_UPDATE_CR3 handling).
6943	*/
6944	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
6945	&& pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
6946	{
6947	rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
6948	AssertRC(rc);
6949	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
6950	}
6951	}
6952	}
6953
6954
6955	/**
6956	* Runs the guest code using VT-x.
6957	*
6958	* @returns VBox status code.
6959	* @param pVM Pointer to the VM.
6960	* @param pVCpu Pointer to the VMCPU.
6961	* @param pCtx Pointer to the guest-CPU context.
6962	*
6963	* @remarks Called with preemption disabled.
6964	*/
6965	VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
6966	{
6967	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6968	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6969
6970	VMXTRANSIENT VmxTransient;
6971	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
6972	int rc = VERR_INTERNAL_ERROR_5;
6973	uint32_t cLoops = 0;
6974
6975	for (;; cLoops++)
6976	{
6977	Assert(!HMR0SuspendPending());
6978	AssertMsg(pVCpu->hm.s.idEnteredCpu == RTMpCpuId(),
6979	("Illegal migration! Entered on CPU %u Current %u cLoops=%u\n", (unsigned)pVCpu->hm.s.idEnteredCpu,
6980	(unsigned)RTMpCpuId(), cLoops));
6981
6982	/* Preparatory work for running guest code, this may return to ring-3 for some last minute updates. */
6983	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
6984	rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
6985	if (rc != VINF_SUCCESS)
6986	break;
6987
6988	/*
6989	* No longjmps to ring-3 from this point on!!!
6990	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
6991	* This also disables flushing of the R0-logger instance (if any).
6992	*/
6993	VMMRZCallRing3Disable(pVCpu);
6994	VMMRZCallRing3RemoveNotification(pVCpu);
6995	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
6996
6997	rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
6998	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
6999
7000	/*
7001	* Restore any residual host-state and save any bits shared between host and guest into the guest-CPU state.
7002	* This will also re-enable longjmps to ring-3 when it has reached a safe point!!!
7003	*/
7004	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
7005	if (RT_UNLIKELY(rc != VINF_SUCCESS)) /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
7006	{
7007	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
7008	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
7009	return rc;
7010	}
7011
7012	/* Handle the VM-exit. */
7013	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
7014	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
7015	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
7016	HMVMX_START_EXIT_DISPATCH_PROF();
7017	#ifdef HMVMX_USE_FUNCTION_TABLE
7018	rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
7019	#else
7020	rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
7021	#endif
7022	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
7023	if (rc != VINF_SUCCESS)
7024	break;
7025	else if (cLoops > pVM->hm.s.cMaxResumeLoops)
7026	{
7027	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
7028	rc = VINF_EM_RAW_INTERRUPT;
7029	break;
7030	}
7031	}
7032
7033	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
7034	if (rc == VERR_EM_INTERPRETER)
7035	rc = VINF_EM_RAW_EMULATE_INSTR;
7036	else if (rc == VINF_EM_RESET)
7037	rc = VINF_EM_TRIPLE_FAULT;
7038	hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rc);
7039	return rc;
7040	}
7041
7042
7043	#ifndef HMVMX_USE_FUNCTION_TABLE
7044	DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
7045	{
7046	int rc;
7047	switch (rcReason)
7048	{
7049	case VMX_EXIT_EPT_MISCONFIG: rc = hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient); break;
7050	case VMX_EXIT_EPT_VIOLATION: rc = hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient); break;
7051	case VMX_EXIT_IO_INSTR: rc = hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient); break;
7052	case VMX_EXIT_CPUID: rc = hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient); break;
7053	case VMX_EXIT_RDTSC: rc = hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient); break;
7054	case VMX_EXIT_RDTSCP: rc = hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient); break;
7055	case VMX_EXIT_APIC_ACCESS: rc = hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient); break;
7056	case VMX_EXIT_XCPT_NMI: rc = hmR0VmxExitXcptNmi(pVCpu, pMixedCtx, pVmxTransient); break;
7057	case VMX_EXIT_MOV_CRX: rc = hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient); break;
7058	case VMX_EXIT_EXT_INT: rc = hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient); break;
7059	case VMX_EXIT_INT_WINDOW: rc = hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient); break;
7060	case VMX_EXIT_MWAIT: rc = hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient); break;
7061	case VMX_EXIT_MONITOR: rc = hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient); break;
7062	case VMX_EXIT_TASK_SWITCH: rc = hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient); break;
7063	case VMX_EXIT_PREEMPT_TIMER: rc = hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient); break;
7064	case VMX_EXIT_RDMSR: rc = hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient); break;
7065	case VMX_EXIT_WRMSR: rc = hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient); break;
7066	case VMX_EXIT_MOV_DRX: rc = hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient); break;
7067	case VMX_EXIT_TPR_BELOW_THRESHOLD: rc = hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient); break;
7068	case VMX_EXIT_HLT: rc = hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient); break;
7069	case VMX_EXIT_INVD: rc = hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient); break;
7070	case VMX_EXIT_INVLPG: rc = hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient); break;
7071	case VMX_EXIT_RSM: rc = hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient); break;
7072	case VMX_EXIT_MTF: rc = hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient); break;
7073	case VMX_EXIT_PAUSE: rc = hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient); break;
7074	case VMX_EXIT_XDTR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); break;
7075	case VMX_EXIT_TR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); break;
7076	case VMX_EXIT_WBINVD: rc = hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient); break;
7077	case VMX_EXIT_XSETBV: rc = hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient); break;
7078	case VMX_EXIT_RDRAND: rc = hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient); break;
7079	case VMX_EXIT_INVPCID: rc = hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient); break;
7080	case VMX_EXIT_GETSEC: rc = hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient); break;
7081	case VMX_EXIT_RDPMC: rc = hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient); break;
7082
7083	case VMX_EXIT_TRIPLE_FAULT: rc = hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient); break;
7084	case VMX_EXIT_NMI_WINDOW: rc = hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient); break;
7085	case VMX_EXIT_INIT_SIGNAL: rc = hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient); break;
7086	case VMX_EXIT_SIPI: rc = hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient); break;
7087	case VMX_EXIT_IO_SMI: rc = hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient); break;
7088	case VMX_EXIT_SMI: rc = hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient); break;
7089	case VMX_EXIT_ERR_MSR_LOAD: rc = hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient); break;
7090	case VMX_EXIT_ERR_INVALID_GUEST_STATE: rc = hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient); break;
7091	case VMX_EXIT_ERR_MACHINE_CHECK: rc = hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient); break;
7092
7093	case VMX_EXIT_VMCALL:
7094	case VMX_EXIT_VMCLEAR:
7095	case VMX_EXIT_VMLAUNCH:
7096	case VMX_EXIT_VMPTRLD:
7097	case VMX_EXIT_VMPTRST:
7098	case VMX_EXIT_VMREAD:
7099	case VMX_EXIT_VMRESUME:
7100	case VMX_EXIT_VMWRITE:
7101	case VMX_EXIT_VMXOFF:
7102	case VMX_EXIT_VMXON:
7103	case VMX_EXIT_INVEPT:
7104	case VMX_EXIT_INVVPID:
7105	case VMX_EXIT_VMFUNC:
7106	rc = hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
7107	break;
7108	default:
7109	rc = hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
7110	break;
7111	}
7112	return rc;
7113	}
7114	#endif
7115
7116	#ifdef DEBUG
7117	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
7118	# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
7119	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
7120
7121	# define HMVMX_ASSERT_PREEMPT_CPUID() \
7122	do \
7123	{ \
7124	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
7125	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
7126	} while (0)
7127
7128	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
7129	do { \
7130	AssertPtr(pVCpu); \
7131	AssertPtr(pMixedCtx); \
7132	AssertPtr(pVmxTransient); \
7133	Assert(pVmxTransient->fVMEntryFailed == false); \
7134	Assert(ASMIntAreEnabled()); \
7135	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD)); \
7136	HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
7137	Log4Func(("vcpu[%u] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-\n", (uint32_t)pVCpu->idCpu)); \
7138	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD)); \
7139	if (VMMR0IsLogFlushDisabled(pVCpu)) \
7140	HMVMX_ASSERT_PREEMPT_CPUID(); \
7141	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
7142	} while (0)
7143
7144	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
7145	do { \
7146	Log4Func(("\n")); \
7147	} while(0)
7148	#else /* Release builds */
7149	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() do { HMVMX_STOP_EXIT_DISPATCH_PROF(); } while(0)
7150	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while(0)
7151	#endif
7152
7153
7154	/**
7155	* Advances the guest RIP after reading it from the VMCS.
7156	*
7157	* @returns VBox status code.
7158	* @param pVCpu Pointer to the VMCPU.
7159	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
7160	* out-of-sync. Make sure to update the required fields
7161	* before using them.
7162	* @param pVmxTransient Pointer to the VMX transient structure.
7163	*
7164	* @remarks No-long-jump zone!!!
7165	*/
7166	DECLINLINE(int) hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7167	{
7168	int rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7169	rc \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
7170	AssertRCReturn(rc, rc);
7171
7172	pMixedCtx->rip += pVmxTransient->cbInstr;
7173	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7174	return rc;
7175	}
7176
7177
7178	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
7179	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
7180	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
7181	/**
7182	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
7183	*/
7184	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7185	{
7186	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7187	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
7188	/* 32-bit Windows hosts (4 cores) has trouble with this; causes higher interrupt latency. */
7189	#if HC_ARCH_BITS == 64 && defined(VBOX_WITH_VMMR0_DISABLE_PREEMPTION)
7190	Assert(ASMIntAreEnabled());
7191	return VINF_SUCCESS;
7192	#else
7193	return VINF_EM_RAW_INTERRUPT;
7194	#endif
7195	}
7196
7197
7198	/**
7199	* VM-exit handler for exceptions and NMIs (VMX_EXIT_XCPT_NMI).
7200	*/
7201	HMVMX_EXIT_DECL hmR0VmxExitXcptNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7202	{
7203	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7204	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
7205
7206	int rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
7207	AssertRCReturn(rc, rc);
7208
7209	uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntrInfo);
7210	Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
7211	&& uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
7212
7213	if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7214	{
7215	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
7216	return VINF_EM_RAW_INTERRUPT;
7217	}
7218
7219	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
7220	rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
7221	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
7222	{
7223	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
7224	return VINF_SUCCESS;
7225	}
7226	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
7227	{
7228	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
7229	return rc;
7230	}
7231
7232	uint32_t uExitIntrInfo = pVmxTransient->uExitIntrInfo;
7233	uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntrInfo);
7234	switch (uIntrType)
7235	{
7236	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
7237	Assert(uVector == X86_XCPT_DB \|\| uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
7238	/* no break */
7239	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
7240	{
7241	switch (uVector)
7242	{
7243	case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
7244	case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
7245	case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
7246	case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
7247	case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
7248	case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
7249	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
7250	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
7251	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7252	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
7253	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7254	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
7255	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7256	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
7257	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7258	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
7259	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7260	#endif
7261	default:
7262	{
7263	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7264	AssertRCReturn(rc, rc);
7265
7266	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
7267	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
7268	{
7269	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
7270	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
7271	rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7272	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
7273	AssertRCReturn(rc, rc);
7274	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntrInfo),
7275	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode,
7276	0 /* GCPtrFaultAddress */);
7277	AssertRCReturn(rc, rc);
7278	}
7279	else
7280	{
7281	AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
7282	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
7283	}
7284	break;
7285	}
7286	}
7287	break;
7288	}
7289
7290	case VMX_EXIT_INTERRUPTION_INFO_TYPE_DB_XCPT:
7291	default:
7292	{
7293	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_CODE;
7294	AssertMsgFailed(("Unexpected interruption code %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntrInfo)));
7295	break;
7296	}
7297	}
7298	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
7299	return rc;
7300	}
7301
7302
7303	/**
7304	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
7305	*/
7306	HMVMX_EXIT_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7307	{
7308	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7309
7310	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
7311	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7312	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7313	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7314	AssertRCReturn(rc, rc);
7315
7316	/* Deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and resume guest execution. */
7317	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
7318	return VINF_SUCCESS;
7319	}
7320
7321
7322	/**
7323	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
7324	*/
7325	HMVMX_EXIT_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7326	{
7327	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7328	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
7329	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7330	}
7331
7332
7333	/**
7334	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
7335	*/
7336	HMVMX_EXIT_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7337	{
7338	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7339	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
7340	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7341	}
7342
7343
7344	/**
7345	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
7346	*/
7347	HMVMX_EXIT_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7348	{
7349	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7350	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
7351	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7352	}
7353
7354
7355	/**
7356	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
7357	*/
7358	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7359	{
7360	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7361	PVM pVM = pVCpu->CTX_SUFF(pVM);
7362	int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7363	if (RT_LIKELY(rc == VINF_SUCCESS))
7364	{
7365	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7366	Assert(pVmxTransient->cbInstr == 2);
7367	}
7368	else
7369	{
7370	AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
7371	rc = VERR_EM_INTERPRETER;
7372	}
7373	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
7374	return rc;
7375	}
7376
7377
7378	/**
7379	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
7380	*/
7381	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7382	{
7383	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7384	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
7385	AssertRCReturn(rc, rc);
7386
7387	if (pMixedCtx->cr4 & X86_CR4_SMXE)
7388	return VINF_EM_RAW_EMULATE_INSTR;
7389
7390	AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
7391	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7392	}
7393
7394
7395	/**
7396	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
7397	*/
7398	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7399	{
7400	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7401	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
7402	AssertRCReturn(rc, rc);
7403
7404	PVM pVM = pVCpu->CTX_SUFF(pVM);
7405	rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7406	if (RT_LIKELY(rc == VINF_SUCCESS))
7407	{
7408	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7409	Assert(pVmxTransient->cbInstr == 2);
7410	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
7411	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
7412	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7413	}
7414	else
7415	{
7416	AssertMsgFailed(("hmR0VmxExitRdtsc: EMInterpretRdtsc failed with %Rrc\n", rc));
7417	rc = VERR_EM_INTERPRETER;
7418	}
7419	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
7420	return rc;
7421	}
7422
7423
7424	/**
7425	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
7426	*/
7427	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7428	{
7429	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7430	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
7431	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
7432	AssertRCReturn(rc, rc);
7433
7434	PVM pVM = pVCpu->CTX_SUFF(pVM);
7435	rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
7436	if (RT_LIKELY(rc == VINF_SUCCESS))
7437	{
7438	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7439	Assert(pVmxTransient->cbInstr == 3);
7440	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
7441	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
7442	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7443	}
7444	else
7445	{
7446	AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
7447	rc = VERR_EM_INTERPRETER;
7448	}
7449	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
7450	return rc;
7451	}
7452
7453
7454	/**
7455	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
7456	*/
7457	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7458	{
7459	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7460	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
7461	rc \|= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
7462	AssertRCReturn(rc, rc);
7463
7464	PVM pVM = pVCpu->CTX_SUFF(pVM);
7465	rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7466	if (RT_LIKELY(rc == VINF_SUCCESS))
7467	{
7468	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7469	Assert(pVmxTransient->cbInstr == 2);
7470	}
7471	else
7472	{
7473	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
7474	rc = VERR_EM_INTERPRETER;
7475	}
7476	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
7477	return rc;
7478	}
7479
7480
7481	/**
7482	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
7483	*/
7484	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7485	{
7486	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7487	PVM pVM = pVCpu->CTX_SUFF(pVM);
7488	Assert(!pVM->hm.s.fNestedPaging);
7489
7490	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
7491	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
7492	AssertRCReturn(rc, rc);
7493
7494	VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
7495	rc = VBOXSTRICTRC_VAL(rc2);
7496	if (RT_LIKELY(rc == VINF_SUCCESS))
7497	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7498	else
7499	{
7500	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
7501	pVmxTransient->uExitQualification, rc));
7502	}
7503	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
7504	return rc;
7505	}
7506
7507
7508	/**
7509	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
7510	*/
7511	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7512	{
7513	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7514	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7515	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7516	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7517	AssertRCReturn(rc, rc);
7518
7519	PVM pVM = pVCpu->CTX_SUFF(pVM);
7520	rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7521	if (RT_LIKELY(rc == VINF_SUCCESS))
7522	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7523	else
7524	{
7525	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
7526	rc = VERR_EM_INTERPRETER;
7527	}
7528	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
7529	return rc;
7530	}
7531
7532
7533	/**
7534	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
7535	*/
7536	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7537	{
7538	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7539	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7540	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7541	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7542	AssertRCReturn(rc, rc);
7543
7544	PVM pVM = pVCpu->CTX_SUFF(pVM);
7545	VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7546	rc = VBOXSTRICTRC_VAL(rc2);
7547	if (RT_LIKELY( rc == VINF_SUCCESS
7548	\|\| rc == VINF_EM_HALT))
7549	{
7550	int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7551	AssertRCReturn(rc3, rc3);
7552
7553	if ( rc == VINF_EM_HALT
7554	&& EMShouldContinueAfterHalt(pVCpu, pMixedCtx))
7555	{
7556	rc = VINF_SUCCESS;
7557	}
7558	}
7559	else
7560	{
7561	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
7562	rc = VERR_EM_INTERPRETER;
7563	}
7564	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_EM_HALT \|\| rc == VERR_EM_INTERPRETER,
7565	("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
7566	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
7567	return rc;
7568	}
7569
7570
7571	/**
7572	* VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
7573	*/
7574	HMVMX_EXIT_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7575	{
7576	/*
7577	* Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
7578	* get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
7579	* executing VMCALL in VMX root operation. If we get here something funny is going on.
7580	* See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
7581	*/
7582	AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7583	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7584	}
7585
7586
7587	/**
7588	* VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
7589	*/
7590	HMVMX_EXIT_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7591	{
7592	/*
7593	* This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
7594	* root operation. If we get there there is something funny going on.
7595	* See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
7596	*/
7597	AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7598	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7599	}
7600
7601
7602	/**
7603	* VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
7604	*/
7605	HMVMX_EXIT_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7606	{
7607	/* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
7608	AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7609	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7610	}
7611
7612
7613	/**
7614	* VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
7615	*/
7616	HMVMX_EXIT_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7617	{
7618	/*
7619	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
7620	* don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
7621	* See Intel spec. 25.3 "Other Causes of VM-exits".
7622	*/
7623	AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7624	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7625	}
7626
7627
7628	/**
7629	* VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
7630	* VM-exit.
7631	*/
7632	HMVMX_EXIT_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7633	{
7634	/*
7635	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM. See Intel spec. "33.14.1 Default Treatment of
7636	* SMI Delivery" and "29.3 VMX Instructions" for "VMXON". It is -NOT- blocked in VMX non-root operation so we can potentially
7637	* still get these exits. See Intel spec. "23.8 Restrictions on VMX operation".
7638	*/
7639	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7640	return VINF_SUCCESS; /** @todo r=ramshankar: correct?. */
7641	}
7642
7643
7644	/**
7645	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
7646	* VM-exit.
7647	*/
7648	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7649	{
7650	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7651	return VINF_EM_RESET;
7652	}
7653
7654
7655	/**
7656	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
7657	*/
7658	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7659	{
7660	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7661	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
7662	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
7663	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7664	AssertRCReturn(rc, rc);
7665
7666	pMixedCtx->rip++;
7667	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7668	if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
7669	rc = VINF_SUCCESS;
7670	else
7671	rc = VINF_EM_HALT;
7672
7673	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
7674	return rc;
7675	}
7676
7677
7678	/**
7679	* VM-exit handler for instructions that result in a #UD exception delivered to
7680	* the guest.
7681	*/
7682	HMVMX_EXIT_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7683	{
7684	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7685	hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
7686	return VINF_SUCCESS;
7687	}
7688
7689
7690	/**
7691	* VM-exit handler for expiry of the VMX preemption timer.
7692	*/
7693	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7694	{
7695	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7696
7697	/* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
7698	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7699
7700	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
7701	PVM pVM = pVCpu->CTX_SUFF(pVM);
7702	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
7703	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
7704	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
7705	}
7706
7707
7708	/**
7709	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
7710	*/
7711	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7712	{
7713	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7714
7715	/* We expose XSETBV to the guest, fallback to the recompiler for emulation. */
7716	/** @todo check if XSETBV is supported by the recompiler. */
7717	return VERR_EM_INTERPRETER;
7718	}
7719
7720
7721	/**
7722	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
7723	*/
7724	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7725	{
7726	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7727
7728	/* The guest should not invalidate the host CPU's TLBs, fallback to recompiler. */
7729	/** @todo implement EMInterpretInvpcid() */
7730	return VERR_EM_INTERPRETER;
7731	}
7732
7733
7734	/**
7735	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
7736	* Error VM-exit.
7737	*/
7738	HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7739	{
7740	uint32_t uIntrState;
7741	HMVMXHCUINTREG uHCReg;
7742	uint64_t u64Val;
7743	uint32_t u32Val;
7744
7745	int rc = hmR0VmxReadEntryIntrInfoVmcs(pVmxTransient);
7746	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
7747	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVCpu, pVmxTransient);
7748	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
7749	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7750	AssertRCReturn(rc, rc);
7751
7752	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntrInfo));
7753	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
7754	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
7755	Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
7756
7757	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
7758	Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
7759	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
7760	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
7761	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
7762	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
7763	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
7764	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
7765	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
7766	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
7767	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
7768	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
7769
7770	PVM pVM = pVCpu->CTX_SUFF(pVM);
7771	HMDumpRegs(pVM, pVCpu, pMixedCtx);
7772
7773	return VERR_VMX_INVALID_GUEST_STATE;
7774	}
7775
7776
7777	/**
7778	* VM-exit handler for VM-entry failure due to an MSR-load
7779	* (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
7780	*/
7781	HMVMX_EXIT_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7782	{
7783	AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7784	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7785	}
7786
7787
7788	/**
7789	* VM-exit handler for VM-entry failure due to a machine-check event
7790	* (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
7791	*/
7792	HMVMX_EXIT_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7793	{
7794	AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7795	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7796	}
7797
7798
7799	/**
7800	* VM-exit handler for all undefined reasons. Should never ever happen.. in
7801	* theory.
7802	*/
7803	HMVMX_EXIT_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7804	{
7805	AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
7806	return VERR_VMX_UNDEFINED_EXIT_CODE;
7807	}
7808
7809
7810	/**
7811	* VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
7812	* (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
7813	* Conditional VM-exit.
7814	*/
7815	HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7816	{
7817	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7818
7819	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
7820	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
7821	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
7822	return VERR_EM_INTERPRETER;
7823	AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7824	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7825	}
7826
7827
7828	/**
7829	* VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
7830	*/
7831	HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7832	{
7833	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7834
7835	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
7836	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
7837	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
7838	return VERR_EM_INTERPRETER;
7839	AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7840	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7841	}
7842
7843
7844	/**
7845	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
7846	*/
7847	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7848	{
7849	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7850
7851	/* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
7852	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7853	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7854	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7855	AssertRCReturn(rc, rc);
7856
7857	PVM pVM = pVCpu->CTX_SUFF(pVM);
7858	rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7859	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER,
7860	("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
7861	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
7862
7863	if (RT_LIKELY(rc == VINF_SUCCESS))
7864	{
7865	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7866	Assert(pVmxTransient->cbInstr == 2);
7867	}
7868	return rc;
7869	}
7870
7871
7872	/**
7873	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
7874	*/
7875	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7876	{
7877	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7878	PVM pVM = pVCpu->CTX_SUFF(pVM);
7879	int rc = VINF_SUCCESS;
7880
7881	/* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
7882	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7883	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7884	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7885	AssertRCReturn(rc, rc);
7886	Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
7887
7888	rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7889	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
7890	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
7891
7892	if (RT_LIKELY(rc == VINF_SUCCESS))
7893	{
7894	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7895
7896	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
7897	if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
7898	&& pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
7899	{
7900	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_APIC_STATE);
7901	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
7902	}
7903	else if (pMixedCtx->ecx == MSR_K6_EFER) /* EFER is the only MSR we auto-load but don't allow write-passthrough. */
7904	{
7905	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
7906	AssertRCReturn(rc, rc);
7907	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_AUTO_MSRS;
7908	}
7909	else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
7910	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7911
7912	/* Update MSRs that are part of the VMCS when MSR-bitmaps are not supported. */
7913	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
7914	{
7915	switch (pMixedCtx->ecx)
7916	{
7917	case MSR_IA32_SYSENTER_CS: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_CS_MSR; break;
7918	case MSR_IA32_SYSENTER_EIP: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_EIP_MSR; break;
7919	case MSR_IA32_SYSENTER_ESP: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_ESP_MSR; break;
7920	case MSR_K8_FS_BASE: /* no break */
7921	case MSR_K8_GS_BASE: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SEGMENT_REGS; break;
7922	case MSR_K8_KERNEL_GS_BASE: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_AUTO_MSRS; break;
7923	}
7924	}
7925	#ifdef VBOX_STRICT
7926	else
7927	{
7928	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
7929	switch (pMixedCtx->ecx)
7930	{
7931	case MSR_IA32_SYSENTER_CS:
7932	case MSR_IA32_SYSENTER_EIP:
7933	case MSR_IA32_SYSENTER_ESP:
7934	case MSR_K8_FS_BASE:
7935	case MSR_K8_GS_BASE:
7936	{
7937	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
7938	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7939	}
7940
7941	case MSR_K8_LSTAR:
7942	case MSR_K6_STAR:
7943	case MSR_K8_SF_MASK:
7944	case MSR_K8_TSC_AUX:
7945	case MSR_K8_KERNEL_GS_BASE:
7946	{
7947	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
7948	pMixedCtx->ecx));
7949	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7950	}
7951	}
7952	}
7953	#endif /* VBOX_STRICT */
7954	}
7955	return rc;
7956	}
7957
7958
7959	/**
7960	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
7961	*/
7962	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7963	{
7964	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7965
7966	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT. */
7967	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
7968	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT)
7969	return VERR_EM_INTERPRETER;
7970	AssertMsgFailed(("Unexpected PAUSE exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7971	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7972	}
7973
7974
7975	/**
7976	* VM-exit handler for when the TPR value is lowered below the specified
7977	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
7978	*/
7979	HMVMX_EXIT_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7980	{
7981	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
7982	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
7983
7984	/*
7985	* The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
7986	* the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and
7987	* resume guest execution.
7988	*/
7989	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
7990	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
7991	return VINF_SUCCESS;
7992	}
7993
7994
7995	/**
7996	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
7997	* VM-exit.
7998	*
7999	* @retval VINF_SUCCESS when guest execution can continue.
8000	* @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
8001	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
8002	* @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
8003	* recompiler.
8004	*/
8005	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8006	{
8007	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8008	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
8009	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8010	AssertRCReturn(rc, rc);
8011
8012	const RTGCUINTPTR uExitQualification = pVmxTransient->uExitQualification;
8013	const uint32_t uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
8014	PVM pVM = pVCpu->CTX_SUFF(pVM);
8015	switch (uAccessType)
8016	{
8017	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
8018	{
8019	#if 0
8020	/* EMInterpretCRxWrite() references a lot of guest state (EFER, RFLAGS, Segment Registers, etc.) Sync entire state */
8021	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8022	#else
8023	rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8024	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8025	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8026	#endif
8027	AssertRCReturn(rc, rc);
8028
8029	rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
8030	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
8031	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
8032	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3);
8033
8034	switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
8035	{
8036	case 0: /* CR0 */
8037	Log4(("CRX CR0 write rc=%d CR0=%#RX64\n", rc, pMixedCtx->cr0));
8038	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
8039	break;
8040	case 2: /* C2 **/
8041	/* Nothing to do here, CR2 it's not part of the VMCS. */
8042	break;
8043	case 3: /* CR3 */
8044	Assert(!pVM->hm.s.fNestedPaging \|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx));
8045	Log4(("CRX CR3 write rc=%d CR3=%#RX64\n", rc, pMixedCtx->cr3));
8046	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR3;
8047	break;
8048	case 4: /* CR4 */
8049	Log4(("CRX CR4 write rc=%d CR4=%#RX64\n", rc, pMixedCtx->cr4));
8050	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR4;
8051	break;
8052	case 8: /* CR8 */
8053	Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
8054	/* CR8 contains the APIC TPR. Was updated by EMInterpretCRxWrite(). */
8055	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
8056	break;
8057	default:
8058	AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
8059	break;
8060	}
8061
8062	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
8063	break;
8064	}
8065
8066	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
8067	{
8068	/* EMInterpretCRxRead() requires EFER MSR, CS. */
8069	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8070	AssertRCReturn(rc, rc);
8071	Assert( !pVM->hm.s.fNestedPaging
8072	\|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx)
8073	\|\| VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
8074
8075	/* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
8076	Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
8077	\|\| !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
8078
8079	rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
8080	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
8081	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
8082	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
8083	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
8084	Log4(("CRX CR%d Read access rc=%d\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification), rc));
8085	break;
8086	}
8087
8088	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
8089	{
8090	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8091	AssertRCReturn(rc, rc);
8092	rc = EMInterpretCLTS(pVM, pVCpu);
8093	AssertRCReturn(rc, rc);
8094	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
8095	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
8096	Log4(("CRX CLTS write rc=%d\n", rc));
8097	break;
8098	}
8099
8100	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
8101	{
8102	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8103	AssertRCReturn(rc, rc);
8104	rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
8105	if (RT_LIKELY(rc == VINF_SUCCESS))
8106	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
8107	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
8108	Log4(("CRX LMSW write rc=%d\n", rc));
8109	break;
8110	}
8111
8112	default:
8113	{
8114	AssertMsgFailed(("Invalid access-type in Mov CRx exit qualification %#x\n", uAccessType));
8115	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
8116	}
8117	}
8118
8119	/* Validate possible error codes. */
8120	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_SYNC_CR3
8121	\|\| rc == VERR_VMX_UNEXPECTED_EXCEPTION);
8122	if (RT_SUCCESS(rc))
8123	{
8124	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8125	AssertRCReturn(rc2, rc2);
8126	}
8127
8128	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
8129	return rc;
8130	}
8131
8132
8133	/**
8134	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
8135	* VM-exit.
8136	*/
8137	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8138	{
8139	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8140	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
8141
8142	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8143	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8144	rc \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
8145	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
8146	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
8147	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
8148	/* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
8149	AssertRCReturn(rc, rc);
8150
8151	Log4(("CS:RIP=%04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
8152
8153	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
8154	uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
8155	uint32_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
8156	bool fIOWrite = (VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
8157	== VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
8158	bool fIOString = (VMX_EXIT_QUALIFICATION_IO_STRING(pVmxTransient->uExitQualification) == 1);
8159	Assert(uIOWidth == 0 \|\| uIOWidth == 1 \|\| uIOWidth == 3);
8160
8161	/* I/O operation lookup arrays. */
8162	static const uint32_t s_aIOSize[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
8163	static const uint32_t s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
8164
8165	const uint32_t cbSize = s_aIOSize[uIOWidth];
8166	const uint32_t cbInstr = pVmxTransient->cbInstr;
8167	PVM pVM = pVCpu->CTX_SUFF(pVM);
8168	if (fIOString)
8169	{
8170	/* INS/OUTS - I/O String instruction. */
8171	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
8172	/** @todo for now manually disassemble later optimize by getting the fields from
8173	* the VMCS. VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR contains the flat pointer
8174	* operand of the instruction. VMX_VMCS32_RO_EXIT_INSTR_INFO contains
8175	* segment prefix info. */
8176	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL);
8177	if (RT_SUCCESS(rc))
8178	{
8179	if (fIOWrite)
8180	{
8181	VBOXSTRICTRC rc2 = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
8182	(DISCPUMODE)pDis->uAddrMode, cbSize);
8183	rc = VBOXSTRICTRC_VAL(rc2);
8184	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
8185	}
8186	else
8187	{
8188	VBOXSTRICTRC rc2 = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
8189	(DISCPUMODE)pDis->uAddrMode, cbSize);
8190	rc = VBOXSTRICTRC_VAL(rc2);
8191	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
8192	}
8193	}
8194	else
8195	{
8196	AssertMsg(rc == VERR_EM_INTERPRETER, ("rc=%Rrc RIP %#RX64\n", rc, pMixedCtx->rip));
8197	rc = VINF_EM_RAW_EMULATE_INSTR;
8198	}
8199	}
8200	else
8201	{
8202	/* IN/OUT - I/O instruction. */
8203	const uint32_t uAndVal = s_aIOOpAnd[uIOWidth];
8204	Assert(!VMX_EXIT_QUALIFICATION_IO_REP(pVmxTransient->uExitQualification));
8205	if (fIOWrite)
8206	{
8207	VBOXSTRICTRC rc2 = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbSize);
8208	rc = VBOXSTRICTRC_VAL(rc2);
8209	if (rc == VINF_IOM_R3_IOPORT_WRITE)
8210	HMR0SavePendingIOPortWrite(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbSize);
8211	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
8212	}
8213	else
8214	{
8215	uint32_t u32Result = 0;
8216	VBOXSTRICTRC rc2 = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbSize);
8217	rc = VBOXSTRICTRC_VAL(rc2);
8218	if (IOM_SUCCESS(rc))
8219	{
8220	/* Save result of I/O IN instr. in AL/AX/EAX. */
8221	pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
8222	}
8223	else if (rc == VINF_IOM_R3_IOPORT_READ)
8224	HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbSize);
8225	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
8226	}
8227	}
8228
8229	if (IOM_SUCCESS(rc))
8230	{
8231	pMixedCtx->rip += cbInstr;
8232	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
8233	if (RT_LIKELY(rc == VINF_SUCCESS))
8234	{
8235	rc = hmR0VmxSaveGuestDebugRegs(pVCpu, pMixedCtx); /* For DR7. */
8236	AssertRCReturn(rc, rc);
8237
8238	/* If any IO breakpoints are armed, then we should check if a debug trap needs to be generated. */
8239	if (pMixedCtx->dr[7] & X86_DR7_ENABLED_MASK)
8240	{
8241	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
8242	for (unsigned i = 0; i < 4; i++)
8243	{
8244	uint32_t uBPLen = s_aIOSize[X86_DR7_GET_LEN(pMixedCtx->dr[7], i)];
8245	if ( ( uIOPort >= pMixedCtx->dr[i]
8246	&& uIOPort < pMixedCtx->dr[i] + uBPLen)
8247	&& (pMixedCtx->dr[7] & (X86_DR7_L(i) \| X86_DR7_G(i)))
8248	&& (pMixedCtx->dr[7] & X86_DR7_RW(i, X86_DR7_RW_IO)) == X86_DR7_RW(i, X86_DR7_RW_IO))
8249	{
8250	Assert(CPUMIsGuestDebugStateActive(pVCpu));
8251	uint64_t uDR6 = ASMGetDR6();
8252
8253	/* Clear all breakpoint status flags and set the one we just hit. */
8254	uDR6 &= ~(X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3);
8255	uDR6 \|= (uint64_t)RT_BIT(i);
8256
8257	/*
8258	* Note: AMD64 Architecture Programmer's Manual 13.1:
8259	* Bits 15:13 of the DR6 register is never cleared by the processor and must
8260	* be cleared by software after the contents have been read.
8261	*/
8262	ASMSetDR6(uDR6);
8263
8264	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
8265	pMixedCtx->dr[7] &= ~X86_DR7_GD;
8266
8267	/* Paranoia. */
8268	pMixedCtx->dr[7] &= 0xffffffff; /* Upper 32 bits MBZ. */
8269	pMixedCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* MBZ. */
8270	pMixedCtx->dr[7] \|= 0x400; /* MB1. */
8271
8272	/* Resync DR7 */
8273	/** @todo probably cheaper to just reload DR7, nothing else needs changing. */
8274	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
8275
8276	/* Set #DB to be injected into the VM and continue guest execution. */
8277	hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
8278	break;
8279	}
8280	}
8281	}
8282	}
8283	}
8284
8285	#ifdef DEBUG
8286	if (rc == VINF_IOM_R3_IOPORT_READ)
8287	Assert(!fIOWrite);
8288	else if (rc == VINF_IOM_R3_IOPORT_WRITE)
8289	Assert(fIOWrite);
8290	else
8291	{
8292	AssertMsg( RT_FAILURE(rc)
8293	\|\| rc == VINF_SUCCESS
8294	\|\| rc == VINF_EM_RAW_EMULATE_INSTR
8295	\|\| rc == VINF_EM_RAW_GUEST_TRAP
8296	\|\| rc == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", rc));
8297	}
8298	#endif
8299
8300	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
8301	return rc;
8302	}
8303
8304
8305	/**
8306	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
8307	* VM-exit.
8308	*/
8309	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8310	{
8311	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8312
8313	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
8314	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8315	AssertRCReturn(rc, rc);
8316	if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
8317	{
8318	rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
8319	AssertRCReturn(rc, rc);
8320	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
8321	{
8322	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
8323
8324	/* Software interrupts and exceptions will be regenerated when the recompiler restarts the instruction. */
8325	if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
8326	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
8327	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
8328	{
8329	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
8330	bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
8331
8332	/* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
8333	Assert(!pVCpu->hm.s.Event.fPending);
8334	pVCpu->hm.s.Event.fPending = true;
8335	pVCpu->hm.s.Event.u64IntrInfo = pVmxTransient->uIdtVectoringInfo;
8336	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
8337	AssertRCReturn(rc, rc);
8338	if (fErrorCodeValid)
8339	pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
8340	else
8341	pVCpu->hm.s.Event.u32ErrCode = 0;
8342	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
8343	&& uVector == X86_XCPT_PF)
8344	{
8345	pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
8346	}
8347
8348	Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
8349	}
8350	}
8351	}
8352
8353	/** @todo Emulate task switch someday, currently just going back to ring-3 for
8354	* emulation. */
8355	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
8356	return VERR_EM_INTERPRETER;
8357	}
8358
8359
8360	/**
8361	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
8362	*/
8363	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8364	{
8365	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8366	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
8367	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
8368	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
8369	AssertRCReturn(rc, rc);
8370	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
8371	return VINF_EM_DBG_STOP;
8372	}
8373
8374
8375	/**
8376	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
8377	*/
8378	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8379	{
8380	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8381
8382	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8383	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
8384	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
8385	return VINF_SUCCESS;
8386	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8387	return rc;
8388
8389	#if 0
8390	/** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
8391	* just sync the whole thing. */
8392	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8393	#else
8394	/* Aggressive state sync. for now. */
8395	rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8396	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8397	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8398	#endif
8399	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8400	AssertRCReturn(rc, rc);
8401
8402	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
8403	uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
8404	switch (uAccessType)
8405	{
8406	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
8407	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
8408	{
8409	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8410	&& VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) == 0x80)
8411	{
8412	AssertMsgFailed(("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
8413	}
8414
8415	RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
8416	GCPhys &= PAGE_BASE_GC_MASK;
8417	GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
8418	PVM pVM = pVCpu->CTX_SUFF(pVM);
8419	Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGv Off=%#x\n", uAccessType, GCPhys,
8420	VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
8421
8422	VBOXSTRICTRC rc2 = IOMMMIOPhysHandler(pVM, pVCpu,
8423	(uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW,
8424	CPUMCTX2CORE(pMixedCtx), GCPhys);
8425	rc = VBOXSTRICTRC_VAL(rc2);
8426	Log4(("ApicAccess rc=%d\n", rc));
8427	if ( rc == VINF_SUCCESS
8428	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8429	\|\| rc == VERR_PAGE_NOT_PRESENT)
8430	{
8431	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8432	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8433	rc = VINF_SUCCESS;
8434	}
8435	break;
8436	}
8437
8438	default:
8439	Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
8440	rc = VINF_EM_RAW_EMULATE_INSTR;
8441	break;
8442	}
8443
8444	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
8445	return rc;
8446	}
8447
8448
8449	/**
8450	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
8451	* VM-exit.
8452	*/
8453	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8454	{
8455	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8456
8457	/* We should -not- get this VM-exit if the guest is debugging. */
8458	if (CPUMIsGuestDebugStateActive(pVCpu))
8459	{
8460	AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
8461	return VERR_VMX_UNEXPECTED_EXIT_CODE;
8462	}
8463
8464	int rc = VERR_INTERNAL_ERROR_5;
8465	if ( !DBGFIsStepping(pVCpu)
8466	&& !CPUMIsHyperDebugStateActive(pVCpu))
8467	{
8468	/* Don't intercept MOV DRx. */
8469	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
8470	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
8471	AssertRCReturn(rc, rc);
8472
8473	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
8474	PVM pVM = pVCpu->CTX_SUFF(pVM);
8475	rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pMixedCtx, true /* include DR6 */);
8476	AssertRC(rc);
8477	Assert(CPUMIsGuestDebugStateActive(pVCpu));
8478
8479	#ifdef VBOX_WITH_STATISTICS
8480	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8481	AssertRCReturn(rc, rc);
8482	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
8483	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
8484	else
8485	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
8486	#endif
8487	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
8488	return VINF_SUCCESS;
8489	}
8490
8491	/*
8492	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date, see
8493	* hmR0VmxSaveGuestAutoLoadStoreMsrs(). Update only the segment registers from the CPU.
8494	*/
8495	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8496	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8497	AssertRCReturn(rc, rc);
8498
8499	PVM pVM = pVCpu->CTX_SUFF(pVM);
8500	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
8501	{
8502	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
8503	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
8504	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
8505	if (RT_SUCCESS(rc))
8506	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
8507	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
8508	}
8509	else
8510	{
8511	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
8512	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
8513	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
8514	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
8515	}
8516
8517	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
8518	if (RT_SUCCESS(rc))
8519	{
8520	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8521	AssertRCReturn(rc2, rc2);
8522	}
8523	return rc;
8524	}
8525
8526
8527	/**
8528	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
8529	* Conditional VM-exit.
8530	*/
8531	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8532	{
8533	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8534	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
8535
8536	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8537	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
8538	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
8539	return VINF_SUCCESS;
8540	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8541	return rc;
8542
8543	RTGCPHYS GCPhys = 0;
8544	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
8545
8546	#if 0
8547	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
8548	#else
8549	/* Aggressive state sync. for now. */
8550	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8551	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8552	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8553	#endif
8554	AssertRCReturn(rc, rc);
8555
8556	/*
8557	* If we succeed, resume guest execution.
8558	* If we fail in interpreting the instruction because we couldn't get the guest physical address
8559	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
8560	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
8561	* weird case. See @bugref{6043}.
8562	*/
8563	PVM pVM = pVCpu->CTX_SUFF(pVM);
8564	VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
8565	rc = VBOXSTRICTRC_VAL(rc2);
8566	Log4(("EPT misconfig at %#RGv RIP=%#RX64 rc=%d\n", GCPhys, pMixedCtx->rip, rc));
8567	if ( rc == VINF_SUCCESS
8568	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8569	\|\| rc == VERR_PAGE_NOT_PRESENT)
8570	{
8571	/* Successfully handled MMIO operation. */
8572	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8573	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8574	rc = VINF_SUCCESS;
8575	}
8576	return rc;
8577	}
8578
8579
8580	/**
8581	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
8582	* VM-exit.
8583	*/
8584	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8585	{
8586	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8587	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
8588
8589	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8590	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
8591	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
8592	return VINF_SUCCESS;
8593	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8594	return rc;
8595
8596	RTGCPHYS GCPhys = 0;
8597	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
8598	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8599	#if 0
8600	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
8601	#else
8602	/* Aggressive state sync. for now. */
8603	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8604	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8605	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8606	#endif
8607	AssertRCReturn(rc, rc);
8608
8609	/* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
8610	AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
8611
8612	RTGCUINT uErrorCode = 0;
8613	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
8614	uErrorCode \|= X86_TRAP_PF_ID;
8615	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
8616	uErrorCode \|= X86_TRAP_PF_RW;
8617	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
8618	uErrorCode \|= X86_TRAP_PF_P;
8619
8620	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
8621
8622	Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:EIP=%04x:%#RX64\n", pVmxTransient->uExitQualification, GCPhys,
8623	uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
8624
8625	/* Handle the pagefault trap for the nested shadow table. */
8626	PVM pVM = pVCpu->CTX_SUFF(pVM);
8627	rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
8628	TRPMResetTrap(pVCpu);
8629
8630	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
8631	if ( rc == VINF_SUCCESS
8632	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8633	\|\| rc == VERR_PAGE_NOT_PRESENT)
8634	{
8635	/* Successfully synced our nested page tables. */
8636	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
8637	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS;
8638	return VINF_SUCCESS;
8639	}
8640
8641	Log4(("EPT return to ring-3 rc=%d\n"));
8642	return rc;
8643	}
8644
8645
8646	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8647	/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
8648	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8649	/**
8650	* VM-exit exception handler for #MF (Math Fault: floating point exception).
8651	*/
8652	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8653	{
8654	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8655	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
8656
8657	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8658	AssertRCReturn(rc, rc);
8659
8660	if (!(pMixedCtx->cr0 & X86_CR0_NE))
8661	{
8662	/* Old-style FPU error reporting needs some extra work. */
8663	/** @todo don't fall back to the recompiler, but do it manually. */
8664	return VERR_EM_INTERPRETER;
8665	}
8666	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8667	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8668	return rc;
8669	}
8670
8671
8672	/**
8673	* VM-exit exception handler for #BP (Breakpoint exception).
8674	*/
8675	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8676	{
8677	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8678	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
8679
8680	/** @todo Try optimize this by not saving the entire guest state unless
8681	* really needed. */
8682	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8683	AssertRCReturn(rc, rc);
8684
8685	PVM pVM = pVCpu->CTX_SUFF(pVM);
8686	rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8687	if (rc == VINF_EM_RAW_GUEST_TRAP)
8688	{
8689	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8690	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8691	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8692	AssertRCReturn(rc, rc);
8693
8694	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8695	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8696	}
8697
8698	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_EM_DBG_BREAKPOINT);
8699	return rc;
8700	}
8701
8702
8703	/**
8704	* VM-exit exception handler for #DB (Debug exception).
8705	*/
8706	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8707	{
8708	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8709	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
8710
8711	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8712	AssertRCReturn(rc, rc);
8713
8714	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
8715	uint64_t uDR6 = X86_DR6_INIT_VAL;
8716	uDR6 \|= (pVmxTransient->uExitQualification
8717	& (X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3 \| X86_DR6_BD \| X86_DR6_BS));
8718	PVM pVM = pVCpu->CTX_SUFF(pVM);
8719	rc = DBGFRZTrap01Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6);
8720	if (rc == VINF_EM_RAW_GUEST_TRAP)
8721	{
8722	/* DR6, DR7.GD and IA32_DEBUGCTL.LBR are not updated yet. See Intel spec. 27.1 "Architectural State before a VM-Exit". */
8723	pMixedCtx->dr[6] = uDR6;
8724
8725	if (CPUMIsGuestDebugStateActive(pVCpu))
8726	ASMSetDR6(pMixedCtx->dr[6]);
8727
8728	rc = hmR0VmxSaveGuestDebugRegs(pVCpu, pMixedCtx);
8729
8730	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
8731	pMixedCtx->dr[7] &= ~X86_DR7_GD;
8732
8733	/* Paranoia. */
8734	pMixedCtx->dr[7] &= 0xffffffff; /* Upper 32 bits MBZ. */
8735	pMixedCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* MBZ. */
8736	pMixedCtx->dr[7] \|= 0x400; /* MB1. */
8737
8738	rc \|= VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
8739	AssertRCReturn(rc,rc);
8740
8741	int rc2 = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8742	rc2 \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8743	rc2 \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8744	AssertRCReturn(rc2, rc2);
8745	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8746	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8747	rc = VINF_SUCCESS;
8748	}
8749
8750	return rc;
8751	}
8752
8753
8754	/**
8755	* VM-exit exception handler for #NM (Device-not-available exception: floating
8756	* point exception).
8757	*/
8758	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8759	{
8760	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8761
8762	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
8763	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
8764	#endif
8765
8766	/* We require CR0 and EFER. EFER is always up-to-date. */
8767	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8768	AssertRCReturn(rc, rc);
8769
8770	/* Lazy FPU loading; load the guest-FPU state transparently and continue execution of the guest. */
8771	PVM pVM = pVCpu->CTX_SUFF(pVM);
8772	rc = CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8773	if (rc == VINF_SUCCESS)
8774	{
8775	Assert(CPUMIsGuestFPUStateActive(pVCpu));
8776	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
8777	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
8778	return VINF_SUCCESS;
8779	}
8780
8781	/* Forward #NM to the guest. */
8782	Assert(rc == VINF_EM_RAW_GUEST_TRAP);
8783	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8784	AssertRCReturn(rc, rc);
8785	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8786	pVmxTransient->cbInstr, 0 /* error code /, 0 / GCPtrFaultAddress */);
8787	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
8788	return rc;
8789	}
8790
8791
8792	/**
8793	* VM-exit exception handler for #GP (General-protection exception).
8794	*
8795	* @remarks Requires pVmxTransient->uExitIntrInfo to be up-to-date.
8796	*/
8797	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8798	{
8799	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8800	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
8801
8802	int rc = VERR_INTERNAL_ERROR_5;
8803	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8804	{
8805	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
8806	/* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
8807	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8808	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8809	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8810	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8811	AssertRCReturn(rc, rc);
8812	Log4(("#GP Gst: RIP %#RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u\n", pMixedCtx->rip, pVmxTransient->uExitIntrErrorCode,
8813	pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu)));
8814	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8815	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8816	return rc;
8817	#else
8818	/* We don't intercept #GP. */
8819	AssertMsgFailed(("Unexpected VM-exit caused by #GP exception\n"));
8820	return VERR_VMX_UNEXPECTED_EXCEPTION;
8821	#endif
8822	}
8823
8824	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
8825	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
8826
8827	/* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
8828	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8829	AssertRCReturn(rc, rc);
8830
8831	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
8832	uint32_t cbOp = 0;
8833	PVM pVM = pVCpu->CTX_SUFF(pVM);
8834	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
8835	if (RT_SUCCESS(rc))
8836	{
8837	rc = VINF_SUCCESS;
8838	Assert(cbOp == pDis->cbInstr);
8839	Log4(("#GP Disas OpCode=%u CS:EIP %04x:%#RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
8840	switch (pDis->pCurInstr->uOpcode)
8841	{
8842	case OP_CLI:
8843	{
8844	pMixedCtx->eflags.Bits.u1IF = 0;
8845	pMixedCtx->rip += pDis->cbInstr;
8846	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS;
8847	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
8848	break;
8849	}
8850
8851	case OP_STI:
8852	{
8853	pMixedCtx->eflags.Bits.u1IF = 1;
8854	pMixedCtx->rip += pDis->cbInstr;
8855	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
8856	Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
8857	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS;
8858	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
8859	break;
8860	}
8861
8862	case OP_HLT:
8863	{
8864	rc = VINF_EM_HALT;
8865	pMixedCtx->rip += pDis->cbInstr;
8866	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
8867	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
8868	break;
8869	}
8870
8871	case OP_POPF:
8872	{
8873	Log4(("POPF CS:RIP %04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
8874	uint32_t cbParm = 0;
8875	uint32_t uMask = 0;
8876	if (pDis->fPrefix & DISPREFIX_OPSIZE)
8877	{
8878	cbParm = 4;
8879	uMask = 0xffffffff;
8880	}
8881	else
8882	{
8883	cbParm = 2;
8884	uMask = 0xffff;
8885	}
8886
8887	/* Get the stack pointer & pop the contents of the stack onto EFlags. */
8888	RTGCPTR GCPtrStack = 0;
8889	X86EFLAGS uEflags;
8890	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
8891	&GCPtrStack);
8892	if (RT_SUCCESS(rc))
8893	{
8894	Assert(sizeof(uEflags.u32) >= cbParm);
8895	uEflags.u32 = 0;
8896	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &uEflags.u32, cbParm);
8897	}
8898	if (RT_FAILURE(rc))
8899	{
8900	rc = VERR_EM_INTERPRETER;
8901	break;
8902	}
8903	Log4(("POPF %x -> %#RX64 mask=%x RIP=%#RX64\n", uEflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
8904	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
8905	\| (uEflags.u32 & X86_EFL_POPF_BITS & uMask);
8906	/* The RF bit is always cleared by POPF; see Intel Instruction reference for POPF. */
8907	pMixedCtx->eflags.Bits.u1RF = 0;
8908	pMixedCtx->esp += cbParm;
8909	pMixedCtx->esp &= uMask;
8910	pMixedCtx->rip += pDis->cbInstr;
8911	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS;
8912	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
8913	break;
8914	}
8915
8916	case OP_PUSHF:
8917	{
8918	uint32_t cbParm = 0;
8919	uint32_t uMask = 0;
8920	if (pDis->fPrefix & DISPREFIX_OPSIZE)
8921	{
8922	cbParm = 4;
8923	uMask = 0xffffffff;
8924	}
8925	else
8926	{
8927	cbParm = 2;
8928	uMask = 0xffff;
8929	}
8930
8931	/* Get the stack pointer & push the contents of eflags onto the stack. */
8932	RTGCPTR GCPtrStack = 0;
8933	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
8934	SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
8935	if (RT_FAILURE(rc))
8936	{
8937	rc = VERR_EM_INTERPRETER;
8938	break;
8939	}
8940	X86EFLAGS uEflags;
8941	uEflags = pMixedCtx->eflags;
8942	/* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
8943	uEflags.Bits.u1RF = 0;
8944	uEflags.Bits.u1VM = 0;
8945
8946	rc = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &uEflags.u, cbParm);
8947	if (RT_FAILURE(rc))
8948	{
8949	rc = VERR_EM_INTERPRETER;
8950	break;
8951	}
8952	Log4(("PUSHF %x -> %#RGv\n", uEflags.u, GCPtrStack));
8953	pMixedCtx->esp -= cbParm;
8954	pMixedCtx->esp &= uMask;
8955	pMixedCtx->rip += pDis->cbInstr;
8956	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP;
8957	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
8958	break;
8959	}
8960
8961	case OP_IRET:
8962	{
8963	/** @todo Handle 32-bit operand sizes and check stack limits. See Intel
8964	* instruction reference. */
8965	RTGCPTR GCPtrStack = 0;
8966	uint32_t uMask = 0xffff;
8967	uint16_t aIretFrame[3];
8968	if (pDis->fPrefix & (DISPREFIX_OPSIZE \| DISPREFIX_ADDRSIZE))
8969	{
8970	rc = VERR_EM_INTERPRETER;
8971	break;
8972	}
8973	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
8974	&GCPtrStack);
8975	if (RT_SUCCESS(rc))
8976	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
8977	if (RT_FAILURE(rc))
8978	{
8979	rc = VERR_EM_INTERPRETER;
8980	break;
8981	}
8982	pMixedCtx->eip = 0;
8983	pMixedCtx->ip = aIretFrame[0];
8984	pMixedCtx->cs.Sel = aIretFrame[1];
8985	pMixedCtx->cs.ValidSel = aIretFrame[1];
8986	pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
8987	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
8988	\| (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
8989	pMixedCtx->sp += sizeof(aIretFrame);
8990	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_SEGMENT_REGS \| HM_CHANGED_GUEST_RSP
8991	\| HM_CHANGED_GUEST_RFLAGS;
8992	Log4(("IRET %#RX32 to %04x:%x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
8993	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
8994	break;
8995	}
8996
8997	case OP_INT:
8998	{
8999	uint16_t uVector = pDis->Param1.uValue & 0xff;
9000	hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
9001	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
9002	break;
9003	}
9004
9005	case OP_INTO:
9006	{
9007	if (pMixedCtx->eflags.Bits.u1OF)
9008	{
9009	hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
9010	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
9011	}
9012	break;
9013	}
9014
9015	default:
9016	{
9017	VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
9018	EMCODETYPE_SUPERVISOR);
9019	rc = VBOXSTRICTRC_VAL(rc2);
9020	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_ALL_GUEST;
9021	Log4(("#GP rc=%Rrc\n", rc));
9022	break;
9023	}
9024	}
9025	}
9026	else
9027	rc = VERR_EM_INTERPRETER;
9028
9029	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_EM_HALT,
9030	("#GP Unexpected rc=%Rrc\n", rc));
9031	return rc;
9032	}
9033
9034
9035	/**
9036	* VM-exit exception handler wrapper for generic exceptions. Simply re-injects
9037	* the exception reported in the VMX transient structure back into the VM.
9038	*
9039	* @remarks Requires uExitIntrInfo in the VMX transient structure to be
9040	* up-to-date.
9041	*/
9042	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9043	{
9044	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
9045
9046	/* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
9047	hmR0VmxCheckExitDueToEventDelivery(). */
9048	int rc = hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
9049	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
9050	AssertRCReturn(rc, rc);
9051	Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
9052	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
9053	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
9054	return VINF_SUCCESS;
9055	}
9056
9057
9058	/**
9059	* VM-exit exception handler for #PF (Page-fault exception).
9060	*/
9061	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9062	{
9063	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
9064	PVM pVM = pVCpu->CTX_SUFF(pVM);
9065	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9066	rc \|= hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
9067	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
9068	AssertRCReturn(rc, rc);
9069
9070	#if defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) \|\| defined(HMVMX_ALWAYS_TRAP_PF)
9071	if (pVM->hm.s.fNestedPaging)
9072	{
9073	if (RT_LIKELY(!pVmxTransient->fVectoringPF))
9074	{
9075	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
9076	pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
9077	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
9078	0 /* cbInstr */, pVmxTransient->uExitIntrErrorCode, pVmxTransient->uExitQualification);
9079	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
9080	}
9081	else
9082	{
9083	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
9084	pVCpu->hm.s.Event.fPending = false; /* A vectoring #PF. */
9085	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
9086	Log4(("Pending #DF due to vectoring #PF. NP\n"));
9087	}
9088	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
9089	return rc;
9090	}
9091	#else
9092	Assert(!pVM->hm.s.fNestedPaging);
9093	#endif
9094
9095	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9096	AssertRCReturn(rc, rc);
9097
9098	Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
9099	pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntrErrorCode, pMixedCtx->cr3));
9100
9101	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntrErrorCode);
9102	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntrErrorCode, CPUMCTX2CORE(pMixedCtx),
9103	(RTGCPTR)pVmxTransient->uExitQualification);
9104
9105	Log4(("#PF: rc=%Rrc\n", rc));
9106	if (rc == VINF_SUCCESS)
9107	{
9108	/* Successfully synced shadow pages tables or emulated an MMIO instruction. */
9109	/** @todo this isn't quite right, what if guest does lgdt with some MMIO
9110	* memory? We don't update the whole state here... */
9111	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
9112	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
9113	TRPMResetTrap(pVCpu);
9114	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
9115	return rc;
9116	}
9117	else if (rc == VINF_EM_RAW_GUEST_TRAP)
9118	{
9119	if (!pVmxTransient->fVectoringPF)
9120	{
9121	/* It's a guest page fault and needs to be reflected to the guest. */
9122	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
9123	TRPMResetTrap(pVCpu);
9124	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
9125	pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
9126	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
9127	0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
9128	}
9129	else
9130	{
9131	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
9132	TRPMResetTrap(pVCpu);
9133	pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF for replace it with #DF. */
9134	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
9135	Log4(("#PF: Pending #DF due to vectoring #PF\n"));
9136	}
9137
9138	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
9139	return VINF_SUCCESS;
9140	}
9141
9142	TRPMResetTrap(pVCpu);
9143	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
9144	return rc;
9145	}
9146

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 46587

Download in other formats: