HMVMXR0.cpp@ 46150

Last change on this file since 46150 was 46143, checked in by vboxsync, 11 years ago
VMM/HMVMXR0: Make single-stepping work in the hypervisor debugger when the monitor-trap flag is not available.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 380.9 KB

Line
1	/* $Id: HMVMXR0.cpp 46143 2013-05-17 14:20:53Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012-2013 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18	/*******************************************************************************
19	* Header Files *
20	*******************************************************************************/
21	#define LOG_GROUP LOG_GROUP_HM
22	#include <iprt/asm-amd64-x86.h>
23	#include <iprt/thread.h>
24	#include <iprt/string.h>
25
26	#include "HMInternal.h"
27	#include <VBox/vmm/vm.h>
28	#include "HWVMXR0.h"
29	#include <VBox/vmm/pdmapi.h>
30	#include <VBox/vmm/dbgf.h>
31	#include <VBox/vmm/iom.h>
32	#include <VBox/vmm/selm.h>
33	#include <VBox/vmm/tm.h>
34	#ifdef VBOX_WITH_REM
35	# include <VBox/vmm/rem.h>
36	#endif
37	#ifdef DEBUG_ramshankar
38	#define HMVMX_SAVE_FULL_GUEST_STATE
39	#define HMVMX_SYNC_FULL_GUEST_STATE
40	#define HMVMX_ALWAYS_TRAP_ALL_XCPTS
41	#define HMVMX_ALWAYS_TRAP_PF
42	#endif
43
44
45	/*******************************************************************************
46	* Defined Constants And Macros *
47	*******************************************************************************/
48	#define HMVMXHCUINTREG RTHCUINTREG
49	#if defined(RT_ARCH_AMD64)
50	# define HMVMX_IS_64BIT_HOST_MODE() (true)
51	#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
52	extern "C" uint32_t g_fVMXIs64bitHost;
53	# define HMVMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
54	# undef HMVMXHCUINTREG
55	# define HMVMXHCUINTREG uint64_t
56	#else
57	# define HMVMX_IS_64BIT_HOST_MODE() (false)
58	#endif
59
60	/** Use the function table. */
61	#define HMVMX_USE_FUNCTION_TABLE
62
63	/** This bit indicates the segment selector is unusable in VT-x. */
64	#define HMVMX_SEL_UNUSABLE RT_BIT(16)
65
66	/** Determine which tagged-TLB flush handler to use. */
67	#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
68	#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
69	#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
70	#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
71
72	/** @name Updated-guest-state flags.
73	* @{ */
74	#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
75	#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
76	#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
77	#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
78	#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
79	#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
80	#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
81	#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
82	#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
83	#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
84	#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
85	#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
86	#define HMVMX_UPDATED_GUEST_FS_BASE_MSR RT_BIT(12)
87	#define HMVMX_UPDATED_GUEST_GS_BASE_MSR RT_BIT(13)
88	#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(14)
89	#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(15)
90	#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(16)
91	#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(17)
92	#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(18)
93	#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(19)
94	#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
95	\| HMVMX_UPDATED_GUEST_RSP \
96	\| HMVMX_UPDATED_GUEST_RFLAGS \
97	\| HMVMX_UPDATED_GUEST_CR0 \
98	\| HMVMX_UPDATED_GUEST_CR3 \
99	\| HMVMX_UPDATED_GUEST_CR4 \
100	\| HMVMX_UPDATED_GUEST_GDTR \
101	\| HMVMX_UPDATED_GUEST_IDTR \
102	\| HMVMX_UPDATED_GUEST_LDTR \
103	\| HMVMX_UPDATED_GUEST_TR \
104	\| HMVMX_UPDATED_GUEST_SEGMENT_REGS \
105	\| HMVMX_UPDATED_GUEST_DEBUG \
106	\| HMVMX_UPDATED_GUEST_FS_BASE_MSR \
107	\| HMVMX_UPDATED_GUEST_GS_BASE_MSR \
108	\| HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
109	\| HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
110	\| HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
111	\| HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
112	\| HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
113	\| HMVMX_UPDATED_GUEST_APIC_STATE)
114	/** @} */
115
116	/**
117	* Flags to skip redundant reads of some common VMCS fields that are not part of
118	* the guest-CPU state but are in the transient structure.
119	*/
120	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
121	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
122	#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
123	#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
124	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
125	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
126
127	/**
128	* Exception bitmap mask for real-mode guests (real-on-v86). We need to intercept all exceptions manually (except #PF).
129	* #NM is also handled spearetely, see hmR0VmxLoadGuestControlRegs(). #PF need not be intercepted even in real-mode if
130	* we have Nested Paging support.
131	*/
132	#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) \| RT_BIT(X86_XCPT_DB) \| RT_BIT(X86_XCPT_NMI) \
133	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
134	\| RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ \| RT_BIT(X86_XCPT_DF) \
135	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
136	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
137	\| RT_BIT(X86_XCPT_MF) \| RT_BIT(X86_XCPT_AC) \| RT_BIT(X86_XCPT_MC) \
138	\| RT_BIT(X86_XCPT_XF))
139
140	/**
141	* Exception bitmap mask for all contributory exceptions.
142	*/
143	#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) \| RT_BIT(X86_XCPT_NP) \| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_TS) \
144	\| RT_BIT(X86_XCPT_DE))
145
146	/** Maximum VM-instruction error number. */
147	#define HMVMX_INSTR_ERROR_MAX 28
148
149	/** Profiling macro. */
150	#ifdef HM_PROFILE_EXIT_DISPATCH
151	# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
152	# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
153	#else
154	# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
155	# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
156	#endif
157
158
159	/*******************************************************************************
160	* Structures and Typedefs *
161	*******************************************************************************/
162	/**
163	* A state structure for holding miscellaneous information across
164	* VMX non-root operation and restored after the transition.
165	*/
166	typedef struct VMXTRANSIENT
167	{
168	/** The host's rflags/eflags. */
169	RTCCUINTREG uEFlags;
170	#if HC_ARCH_BITS == 32
171	uint32_t u32Alignment0;
172	#endif
173	/** The guest's LSTAR MSR value used for TPR patching for 32-bit guests. */
174	uint64_t u64LStarMsr;
175	/** The guest's TPR value used for TPR shadowing. */
176	uint8_t u8GuestTpr;
177	/** Alignment. */
178	uint8_t abAlignment0[6];
179
180	/** The basic VM-exit reason. */
181	uint16_t uExitReason;
182	/** Alignment. */
183	uint16_t u16Alignment0;
184	/** The VM-exit interruption error code. */
185	uint32_t uExitIntrErrorCode;
186	/** The VM-exit exit qualification. */
187	uint64_t uExitQualification;
188
189	/** The VM-exit interruption-information field. */
190	uint32_t uExitIntrInfo;
191	/** The VM-exit instruction-length field. */
192	uint32_t cbInstr;
193	/** Whether the VM-entry failed or not. */
194	bool fVMEntryFailed;
195	/** Alignment. */
196	uint8_t abAlignment1[5];
197
198	/** The VM-entry interruption-information field. */
199	uint32_t uEntryIntrInfo;
200	/** The VM-entry exception error code field. */
201	uint32_t uEntryXcptErrorCode;
202	/** The VM-entry instruction length field. */
203	uint32_t cbEntryInstr;
204
205	/** IDT-vectoring information field. */
206	uint32_t uIdtVectoringInfo;
207	/** IDT-vectoring error code. */
208	uint32_t uIdtVectoringErrorCode;
209
210	/** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_. /
211	uint32_t fVmcsFieldsRead;
212	/** Whether TSC-offsetting should be setup before VM-entry. */
213	bool fUpdateTscOffsettingAndPreemptTimer;
214	/** Whether the VM-exit was caused by a page-fault during delivery of a
215	* contributary exception or a page-fault. */
216	bool fVectoringPF;
217	} VMXTRANSIENT, *PVMXTRANSIENT;
218	AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
219	AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntrInfo, sizeof(uint64_t));
220	AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntrInfo, sizeof(uint64_t));
221
222
223	/**
224	* MSR-bitmap read permissions.
225	*/
226	typedef enum VMXMSREXITREAD
227	{
228	/** Reading this MSR causes a VM-exit. */
229	VMXMSREXIT_INTERCEPT_READ = 0xb,
230	/** Reading this MSR does not cause a VM-exit. */
231	VMXMSREXIT_PASSTHRU_READ
232	} VMXMSREXITREAD;
233
234	/**
235	* MSR-bitmap write permissions.
236	*/
237	typedef enum VMXMSREXITWRITE
238	{
239	/** Writing to this MSR causes a VM-exit. */
240	VMXMSREXIT_INTERCEPT_WRITE = 0xd,
241	/** Writing to this MSR does not cause a VM-exit. */
242	VMXMSREXIT_PASSTHRU_WRITE
243	} VMXMSREXITWRITE;
244
245
246	/*******************************************************************************
247	* Internal Functions *
248	*******************************************************************************/
249	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr);
250	static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntrInfo, uint32_t cbInstr,
251	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState);
252	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
253	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
254	#endif
255	#ifndef HMVMX_USE_FUNCTION_TABLE
256	DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
257	#define HMVMX_EXIT_DECL static int
258	#else
259	#define HMVMX_EXIT_DECL static DECLCALLBACK(int)
260	#endif
261
262	HMVMX_EXIT_DECL hmR0VmxExitXcptNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
263	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
264	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
265	HMVMX_EXIT_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
266	HMVMX_EXIT_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
267	HMVMX_EXIT_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
268	HMVMX_EXIT_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
269	HMVMX_EXIT_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
270	HMVMX_EXIT_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
271	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
272	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
273	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
274	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
275	HMVMX_EXIT_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
276	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
277	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
278	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
279	HMVMX_EXIT_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
280	HMVMX_EXIT_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
281	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
282	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
283	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
284	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
285	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
286	HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
287	HMVMX_EXIT_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
288	HMVMX_EXIT_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
289	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
290	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
291	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
292	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
293	HMVMX_EXIT_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
294	HMVMX_EXIT_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
295	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
296	HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
297	HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
298	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
299	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
300	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
301	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
302	HMVMX_EXIT_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
303	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
304	HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
305	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
306
307	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
308	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
309	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
310	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
311	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
312	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
313	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
314
315
316	/*******************************************************************************
317	* Global Variables *
318	*******************************************************************************/
319	#ifdef HMVMX_USE_FUNCTION_TABLE
320	/**
321	* VM-exit handler.
322	*
323	* @returns VBox status code.
324	* @param pVCpu Pointer to the VMCPU.
325	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
326	* out-of-sync. Make sure to update the required
327	* fields before using them.
328	* @param pVmxTransient Pointer to the VMX-transient structure.
329	*/
330	typedef DECLCALLBACK(int) FNVMEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
331	/** Pointer to VM-exit handler. */
332	typedef FNVMEXITHANDLER *const PFNVMEXITHANDLER;
333
334	/**
335	* VMX_EXIT dispatch table.
336	*/
337	static const PFNVMEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
338	{
339	/* 00 VMX_EXIT_XCPT_NMI */ hmR0VmxExitXcptNmi,
340	/* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
341	/* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
342	/* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
343	/* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
344	/* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
345	/* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
346	/* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
347	/* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
348	/* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
349	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
350	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
351	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
352	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
353	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
354	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
355	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
356	/* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
357	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitSetPendingXcptUD,
358	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
359	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
360	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
361	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
362	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
363	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
364	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
365	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
366	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
367	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
368	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
369	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
370	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
371	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
372	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
373	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
374	/* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
375	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
376	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
377	/* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
378	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
379	/* 40 UNDEFINED */ hmR0VmxExitPause,
380	/* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
381	/* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
382	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
383	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
384	/* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
385	/* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
386	/* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
387	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
388	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
389	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
390	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
391	/* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
392	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
393	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
394	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
395	/* 56 UNDEFINED */ hmR0VmxExitErrUndefined,
396	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
397	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
398	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD
399	};
400	#endif /* HMVMX_USE_FUNCTION_TABLE */
401
402	#ifdef VBOX_STRICT
403	static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
404	{
405	/* 0 */ "(Not Used)",
406	/* 1 */ "VMCALL executed in VMX root operation.",
407	/* 2 */ "VMCLEAR with invalid physical address.",
408	/* 3 */ "VMCLEAR with VMXON pointer.",
409	/* 4 */ "VMLAUNCH with non-clear VMCS.",
410	/* 5 */ "VMRESUME with non-launched VMCS.",
411	/* 6 */ "VMRESUME after VMXOFF",
412	/* 7 */ "VM entry with invalid control fields.",
413	/* 8 */ "VM entry with invalid host state fields.",
414	/* 9 */ "VMPTRLD with invalid physical address.",
415	/* 10 */ "VMPTRLD with VMXON pointer.",
416	/* 11 */ "VMPTRLD with incorrect revision identifier.",
417	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
418	/* 13 */ "VMWRITE to read-only VMCS component.",
419	/* 14 */ "(Not Used)",
420	/* 15 */ "VMXON executed in VMX root operation.",
421	/* 16 */ "VM entry with invalid executive-VMCS pointer.",
422	/* 17 */ "VM entry with non-launched executing VMCS.",
423	/* 18 */ "VM entry with executive-VMCS pointer not VMXON pointer.",
424	/* 19 */ "VMCALL with non-clear VMCS.",
425	/* 20 */ "VMCALL with invalid VM-exit control fields.",
426	/* 21 */ "(Not Used)",
427	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
428	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
429	/* 24 */ "VMCALL with invalid SMM-monitor features.",
430	/* 25 */ "VM entry with invalid VM-execution control fields in executive VMCS.",
431	/* 26 */ "VM entry with events blocked by MOV SS.",
432	/* 27 */ "(Not Used)",
433	/* 28 */ "Invalid operand to INVEPT/INVVPID."
434	};
435	#endif /* VBOX_STRICT */
436
437
438
439	/**
440	* Updates the VM's last error record. If there was a VMX instruction error,
441	* reads the error data from the VMCS and updates VCPU's last error record as
442	* well.
443	*
444	* @param pVM Pointer to the VM.
445	* @param pVCpu Pointer to the VMCPU (can be NULL if @a rc is not
446	* VERR_VMX_UNABLE_TO_START_VM or
447	* VERR_VMX_INVALID_VMCS_FIELD).
448	* @param rc The error code.
449	*/
450	static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
451	{
452	AssertPtr(pVM);
453	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
454	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
455	{
456	AssertPtrReturnVoid(pVCpu);
457	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.lasterror.u32InstrError);
458	}
459	pVM->hm.s.lLastError = rc;
460	}
461
462
463	/**
464	* Reads the VM-entry interruption-information field from the VMCS into the VMX
465	* transient structure.
466	*
467	* @returns VBox status code.
468	* @param pVmxTransient Pointer to the VMX transient structure.
469	*
470	* @remarks No-long-jump zone!!!
471	*/
472	DECLINLINE(int) hmR0VmxReadEntryIntrInfoVmcs(PVMXTRANSIENT pVmxTransient)
473	{
474	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntrInfo);
475	AssertRCReturn(rc, rc);
476	return VINF_SUCCESS;
477	}
478
479
480	/**
481	* Reads the VM-entry exception error code field from the VMCS into
482	* the VMX transient structure.
483	*
484	* @returns VBox status code.
485	* @param pVmxTransient Pointer to the VMX transient structure.
486	*
487	* @remarks No-long-jump zone!!!
488	*/
489	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
490	{
491	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
492	AssertRCReturn(rc, rc);
493	return VINF_SUCCESS;
494	}
495
496
497	/**
498	* Reads the VM-entry exception error code field from the VMCS into
499	* the VMX transient structure.
500	*
501	* @returns VBox status code.
502	* @param pVCpu Pointer to the VMCPU.
503	* @param pVmxTransient Pointer to the VMX transient structure.
504	*
505	* @remarks No-long-jump zone!!!
506	*/
507	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
508	{
509	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
510	AssertRCReturn(rc, rc);
511	return VINF_SUCCESS;
512	}
513
514
515	/**
516	* Reads the VM-exit interruption-information field from the VMCS into the VMX
517	* transient structure.
518	*
519	* @returns VBox status code.
520	* @param pVCpu Pointer to the VMCPU.
521	* @param pVmxTransient Pointer to the VMX transient structure.
522	*/
523	DECLINLINE(int) hmR0VmxReadExitIntrInfoVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
524	{
525	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
526	{
527	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntrInfo);
528	AssertRCReturn(rc, rc);
529	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
530	}
531	return VINF_SUCCESS;
532	}
533
534
535	/**
536	* Reads the VM-exit interruption error code from the VMCS into the VMX
537	* transient structure.
538	*
539	* @returns VBox status code.
540	* @param pVCpu Pointer to the VMCPU.
541	* @param pVmxTransient Pointer to the VMX transient structure.
542	*/
543	DECLINLINE(int) hmR0VmxReadExitIntrErrorCodeVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
544	{
545	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
546	{
547	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntrErrorCode);
548	AssertRCReturn(rc, rc);
549	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
550	}
551	return VINF_SUCCESS;
552	}
553
554
555	/**
556	* Reads the VM-exit instruction length field from the VMCS into the VMX
557	* transient structure.
558	*
559	* @returns VBox status code.
560	* @param pVCpu Pointer to the VMCPU.
561	* @param pVmxTransient Pointer to the VMX transient structure.
562	*/
563	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
564	{
565	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
566	{
567	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
568	AssertRCReturn(rc, rc);
569	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
570	}
571	return VINF_SUCCESS;
572	}
573
574
575	/**
576	* Reads the exit qualification from the VMCS into the VMX transient structure.
577	*
578	* @returns VBox status code.
579	* @param pVCpu Pointer to the VMCPU.
580	* @param pVmxTransient Pointer to the VMX transient structure.
581	*/
582	DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
583	{
584	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
585	{
586	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification);
587	AssertRCReturn(rc, rc);
588	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
589	}
590	return VINF_SUCCESS;
591	}
592
593
594	/**
595	* Reads the IDT-vectoring information field from the VMCS into the VMX
596	* transient structure.
597	*
598	* @returns VBox status code.
599	* @param pVmxTransient Pointer to the VMX transient structure.
600	*
601	* @remarks No-long-jump zone!!!
602	*/
603	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
604	{
605	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
606	{
607	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
608	AssertRCReturn(rc, rc);
609	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
610	}
611	return VINF_SUCCESS;
612	}
613
614
615	/**
616	* Reads the IDT-vectoring error code from the VMCS into the VMX
617	* transient structure.
618	*
619	* @returns VBox status code.
620	* @param pVmxTransient Pointer to the VMX transient structure.
621	*/
622	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
623	{
624	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
625	{
626	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
627	AssertRCReturn(rc, rc);
628	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
629	}
630	return VINF_SUCCESS;
631	}
632
633
634	/**
635	* Enters VMX root mode operation on the current CPU.
636	*
637	* @returns VBox status code.
638	* @param pVM Pointer to the VM (optional, can be NULL, after
639	* a resume).
640	* @param HCPhysCpuPage Physical address of the VMXON region.
641	* @param pvCpuPage Pointer to the VMXON region.
642	*/
643	static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
644	{
645	AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
646	AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
647	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
648
649	if (pVM)
650	{
651	/* Write the VMCS revision dword to the VMXON region. */
652	(uint32_t )pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.msr.vmx_basic_info);
653	}
654
655	/* Enable the VMX bit in CR4 if necessary. */
656	RTCCUINTREG uCr4 = ASMGetCR4();
657	if (!(uCr4 & X86_CR4_VMXE))
658	ASMSetCR4(uCr4 \| X86_CR4_VMXE);
659
660	/* Enter VMX root mode. */
661	int rc = VMXEnable(HCPhysCpuPage);
662	if (RT_FAILURE(rc))
663	ASMSetCR4(uCr4);
664
665	return rc;
666	}
667
668
669	/**
670	* Exits VMX root mode operation on the current CPU.
671	*
672	* @returns VBox status code.
673	*/
674	static int hmR0VmxLeaveRootMode(void)
675	{
676	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
677
678	/* If we're for some reason not in VMX root mode, then don't leave it. */
679	if (ASMGetCR4() & X86_CR4_VMXE)
680	{
681	/* Exit VMX root mode and clear the VMX bit in CR4 */
682	VMXDisable();
683	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
684	return VINF_SUCCESS;
685	}
686
687	return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
688	}
689
690
691	/**
692	* Allocates and maps one physically contiguous page. The allocated page is
693	* zero'd out. (Used by various VT-x structures).
694	*
695	* @returns IPRT status code.
696	* @param pMemObj Pointer to the ring-0 memory object.
697	* @param ppVirt Where to store the virtual address of the
698	* allocation.
699	* @param pPhys Where to store the physical address of the
700	* allocation.
701	*/
702	DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
703	{
704	AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
705	AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
706	AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
707
708	int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
709	if (RT_FAILURE(rc))
710	return rc;
711	ppVirt = RTR0MemObjAddress(pMemObj);
712	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
713	ASMMemZero32(*ppVirt, PAGE_SIZE);
714	return VINF_SUCCESS;
715	}
716
717
718	/**
719	* Frees and unmaps an allocated physical page.
720	*
721	* @param pMemObj Pointer to the ring-0 memory object.
722	* @param ppVirt Where to re-initialize the virtual address of
723	* allocation as 0.
724	* @param pHCPhys Where to re-initialize the physical address of the
725	* allocation as 0.
726	*/
727	DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
728	{
729	AssertPtr(pMemObj);
730	AssertPtr(ppVirt);
731	AssertPtr(pHCPhys);
732	if (*pMemObj != NIL_RTR0MEMOBJ)
733	{
734	int rc = RTR0MemObjFree(pMemObj, true / fFreeMappings */);
735	AssertRC(rc);
736	*pMemObj = NIL_RTR0MEMOBJ;
737	*ppVirt = 0;
738	*pHCPhys = 0;
739	}
740	}
741
742
743	/**
744	* Worker function to free VT-x related structures.
745	*
746	* @returns IPRT status code.
747	* @param pVM Pointer to the VM.
748	*/
749	static void hmR0VmxStructsFree(PVM pVM)
750	{
751	for (VMCPUID i = 0; i < pVM->cCpus; i++)
752	{
753	PVMCPU pVCpu = &pVM->aCpus[i];
754	AssertPtr(pVCpu);
755
756	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
757	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
758	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
759	#endif
760
761	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
762	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
763
764	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
765	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
766	}
767
768	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
769	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
770	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
771	#endif
772	}
773
774
775	/**
776	* Worker function to allocate VT-x related VM structures.
777	*
778	* @returns IPRT status code.
779	* @param pVM Pointer to the VM.
780	*/
781	static int hmR0VmxStructsAlloc(PVM pVM)
782	{
783	/*
784	* Initialize members up-front so we can cleanup properly on allocation failure.
785	*/
786	#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
787	pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
788	pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
789	pVM->hm.s.vmx.HCPhys##a_Name = 0;
790
791	#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
792	pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
793	pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
794	pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
795
796	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
797	VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
798	#endif
799	VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
800
801	AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
802	for (VMCPUID i = 0; i < pVM->cCpus; i++)
803	{
804	PVMCPU pVCpu = &pVM->aCpus[i];
805	VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
806	VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
807	VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
808	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
809	VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
810	VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
811	#endif
812	}
813	#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
814	#undef VMXLOCAL_INIT_VM_MEMOBJ
815
816	/*
817	* Allocate all the VT-x structures.
818	*/
819	int rc = VINF_SUCCESS;
820	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
821	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
822	if (RT_FAILURE(rc))
823	goto cleanup;
824	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
825	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xDEADBEEFDEADBEEF);
826	#endif
827
828	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
829	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
830	{
831	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
832	&pVM->hm.s.vmx.HCPhysApicAccess);
833	if (RT_FAILURE(rc))
834	goto cleanup;
835	}
836
837	/*
838	* Initialize per-VCPU VT-x structures.
839	*/
840	for (VMCPUID i = 0; i < pVM->cCpus; i++)
841	{
842	PVMCPU pVCpu = &pVM->aCpus[i];
843	AssertPtr(pVCpu);
844
845	/* Allocate the VM control structure (VMCS). */
846	AssertReturn(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.msr.vmx_basic_info) <= PAGE_SIZE, VERR_INTERNAL_ERROR);
847	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
848	if (RT_FAILURE(rc))
849	goto cleanup;
850
851	/* Allocate the Virtual-APIC page for transparent TPR accesses. */
852	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
853	{
854	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
855	&pVCpu->hm.s.vmx.HCPhysVirtApic);
856	if (RT_FAILURE(rc))
857	goto cleanup;
858	}
859
860	/* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for transparent accesses of specific MSRs. */
861	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
862	{
863	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
864	&pVCpu->hm.s.vmx.HCPhysMsrBitmap);
865	if (RT_FAILURE(rc))
866	goto cleanup;
867	memset(pVCpu->hm.s.vmx.pvMsrBitmap, 0xff, PAGE_SIZE);
868	}
869
870	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
871	/* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
872	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
873	if (RT_FAILURE(rc))
874	goto cleanup;
875
876	/* Allocate the VM-exit MSR-load page for the host MSRs. */
877	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
878	if (RT_FAILURE(rc))
879	goto cleanup;
880	#endif
881	}
882
883	return VINF_SUCCESS;
884
885	cleanup:
886	hmR0VmxStructsFree(pVM);
887	return rc;
888	}
889
890
891	/**
892	* Does global VT-x initialization (called during module initialization).
893	*
894	* @returns VBox status code.
895	*/
896	VMMR0DECL(int) VMXR0GlobalInit(void)
897	{
898	#ifdef HMVMX_USE_FUNCTION_TABLE
899	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
900	# ifdef VBOX_STRICT
901	for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
902	Assert(g_apfnVMExitHandlers[i]);
903	# endif
904	#endif
905	return VINF_SUCCESS;
906	}
907
908
909	/**
910	* Does global VT-x termination (called during module termination).
911	*/
912	VMMR0DECL(void) VMXR0GlobalTerm()
913	{
914	/* Nothing to do currently. */
915	}
916
917
918	/**
919	* Sets up and activates VT-x on the current CPU.
920	*
921	* @returns VBox status code.
922	* @param pCpu Pointer to the global CPU info struct.
923	* @param pVM Pointer to the VM (can be NULL after a host resume
924	* operation).
925	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
926	* fEnabledByHost is true).
927	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
928	* @a fEnabledByHost is true).
929	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
930	* enable VT-x/AMD-V on the host.
931	*/
932	VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBLCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost)
933	{
934	AssertReturn(pCpu, VERR_INVALID_PARAMETER);
935	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
936
937	if (!fEnabledByHost)
938	{
939	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
940	if (RT_FAILURE(rc))
941	return rc;
942	}
943
944	/*
945	* Flush all VPIDs (in case we or any other hypervisor have been using VPIDs) so that
946	* we can avoid an explicit flush while using new VPIDs. We would still need to flush
947	* each time while reusing a VPID after hitting the MaxASID limit once.
948	*/
949	if ( pVM
950	&& pVM->hm.s.vmx.fVpid
951	&& (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS))
952	{
953	hmR0VmxFlushVpid(pVM, NULL /* pvCpu /, VMX_FLUSH_VPID_ALL_CONTEXTS, 0 / GCPtr */);
954	pCpu->fFlushAsidBeforeUse = false;
955	}
956	else
957	pCpu->fFlushAsidBeforeUse = true;
958
959	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
960	++pCpu->cTlbFlushes;
961
962	return VINF_SUCCESS;
963	}
964
965
966	/**
967	* Deactivates VT-x on the current CPU.
968	*
969	* @returns VBox status code.
970	* @param pCpu Pointer to the global CPU info struct.
971	* @param pvCpuPage Pointer to the VMXON region.
972	* @param HCPhysCpuPage Physical address of the VMXON region.
973	*/
974	VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBLCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
975	{
976	NOREF(pCpu);
977	NOREF(pvCpuPage);
978	NOREF(HCPhysCpuPage);
979
980	return hmR0VmxLeaveRootMode();
981	}
982
983
984	/**
985	* Sets the permission bits for the specified MSR in the MSR bitmap.
986	*
987	* @param pVCpu Pointer to the VMCPU.
988	* @param uMSR The MSR value.
989	* @param enmRead Whether reading this MSR causes a VM-exit.
990	* @param enmWrite Whether writing this MSR causes a VM-exit.
991	*/
992	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
993	{
994	int32_t iBit;
995	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
996
997	/*
998	* Layout:
999	* 0x000 - 0x3ff - Low MSR read bits
1000	* 0x400 - 0x7ff - High MSR read bits
1001	* 0x800 - 0xbff - Low MSR write bits
1002	* 0xc00 - 0xfff - High MSR write bits
1003	*/
1004	if (uMsr <= 0x00001FFF)
1005	iBit = uMsr;
1006	else if ( uMsr >= 0xC0000000
1007	&& uMsr <= 0xC0001FFF)
1008	{
1009	iBit = (uMsr - 0xC0000000);
1010	pbMsrBitmap += 0x400;
1011	}
1012	else
1013	{
1014	AssertMsgFailed(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1015	return;
1016	}
1017
1018	Assert(iBit <= 0x1fff);
1019	if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1020	ASMBitSet(pbMsrBitmap, iBit);
1021	else
1022	ASMBitClear(pbMsrBitmap, iBit);
1023
1024	if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1025	ASMBitSet(pbMsrBitmap + 0x800, iBit);
1026	else
1027	ASMBitClear(pbMsrBitmap + 0x800, iBit);
1028	}
1029
1030
1031	/**
1032	* Flushes the TLB using EPT.
1033	*
1034	* @returns VBox status code.
1035	* @param pVM Pointer to the VM.
1036	* @param pVCpu Pointer to the VMCPU.
1037	* @param enmFlush Type of flush.
1038	*/
1039	static void hmR0VmxFlushEpt(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush)
1040	{
1041	AssertPtr(pVM);
1042	Assert(pVM->hm.s.fNestedPaging);
1043
1044	LogFlowFunc(("pVM=%p pVCpu=%p enmFlush=%d\n", pVM, pVCpu, enmFlush));
1045
1046	uint64_t descriptor[2];
1047	descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1048	descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1049
1050	int rc = VMXR0InvEPT(enmFlush, &descriptor[0]);
1051	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu->hm.s.vmx.HCPhysEPTP, rc));
1052	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1053	}
1054
1055
1056	/**
1057	* Flushes the TLB using VPID.
1058	*
1059	* @returns VBox status code.
1060	* @param pVM Pointer to the VM.
1061	* @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1062	* enmFlush).
1063	* @param enmFlush Type of flush.
1064	* @param GCPtr Virtual address of the page to flush (can be 0 depending
1065	* on @a enmFlush).
1066	*/
1067	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr)
1068	{
1069	AssertPtr(pVM);
1070	Assert(pVM->hm.s.vmx.fVpid);
1071
1072	uint64_t descriptor[2];
1073	if (enmFlush == VMX_FLUSH_VPID_ALL_CONTEXTS)
1074	{
1075	descriptor[0] = 0;
1076	descriptor[1] = 0;
1077	}
1078	else
1079	{
1080	AssertPtr(pVCpu);
1081	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1082	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1083	descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1084	descriptor[1] = GCPtr;
1085	}
1086
1087	int rc = VMXR0InvVPID(enmFlush, &descriptor[0]); NOREF(rc);
1088	AssertMsg(rc == VINF_SUCCESS,
1089	("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1090	if ( RT_SUCCESS(rc)
1091	&& pVCpu)
1092	{
1093	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1094	}
1095	}
1096
1097
1098	/**
1099	* Invalidates a guest page by guest virtual address. Only relevant for
1100	* EPT/VPID, otherwise there is nothing really to invalidate.
1101	*
1102	* @returns VBox status code.
1103	* @param pVM Pointer to the VM.
1104	* @param pVCpu Pointer to the VMCPU.
1105	* @param GCVirt Guest virtual address of the page to invalidate.
1106	*/
1107	VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1108	{
1109	AssertPtr(pVM);
1110	AssertPtr(pVCpu);
1111	LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1112
1113	bool fFlushPending = VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1114	if (!fFlushPending)
1115	{
1116	/*
1117	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1118	* See @bugref{6043} and @bugref{6177}.
1119	*
1120	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1121	* function maybe called in a loop with individual addresses.
1122	*/
1123	if (pVM->hm.s.vmx.fVpid)
1124	{
1125	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1126	{
1127	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, GCVirt);
1128	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1129	}
1130	else
1131	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1132	}
1133	else if (pVM->hm.s.fNestedPaging)
1134	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1135	}
1136
1137	return VINF_SUCCESS;
1138	}
1139
1140
1141	/**
1142	* Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1143	* otherwise there is nothing really to invalidate.
1144	*
1145	* @returns VBox status code.
1146	* @param pVM Pointer to the VM.
1147	* @param pVCpu Pointer to the VMCPU.
1148	* @param GCPhys Guest physical address of the page to invalidate.
1149	*/
1150	VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1151	{
1152	LogFlowFunc(("%RGp\n", GCPhys));
1153
1154	/*
1155	* We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1156	* by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1157	* This function might be called in a loop.
1158	*/
1159	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1160	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1161	return VINF_SUCCESS;
1162	}
1163
1164
1165	/**
1166	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1167	* case where neither EPT nor VPID is supported by the CPU.
1168	*
1169	* @param pVM Pointer to the VM.
1170	* @param pVCpu Pointer to the VMCPU.
1171	*
1172	* @remarks Called with interrupts disabled.
1173	*/
1174	static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu)
1175	{
1176	NOREF(pVM);
1177	AssertPtr(pVCpu);
1178	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1179	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1180
1181	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1182	AssertPtr(pCpu);
1183
1184	pVCpu->hm.s.TlbShootdown.cPages = 0;
1185	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1186	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1187	pVCpu->hm.s.fForceTLBFlush = false;
1188	return;
1189	}
1190
1191
1192	/**
1193	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1194	*
1195	* @param pVM Pointer to the VM.
1196	* @param pVCpu Pointer to the VMCPU.
1197	* @remarks All references to "ASID" in this function pertains to "VPID" in
1198	* Intel's nomenclature. The reason is, to avoid confusion in compare
1199	* statements since the host-CPU copies are named "ASID".
1200	*
1201	* @remarks Called with interrupts disabled.
1202	*/
1203	static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu)
1204	{
1205	AssertPtr(pVM);
1206	AssertPtr(pVCpu);
1207	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1208	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1209	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1210
1211	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1212	AssertPtr(pCpu);
1213
1214	/*
1215	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1216	* This can happen both for start & resume due to long jumps back to ring-3.
1217	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1218	* or the host Cpu is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1219	*/
1220	bool fNewASID = false;
1221	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1222	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1223	{
1224	pVCpu->hm.s.fForceTLBFlush = true;
1225	fNewASID = true;
1226	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1227	}
1228
1229	/*
1230	* Check for explicit TLB shootdowns.
1231	*/
1232	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1233	{
1234	pVCpu->hm.s.fForceTLBFlush = true;
1235	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1236	}
1237
1238	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1239	if (pVCpu->hm.s.fForceTLBFlush)
1240	{
1241	if (fNewASID)
1242	{
1243	++pCpu->uCurrentAsid;
1244	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1245	{
1246	pCpu->uCurrentAsid = 1; /* start at 1; host uses 0 */
1247	pCpu->cTlbFlushes++;
1248	pCpu->fFlushAsidBeforeUse = true;
1249	}
1250
1251	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1252	if (pCpu->fFlushAsidBeforeUse)
1253	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1254	}
1255	else
1256	{
1257	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
1258	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_SINGLE_CONTEXT, 0 /* GCPtr */);
1259	else
1260	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1261	}
1262
1263	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1264	pVCpu->hm.s.fForceTLBFlush = false;
1265	}
1266	else
1267	{
1268	AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
1269	("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
1270	pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
1271	pCpu->uCurrentAsid, pCpu->cTlbFlushes));
1272
1273	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1274	* not be executed. See hmQueueInvlPage() where it is commented
1275	* out. Support individual entry flushing someday. */
1276	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1277	{
1278	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1279
1280	/*
1281	* Flush individual guest entries using VPID from the TLB or as little as possible with EPT
1282	* as supported by the CPU.
1283	*/
1284	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1285	{
1286	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1287	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1288	}
1289	else
1290	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1291	}
1292	else
1293	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1294	}
1295	pVCpu->hm.s.TlbShootdown.cPages = 0;
1296	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1297
1298	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1299	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1300	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1301	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1302	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1303	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1304
1305	/* Update VMCS with the VPID. */
1306	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1307	AssertRC(rc);
1308	}
1309
1310
1311	/**
1312	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
1313	*
1314	* @returns VBox status code.
1315	* @param pVM Pointer to the VM.
1316	* @param pVCpu Pointer to the VMCPU.
1317	*
1318	* @remarks Called with interrupts disabled.
1319	*/
1320	static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu)
1321	{
1322	AssertPtr(pVM);
1323	AssertPtr(pVCpu);
1324	AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
1325	AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
1326
1327	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1328	AssertPtr(pCpu);
1329
1330	/*
1331	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1332	* This can happen both for start & resume due to long jumps back to ring-3.
1333	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
1334	*/
1335	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1336	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1337	{
1338	pVCpu->hm.s.fForceTLBFlush = true;
1339	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1340	}
1341
1342	/* Check for explicit TLB shootdown flushes. */
1343	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1344	{
1345	pVCpu->hm.s.fForceTLBFlush = true;
1346	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1347	}
1348
1349	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1350	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1351
1352	if (pVCpu->hm.s.fForceTLBFlush)
1353	{
1354	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1355	pVCpu->hm.s.fForceTLBFlush = false;
1356	}
1357	else
1358	{
1359	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1360	* not be executed. See hmQueueInvlPage() where it is commented
1361	* out. Support individual entry flushing someday. */
1362	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1363	{
1364	/* We cannot flush individual entries without VPID support. Flush using EPT. */
1365	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1366	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1367	}
1368	else
1369	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1370	}
1371
1372	pVCpu->hm.s.TlbShootdown.cPages = 0;
1373	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1374	}
1375
1376
1377	/**
1378	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
1379	*
1380	* @returns VBox status code.
1381	* @param pVM Pointer to the VM.
1382	* @param pVCpu Pointer to the VMCPU.
1383	*
1384	* @remarks Called with interrupts disabled.
1385	*/
1386	static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu)
1387	{
1388	AssertPtr(pVM);
1389	AssertPtr(pVCpu);
1390	AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
1391	AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
1392
1393	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1394
1395	/*
1396	* Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
1397	* This can happen both for start & resume due to long jumps back to ring-3.
1398	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1399	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1400	*/
1401	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1402	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1403	{
1404	pVCpu->hm.s.fForceTLBFlush = true;
1405	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1406	}
1407
1408	/* Check for explicit TLB shootdown flushes. */
1409	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1410	{
1411	/*
1412	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
1413	* we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
1414	* pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
1415	*/
1416	pVCpu->hm.s.fForceTLBFlush = true;
1417	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1418	}
1419
1420	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1421	if (pVCpu->hm.s.fForceTLBFlush)
1422	{
1423	++pCpu->uCurrentAsid;
1424	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1425	{
1426	pCpu->uCurrentAsid = 1; /* start at 1; host uses 0 */
1427	pCpu->fFlushAsidBeforeUse = true;
1428	pCpu->cTlbFlushes++;
1429	}
1430
1431	pVCpu->hm.s.fForceTLBFlush = false;
1432	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1433	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1434	if (pCpu->fFlushAsidBeforeUse)
1435	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1436	}
1437	else
1438	{
1439	AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
1440	("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
1441	pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
1442	pCpu->uCurrentAsid, pCpu->cTlbFlushes));
1443
1444	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1445	* not be executed. See hmQueueInvlPage() where it is commented
1446	* out. Support individual entry flushing someday. */
1447	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1448	{
1449	/* Flush individual guest entries using VPID or as little as possible with EPT as supported by the CPU. */
1450	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1451	{
1452	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1453	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1454	}
1455	else
1456	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1457	}
1458	else
1459	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1460	}
1461
1462	pVCpu->hm.s.TlbShootdown.cPages = 0;
1463	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1464
1465	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1466	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1467	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1468	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1469	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1470	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1471
1472	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1473	AssertRC(rc);
1474	}
1475
1476
1477	/**
1478	* Flushes the guest TLB entry based on CPU capabilities.
1479	*
1480	* @param pVCpu Pointer to the VMCPU.
1481	*/
1482	DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu)
1483	{
1484	PVM pVM = pVCpu->CTX_SUFF(pVM);
1485	switch (pVM->hm.s.vmx.uFlushTaggedTlb)
1486	{
1487	case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu); break;
1488	case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu); break;
1489	case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu); break;
1490	case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu); break;
1491	default:
1492	AssertMsgFailed(("Invalid flush-tag function identifier\n"));
1493	break;
1494	}
1495	}
1496
1497
1498	/**
1499	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
1500	* TLB entries from the host TLB before VM-entry.
1501	*
1502	* @returns VBox status code.
1503	* @param pVM Pointer to the VM.
1504	*/
1505	static int hmR0VmxSetupTaggedTlb(PVM pVM)
1506	{
1507	/*
1508	* Determine optimal flush type for nested paging.
1509	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
1510	* guest execution (see hmR3InitFinalizeR0()).
1511	*/
1512	if (pVM->hm.s.fNestedPaging)
1513	{
1514	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
1515	{
1516	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
1517	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_SINGLE_CONTEXT;
1518	else if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1519	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_ALL_CONTEXTS;
1520	else
1521	{
1522	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
1523	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1524	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1525	}
1526
1527	/* Make sure the write-back cacheable memory type for EPT is supported. */
1528	if (!(pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB))
1529	{
1530	LogRel(("hmR0VmxSetupTaggedTlb: Unsupported EPTP memory type %#x.\n", pVM->hm.s.vmx.msr.vmx_ept_vpid_caps));
1531	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1532	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1533	}
1534	}
1535	else
1536	{
1537	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
1538	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1539	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1540	}
1541	}
1542
1543	/*
1544	* Determine optimal flush type for VPID.
1545	*/
1546	if (pVM->hm.s.vmx.fVpid)
1547	{
1548	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
1549	{
1550	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
1551	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_SINGLE_CONTEXT;
1552	else if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
1553	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_ALL_CONTEXTS;
1554	else
1555	{
1556	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
1557	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1558	LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
1559	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
1560	LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
1561	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1562	pVM->hm.s.vmx.fVpid = false;
1563	}
1564	}
1565	else
1566	{
1567	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
1568	Log(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
1569	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1570	pVM->hm.s.vmx.fVpid = false;
1571	}
1572	}
1573
1574	/*
1575	* Setup the handler for flushing tagged-TLBs.
1576	*/
1577	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
1578	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
1579	else if (pVM->hm.s.fNestedPaging)
1580	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
1581	else if (pVM->hm.s.vmx.fVpid)
1582	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
1583	else
1584	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
1585	return VINF_SUCCESS;
1586	}
1587
1588
1589	/**
1590	* Sets up pin-based VM-execution controls in the VMCS.
1591	*
1592	* @returns VBox status code.
1593	* @param pVM Pointer to the VM.
1594	* @param pVCpu Pointer to the VMCPU.
1595	*/
1596	static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
1597	{
1598	AssertPtr(pVM);
1599	AssertPtr(pVCpu);
1600
1601	uint32_t val = pVM->hm.s.vmx.msr.vmx_pin_ctls.n.disallowed0; /* Bits set here must always be set. */
1602	uint32_t zap = pVM->hm.s.vmx.msr.vmx_pin_ctls.n.allowed1; /* Bits cleared here must always be cleared. */
1603
1604	val \|= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts causes a VM-exits. */
1605	\| VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts causes a VM-exit. */
1606	Assert(!(val & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI));
1607
1608	/* Enable the VMX preemption timer. */
1609	if (pVM->hm.s.vmx.fUsePreemptTimer)
1610	{
1611	Assert(pVM->hm.s.vmx.msr.vmx_pin_ctls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
1612	val \|= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
1613	}
1614
1615	if ((val & zap) != val)
1616	{
1617	LogRel(("hmR0VmxSetupPinCtls: invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1618	pVM->hm.s.vmx.msr.vmx_pin_ctls.n.disallowed0, val, zap));
1619	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1620	}
1621
1622	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
1623	AssertRCReturn(rc, rc);
1624
1625	/* Update VCPU with the currently set pin-based VM-execution controls. */
1626	pVCpu->hm.s.vmx.u32PinCtls = val;
1627	return rc;
1628	}
1629
1630
1631	/**
1632	* Sets up processor-based VM-execution controls in the VMCS.
1633	*
1634	* @returns VBox status code.
1635	* @param pVM Pointer to the VM.
1636	* @param pVMCPU Pointer to the VMCPU.
1637	*/
1638	static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
1639	{
1640	AssertPtr(pVM);
1641	AssertPtr(pVCpu);
1642
1643	int rc = VERR_INTERNAL_ERROR_5;
1644	uint32_t val = pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0; /* Bits set here must be set in the VMCS. */
1645	uint32_t zap = pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1646
1647	val \|= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
1648	\| VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
1649	\| VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
1650	\| VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
1651	\| VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
1652	\| VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
1653	\| VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
1654
1655	/* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
1656	if ( !(pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
1657	\|\| (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
1658	{
1659	LogRel(("hmR0VmxSetupProcCtls: unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
1660	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1661	}
1662
1663	/* Without nested paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
1664	if (!pVM->hm.s.fNestedPaging)
1665	{
1666	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
1667	val \|= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
1668	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
1669	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
1670	}
1671
1672	/* Use TPR shadowing if supported by the CPU. */
1673	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
1674	{
1675	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
1676	Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
1677	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
1678	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
1679	AssertRCReturn(rc, rc);
1680
1681	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
1682	/* CR8 writes causes a VM-exit based on TPR threshold. */
1683	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
1684	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
1685	}
1686	else
1687	{
1688	val \|= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads causes a VM-exit. */
1689	\| VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes causes a VM-exit. */
1690	}
1691
1692	/* Use MSR-bitmaps if supported by the CPU. */
1693	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1694	{
1695	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
1696
1697	Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1698	Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
1699	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1700	AssertRCReturn(rc, rc);
1701
1702	/*
1703	* The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
1704	* automatically (either as part of the MSR-load/store areas or dedicated fields in the VMCS).
1705	*/
1706	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1707	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1708	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1709	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1710	hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1711	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1712	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1713	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1714	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1715	}
1716
1717	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
1718	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1719	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
1720
1721	if ((val & zap) != val)
1722	{
1723	LogRel(("hmR0VmxSetupProcCtls: invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1724	pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0, val, zap));
1725	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1726	}
1727
1728	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
1729	AssertRCReturn(rc, rc);
1730
1731	/* Update VCPU with the currently set processor-based VM-execution controls. */
1732	pVCpu->hm.s.vmx.u32ProcCtls = val;
1733
1734	/*
1735	* Secondary processor-based VM-execution controls.
1736	*/
1737	if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
1738	{
1739	val = pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
1740	zap = pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1741
1742	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
1743	val \|= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
1744
1745	if (pVM->hm.s.fNestedPaging)
1746	val \|= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
1747	else
1748	{
1749	/*
1750	* Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
1751	* VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT when INVPCID is executed by the guest.
1752	* See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
1753	*/
1754	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
1755	val \|= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
1756	}
1757
1758	if (pVM->hm.s.vmx.fVpid)
1759	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
1760
1761	if (pVM->hm.s.vmx.fUnrestrictedGuest)
1762	val \|= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
1763
1764	/* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
1765	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
1766	* done dynamically. */
1767	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
1768	{
1769	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
1770	Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
1771	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
1772	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
1773	AssertRCReturn(rc, rc);
1774	}
1775
1776	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
1777	{
1778	val \|= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
1779	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1780	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_TSC_AUX, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1781	}
1782
1783	if ((val & zap) != val)
1784	{
1785	LogRel(("hmR0VmxSetupProcCtls: invalid secondary processor-based VM-execution controls combo! "
1786	"cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0, val, zap));
1787	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1788	}
1789
1790	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
1791	AssertRCReturn(rc, rc);
1792
1793	/* Update VCPU with the currently set secondary processor-based VM-execution controls. */
1794	pVCpu->hm.s.vmx.u32ProcCtls2 = val;
1795	}
1796
1797	return VINF_SUCCESS;
1798	}
1799
1800
1801	/**
1802	* Sets up miscellaneous (everything other than Pin & Processor-based
1803	* VM-execution) control fields in the VMCS.
1804	*
1805	* @returns VBox status code.
1806	* @param pVM Pointer to the VM.
1807	* @param pVCpu Pointer to the VMCPU.
1808	*/
1809	static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
1810	{
1811	AssertPtr(pVM);
1812	AssertPtr(pVCpu);
1813
1814	/** @todo Shouldn't we able to avoid initializing with 0? */
1815	int rc = VERR_GENERAL_FAILURE;
1816
1817	/* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestControlRegs())*/
1818	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0); AssertRCReturn(rc, rc);
1819	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0); AssertRCReturn(rc, rc);
1820
1821	/*
1822	* Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
1823	* and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
1824	* We thus use the exception bitmap to control it rather than use both.
1825	*/
1826	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0); AssertRCReturn(rc, rc);
1827	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0); AssertRCReturn(rc, rc);
1828
1829	/** @todo Explore possibility of using IO-bitmaps. */
1830	/* All IO & IOIO instructions cause VM-exits. */
1831	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0); AssertRCReturn(rc, rc);
1832	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0); AssertRCReturn(rc, rc);
1833
1834	/* Initialize the MSR-bitmap area. */
1835	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
1836	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0); AssertRCReturn(rc, rc);
1837	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
1838
1839	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
1840	/* Setup MSR autoloading/storing. */
1841	Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
1842	Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
1843	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1844	AssertRCReturn(rc, rc);
1845	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1846	AssertRCReturn(rc, rc);
1847
1848	Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
1849	Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
1850	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
1851	AssertRCReturn(rc, rc);
1852	#endif
1853
1854	/* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
1855	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
1856	AssertRCReturn(rc, rc);
1857
1858	/* Setup debug controls */
1859	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
1860	AssertRCReturn(rc, rc);
1861	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
1862	AssertRCReturn(rc, rc);
1863	return rc;
1864	}
1865
1866
1867	/**
1868	* Sets up the initial exception bitmap in the VMCS based on static conditions
1869	* (i.e. conditions that cannot ever change at runtime).
1870	*
1871	* @returns VBox status code.
1872	* @param pVM Pointer to the VM.
1873	* @param pVCpu Pointer to the VMCPU.
1874	*/
1875	static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
1876	{
1877	AssertPtr(pVM);
1878	AssertPtr(pVCpu);
1879
1880	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
1881
1882	uint32_t u32XcptBitmap = 0;
1883
1884	/* Without nested paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
1885	if (!pVM->hm.s.fNestedPaging)
1886	u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
1887
1888	pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
1889	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
1890	AssertRCReturn(rc, rc);
1891	return rc;
1892	}
1893
1894
1895	/**
1896	* Sets up the initial guest-state mask. The guest-state mask is consulted
1897	* before reading guest-state fields from the VMCS as VMREADs can be expensive
1898	* for the nested virtualization case (as it would cause a VM-exit).
1899	*
1900	* @param pVCpu Pointer to the VMCPU.
1901	*/
1902	static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
1903	{
1904	/* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
1905	pVCpu->hm.s.vmx.fUpdatedGuestState = HMVMX_UPDATED_GUEST_ALL;
1906	return VINF_SUCCESS;
1907	}
1908
1909
1910	/**
1911	* Does per-VM VT-x initialization.
1912	*
1913	* @returns VBox status code.
1914	* @param pVM Pointer to the VM.
1915	*/
1916	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
1917	{
1918	LogFlowFunc(("pVM=%p\n", pVM));
1919
1920	int rc = hmR0VmxStructsAlloc(pVM);
1921	if (RT_FAILURE(rc))
1922	{
1923	LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
1924	return rc;
1925	}
1926
1927	return VINF_SUCCESS;
1928	}
1929
1930
1931	/**
1932	* Does per-VM VT-x termination.
1933	*
1934	* @returns VBox status code.
1935	* @param pVM Pointer to the VM.
1936	*/
1937	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
1938	{
1939	LogFlowFunc(("pVM=%p\n", pVM));
1940
1941	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1942	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
1943	ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
1944	#endif
1945	hmR0VmxStructsFree(pVM);
1946	return VINF_SUCCESS;
1947	}
1948
1949
1950	/**
1951	* Sets up the VM for execution under VT-x.
1952	* This function is only called once per-VM during initalization.
1953	*
1954	* @returns VBox status code.
1955	* @param pVM Pointer to the VM.
1956	*/
1957	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
1958	{
1959	AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
1960	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1961
1962	LogFlowFunc(("pVM=%p\n", pVM));
1963
1964	/*
1965	* Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable must always be allocated.
1966	* We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0().
1967	*/
1968	/* -XXX- change hmR3InitFinalizeR0() to fail if pRealModeTSS alloc fails. */
1969	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
1970	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
1971	\|\| !pVM->hm.s.vmx.pRealModeTSS))
1972	{
1973	LogRel(("VMXR0SetupVM: invalid real-on-v86 state.\n"));
1974	return VERR_INTERNAL_ERROR;
1975	}
1976
1977	/* Initialize these always, see hmR3InitFinalizeR0().*/
1978	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NONE;
1979	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NONE;
1980
1981	/* Setup the tagged-TLB flush handlers. */
1982	int rc = hmR0VmxSetupTaggedTlb(pVM);
1983	if (RT_FAILURE(rc))
1984	{
1985	LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
1986	return rc;
1987	}
1988
1989	for (VMCPUID i = 0; i < pVM->cCpus; i++)
1990	{
1991	PVMCPU pVCpu = &pVM->aCpus[i];
1992	AssertPtr(pVCpu);
1993	AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
1994
1995	/* Set revision dword at the beginning of the VMCS structure. */
1996	(uint32_t )pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.msr.vmx_basic_info);
1997
1998	/* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
1999	rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
2000	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVMCS failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2001	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2002
2003	/* Load this VMCS as the current VMCS. */
2004	rc = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
2005	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVMCS failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2006	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2007
2008	rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2009	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2010	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2011
2012	rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2013	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2014	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2015
2016	rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2017	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2018	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2019
2020	rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2021	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2022	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2023
2024	rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2025	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2026	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2027
2028	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2029	rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2030	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2031	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2032	#endif
2033
2034	/* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2035	rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
2036	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVMCS(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2037	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2038
2039	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2040	}
2041
2042	return VINF_SUCCESS;
2043	}
2044
2045
2046	/**
2047	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2048	* the VMCS.
2049	*
2050	* @returns VBox status code.
2051	* @param pVM Pointer to the VM.
2052	* @param pVCpu Pointer to the VMCPU.
2053	*/
2054	DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2055	{
2056	RTCCUINTREG uReg = ASMGetCR0();
2057	int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2058	AssertRCReturn(rc, rc);
2059
2060	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2061	/* For the darwin 32-bit hybrid kernel, we need the 64-bit CR3 as it uses 64-bit paging. */
2062	if (HMVMX_IS_64BIT_HOST_MODE())
2063	{
2064	uint64_t uRegCR3 = HMR0Get64bitCR3();
2065	rc = VMXWriteVmcs64(VMX_VMCS_HOST_CR3, uRegCR3);
2066	}
2067	else
2068	#endif
2069	{
2070	uReg = ASMGetCR3();
2071	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2072	}
2073	AssertRCReturn(rc, rc);
2074
2075	uReg = ASMGetCR4();
2076	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2077	AssertRCReturn(rc, rc);
2078	return rc;
2079	}
2080
2081
2082	/**
2083	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2084	* the host-state area in the VMCS.
2085	*
2086	* @returns VBox status code.
2087	* @param pVM Pointer to the VM.
2088	* @param pVCpu Pointer to the VMCPU.
2089	*/
2090	DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2091	{
2092	int rc = VERR_INTERNAL_ERROR_5;
2093	RTSEL uSelCS = 0;
2094	RTSEL uSelSS = 0;
2095	RTSEL uSelDS = 0;
2096	RTSEL uSelES = 0;
2097	RTSEL uSelFS = 0;
2098	RTSEL uSelGS = 0;
2099	RTSEL uSelTR = 0;
2100
2101	/*
2102	* Host Selector registers.
2103	*/
2104	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2105	if (HMVMX_IS_64BIT_HOST_MODE())
2106	{
2107	uSelCS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
2108	uSelSS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
2109	}
2110	else
2111	{
2112	/* Seems darwin uses the LDT (TI flag is set) in the CS & SS selectors which VT-x doesn't like. */
2113	uSelCS = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
2114	uSelSS = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
2115	}
2116	#else
2117	uSelCS = ASMGetCS();
2118	uSelSS = ASMGetSS();
2119	#endif
2120
2121	/* Note: VT-x is picky about the RPL of the selectors here; we'll restore them manually. */
2122	uSelTR = ASMGetTR();
2123
2124	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2125	/** @todo Verify if we have any platform that actually run with DS or ES with
2126	* RPL != 0 in kernel space. */
2127	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2128	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2129	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2130	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2131	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2132	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2133	Assert(uSelCS != 0);
2134	Assert(uSelTR != 0);
2135
2136	/* Assertion is right but we would not have updated u32ExitCtls yet. */
2137	#if 0
2138	if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2139	Assert(uSelSS != 0);
2140	#endif
2141
2142	/* Write these host selector fields into the host-state area in the VMCS. */
2143	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_CS, uSelCS); AssertRCReturn(rc, rc);
2144	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_SS, uSelSS); AssertRCReturn(rc, rc);
2145	/* Avoid the VMWRITEs as we set the following segments to 0 and the VMCS fields are already 0 (since g_HvmR0 is static) */
2146	#if 0
2147	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_DS, uSelDS); AssertRCReturn(rc, rc);
2148	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_ES, uSelES); AssertRCReturn(rc, rc);
2149	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_FS, uSelFS); AssertRCReturn(rc, rc);
2150	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_GS, uSelGS); AssertRCReturn(rc, rc);
2151	#endif
2152	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_TR, uSelTR); AssertRCReturn(rc, rc);
2153
2154	/*
2155	* Host GDTR and IDTR.
2156	*/
2157	/** @todo Despite VT-x -not- restoring the limits on GDTR and IDTR it should
2158	* be safe to -not- save and restore GDTR and IDTR in the assembly
2159	* code and just do it here and don't care if the limits are zapped on
2160	* VM-exit. */
2161	RTGDTR Gdtr;
2162	RT_ZERO(Gdtr);
2163	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2164	if (HMVMX_IS_64BIT_HOST_MODE())
2165	{
2166	X86XDTR64 Gdtr64;
2167	X86XDTR64 Idtr64;
2168	HMR0Get64bitGdtrAndIdtr(&Gdtr64, &Idtr64);
2169	rc = VMXWriteVmcs64(VMX_VMCS_HOST_GDTR_BASE, Gdtr64.uAddr); AssertRCReturn(rc, rc);
2170	rc = VMXWriteVmcs64(VMX_VMCS_HOST_IDTR_BASE, Idtr64.uAddr); AssertRCReturn(rc, rc);
2171
2172	Gdtr.cbGdt = Gdtr64.cb;
2173	Gdtr.pGdt = (uintptr_t)Gdtr64.uAddr;
2174	}
2175	else
2176	#endif
2177	{
2178	RTIDTR Idtr;
2179	ASMGetGDTR(&Gdtr);
2180	ASMGetIDTR(&Idtr);
2181	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt); AssertRCReturn(rc, rc);
2182	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt); AssertRCReturn(rc, rc);
2183	}
2184
2185	/*
2186	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
2187	* is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
2188	*/
2189	if ((uSelTR & X86_SEL_MASK) > Gdtr.cbGdt)
2190	{
2191	AssertMsgFailed(("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit.TR=%RTsel Gdtr.cbGdt=%#x\n", uSelTR, Gdtr.cbGdt));
2192	return VERR_VMX_INVALID_HOST_STATE;
2193	}
2194
2195	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
2196	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2197	if (HMVMX_IS_64BIT_HOST_MODE())
2198	{
2199	/* We need the 64-bit TR base for hybrid darwin. */
2200	uint64_t u64TRBase = X86DESC64_BASE((PX86DESC64)pDesc);
2201	rc = VMXWriteVmcs64(VMX_VMCS_HOST_TR_BASE, u64TRBase);
2202	}
2203	else
2204	#endif
2205	{
2206	uintptr_t uTRBase;
2207	#if HC_ARCH_BITS == 64
2208	uTRBase = X86DESC64_BASE(pDesc);
2209	#else
2210	uTRBase = X86DESC_BASE(pDesc);
2211	#endif
2212	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
2213	}
2214	AssertRCReturn(rc, rc);
2215
2216	/*
2217	* Host FS base and GS base.
2218	* For 32-bit hosts the base is handled by the assembly code where we push/pop FS and GS which .
2219	* would take care of the bases. In 64-bit, the MSRs come into play.
2220	*/
2221	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2222	if (HMVMX_IS_64BIT_HOST_MODE())
2223	{
2224	uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
2225	uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
2226	rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase);
2227	AssertRCReturn(rc, rc);
2228	rc = VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase);
2229	AssertRCReturn(rc, rc);
2230	}
2231	#endif
2232	return rc;
2233	}
2234
2235
2236	/**
2237	* Saves certain host MSRs in the VM-Exit MSR-load area and some in the
2238	* host-state area of the VMCS. Theses MSRs will be automatically restored on
2239	* the host after every successful VM exit.
2240	*
2241	* @returns VBox status code.
2242	* @param pVM Pointer to the VM.
2243	* @param pVCpu Pointer to the VMCPU.
2244	*/
2245	DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
2246	{
2247	AssertPtr(pVCpu);
2248	AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
2249
2250	int rc = VINF_SUCCESS;
2251	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
2252	PVMXMSR pHostMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvHostMsr;
2253	uint32_t cHostMsrs = 0;
2254	uint32_t u32HostExtFeatures = pVM->hm.s.cpuid.u32AMDFeatureEDX;
2255
2256	if (u32HostExtFeatures & (X86_CPUID_EXT_FEATURE_EDX_NX \| X86_CPUID_EXT_FEATURE_EDX_LONG_MODE))
2257	{
2258	pHostMsr->u32IndexMSR = MSR_K6_EFER;
2259	pHostMsr->u32Reserved = 0;
2260	# if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2261	if (CPUMIsGuestInLongMode(pVCpu))
2262	{
2263	/* Must match the EFER value in our 64 bits switcher. */
2264	pHostMsr->u64Value = ASMRdMsr(MSR_K6_EFER) \| MSR_K6_EFER_LME \| MSR_K6_EFER_SCE \| MSR_K6_EFER_NXE;
2265	}
2266	else
2267	# endif
2268	pHostMsr->u64Value = ASMRdMsr(MSR_K6_EFER);
2269	pHostMsr++; cHostMsrs++;
2270	}
2271
2272	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2273	if (HMVMX_IS_64BIT_HOST_MODE())
2274	{
2275	pHostMsr->u32IndexMSR = MSR_K6_STAR;
2276	pHostMsr->u32Reserved = 0;
2277	pHostMsr->u64Value = ASMRdMsr(MSR_K6_STAR); /* legacy syscall eip, cs & ss */
2278	pHostMsr++; cHostMsrs++;
2279	pHostMsr->u32IndexMSR = MSR_K8_LSTAR;
2280	pHostMsr->u32Reserved = 0;
2281	pHostMsr->u64Value = ASMRdMsr(MSR_K8_LSTAR); /* 64-bit mode syscall rip */
2282	pHostMsr++; cHostMsrs++;
2283	pHostMsr->u32IndexMSR = MSR_K8_SF_MASK;
2284	pHostMsr->u32Reserved = 0;
2285	pHostMsr->u64Value = ASMRdMsr(MSR_K8_SF_MASK); /* syscall flag mask */
2286	pHostMsr++; cHostMsrs++;
2287	pHostMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
2288	pHostMsr->u32Reserved = 0;
2289	pHostMsr->u64Value = ASMRdMsr(MSR_K8_KERNEL_GS_BASE); /* swapgs exchange value */
2290	pHostMsr++; cHostMsrs++;
2291	}
2292	# endif
2293
2294	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
2295	if (RT_UNLIKELY(cHostMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc)))
2296	{
2297	LogRel(("cHostMsrs=%u Cpu=%u\n", cHostMsrs, (unsigned)MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc)));
2298	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2299	}
2300
2301	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cHostMsrs);
2302	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
2303
2304	/*
2305	* Host Sysenter MSRs.
2306	*/
2307	rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
2308	AssertRCReturn(rc, rc);
2309	# ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2310	if (HMVMX_IS_64BIT_HOST_MODE())
2311	{
2312	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2313	AssertRCReturn(rc, rc);
2314	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2315	}
2316	else
2317	{
2318	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2319	AssertRCReturn(rc, rc);
2320	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2321	}
2322	# elif HC_ARCH_BITS == 32
2323	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2324	AssertRCReturn(rc, rc);
2325	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2326	# else
2327	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2328	AssertRCReturn(rc, rc);
2329	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2330	# endif
2331	AssertRCReturn(rc, rc);
2332
2333	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT, IA32_EFER, also see
2334	* hmR0VmxSetupExitCtls() !! */
2335	return rc;
2336	}
2337
2338
2339	/**
2340	* Sets up VM-entry controls in the VMCS. These controls can affect things done
2341	* on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
2342	* controls".
2343	*
2344	* @returns VBox status code.
2345	* @param pVCpu Pointer to the VMCPU.
2346	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2347	* out-of-sync. Make sure to update the required fields
2348	* before using them.
2349	*
2350	* @remarks No-long-jump zone!!!
2351	*/
2352	DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2353	{
2354	int rc = VINF_SUCCESS;
2355	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_ENTRY_CTLS)
2356	{
2357	PVM pVM = pVCpu->CTX_SUFF(pVM);
2358	uint32_t val = pVM->hm.s.vmx.msr.vmx_entry.n.disallowed0; /* Bits set here must be set in the VMCS. */
2359	uint32_t zap = pVM->hm.s.vmx.msr.vmx_entry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2360
2361	/* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
2362	val \|= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
2363
2364	/* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
2365	if (CPUMIsGuestInLongModeEx(pMixedCtx))
2366	val \|= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
2367	else
2368	Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
2369
2370	/*
2371	* The following should not be set (since we're not in SMM mode):
2372	* - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
2373	* - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
2374	*/
2375
2376	/** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
2377	* VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR,
2378	* VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR */
2379
2380	if ((val & zap) != val)
2381	{
2382	LogRel(("hmR0VmxLoadGuestEntryCtls: invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2383	pVM->hm.s.vmx.msr.vmx_entry.n.disallowed0, val, zap));
2384	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2385	}
2386
2387	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
2388	AssertRCReturn(rc, rc);
2389
2390	/* Update VCPU with the currently set VM-exit controls. */
2391	pVCpu->hm.s.vmx.u32EntryCtls = val;
2392	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_ENTRY_CTLS;
2393	}
2394	return rc;
2395	}
2396
2397
2398	/**
2399	* Sets up the VM-exit controls in the VMCS.
2400	*
2401	* @returns VBox status code.
2402	* @param pVM Pointer to the VM.
2403	* @param pVCpu Pointer to the VMCPU.
2404	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2405	* out-of-sync. Make sure to update the required fields
2406	* before using them.
2407	*
2408	* @remarks requires EFER.
2409	*/
2410	DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2411	{
2412	int rc = VINF_SUCCESS;
2413	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_EXIT_CTLS)
2414	{
2415	PVM pVM = pVCpu->CTX_SUFF(pVM);
2416	uint32_t val = pVM->hm.s.vmx.msr.vmx_exit.n.disallowed0; /* Bits set here must be set in the VMCS. */
2417	uint32_t zap = pVM->hm.s.vmx.msr.vmx_exit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2418
2419	/* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
2420	val \|= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
2421
2422	/* Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary. */
2423	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2424	if (HMVMX_IS_64BIT_HOST_MODE())
2425	val \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
2426	else
2427	Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
2428	#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
2429	if (CPUMIsGuestInLongModeEx(pMixedCtx))
2430	val \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE; /* The switcher goes to long mode. */
2431	else
2432	Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
2433	#endif
2434
2435	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
2436	Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
2437
2438	/** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
2439	* VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
2440	* VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR,
2441	* VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR,
2442	* VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR. */
2443
2444	if (pVM->hm.s.vmx.msr.vmx_exit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER)
2445	val \|= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
2446
2447	if ((val & zap) != val)
2448	{
2449	LogRel(("hmR0VmxSetupProcCtls: invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2450	pVM->hm.s.vmx.msr.vmx_exit.n.disallowed0, val, zap));
2451	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2452	}
2453
2454	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
2455	AssertRCReturn(rc, rc);
2456
2457	/* Update VCPU with the currently set VM-exit controls. */
2458	pVCpu->hm.s.vmx.u32ExitCtls = val;
2459	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_EXIT_CTLS;
2460	}
2461	return rc;
2462	}
2463
2464
2465	/**
2466	* Loads the guest APIC and related state.
2467	*
2468	* @returns VBox status code.
2469	* @param pVM Pointer to the VM.
2470	* @param pVCpu Pointer to the VMCPU.
2471	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2472	* out-of-sync. Make sure to update the required fields
2473	* before using them.
2474	*/
2475	DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2476	{
2477	int rc = VINF_SUCCESS;
2478	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_APIC_STATE)
2479	{
2480	/* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
2481	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2482	{
2483	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2484
2485	bool fPendingIntr = false;
2486	uint8_t u8Tpr = 0;
2487	uint8_t u8PendingIntr = 0;
2488	rc = PDMApicGetTPR(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
2489	AssertRCReturn(rc, rc);
2490
2491	/*
2492	* If there are external interrupts pending but masked by the TPR value, instruct VT-x to cause a VM-exit when
2493	* the guest lowers its TPR below the highest-priority pending interrupt and we can deliver the interrupt.
2494	* If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
2495	* the interrupt when we VM-exit for other reasons.
2496	*/
2497	pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8Tpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
2498	uint32_t u32TprThreshold = 0;
2499	if (fPendingIntr)
2500	{
2501	/* Bits 3-0 of the TPR threshold field correspond to bits 7-4 of the TPR (which is the Task-Priority Class). */
2502	const uint8_t u8PendingPriority = (u8PendingIntr >> 4);
2503	const uint8_t u8TprPriority = (u8Tpr >> 4) & 7;
2504	if (u8PendingPriority <= u8TprPriority)
2505	u32TprThreshold = u8PendingPriority;
2506	else
2507	u32TprThreshold = u8TprPriority; /* Required for Vista 64-bit guest, see @bugref{6398}. */
2508	}
2509	Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
2510
2511	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
2512	AssertRCReturn(rc, rc);
2513
2514	/* 32-bit guests uses LSTAR MSR for patching guest code which touches the TPR. */
2515	if (pVCpu->CTX_SUFF(pVM)->hm.s.fTPRPatchingActive)
2516	{
2517	Assert(!CPUMIsGuestInLongModeEx(pMixedCtx)); /* EFER always up-to-date. */
2518	pMixedCtx->msrLSTAR = u8Tpr;
2519	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2520	{
2521	/* If there are interrupts pending, intercept CR8 writes, otherwise don't intercept CR8 reads or writes. */
2522	if (fPendingIntr)
2523	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_INTERCEPT_WRITE);
2524	else
2525	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2526	}
2527	}
2528	}
2529
2530	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_APIC_STATE;
2531	}
2532	return rc;
2533	}
2534
2535
2536	/**
2537	* Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
2538	*
2539	* @returns
2540	* @param pVCpu Pointer to the VMCPU.
2541	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2542	* out-of-sync. Make sure to update the required fields
2543	* before using them.
2544	*
2545	* @remarks No-long-jump zone!!!
2546	* @remarks Has side-effects with VMCPU_FF_INHIBIT_INTERRUPTS force-flag.
2547	*/
2548	DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2549	{
2550	/*
2551	* Instructions like STI and MOV SS inhibit interrupts till the next instruction completes. Check if we should
2552	* inhibit interrupts or clear any existing interrupt-inhibition.
2553	*/
2554	uint32_t uIntrState = 0;
2555	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
2556	{
2557	/* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
2558	AssertMsg((pVCpu->hm.s.vmx.fUpdatedGuestState & (HMVMX_UPDATED_GUEST_RIP \| HMVMX_UPDATED_GUEST_RFLAGS))
2559	== (HMVMX_UPDATED_GUEST_RIP \| HMVMX_UPDATED_GUEST_RFLAGS), ("%#x\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
2560	if (pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
2561	{
2562	/*
2563	* We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
2564	* VT-x the flag's condition to be cleared is met and thus the cleared state is correct.
2565	*/
2566	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2567	}
2568	else if (pMixedCtx->eflags.Bits.u1IF)
2569	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
2570	else
2571	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
2572	}
2573	return uIntrState;
2574	}
2575
2576
2577	/**
2578	* Loads the guest's interruptibility-state into the guest-state area in the
2579	* VMCS.
2580	*
2581	* @returns VBox status code.
2582	* @param pVCpu Pointer to the VMCPU.
2583	* @param uIntrState The interruptibility-state to set.
2584	*/
2585	static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
2586	{
2587	AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
2588	Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
2589	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
2590	AssertRCReturn(rc, rc);
2591	return rc;
2592	}
2593
2594
2595	/**
2596	* Loads the guest's RIP into the guest-state area in the VMCS.
2597	*
2598	* @returns VBox status code.
2599	* @param pVCpu Pointer to the VMCPU.
2600	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2601	* out-of-sync. Make sure to update the required fields
2602	* before using them.
2603	*
2604	* @remarks No-long-jump zone!!!
2605	*/
2606	static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2607	{
2608	int rc = VINF_SUCCESS;
2609	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RIP)
2610	{
2611	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
2612	AssertRCReturn(rc, rc);
2613	Log(("Load: VMX_VMCS_GUEST_RIP=%#RX64\n", pMixedCtx->rip));
2614	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RIP;
2615	}
2616	return rc;
2617	}
2618
2619
2620	/**
2621	* Loads the guest's RSP into the guest-state area in the VMCS.
2622	*
2623	* @returns VBox status code.
2624	* @param pVCpu Pointer to the VMCPU.
2625	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2626	* out-of-sync. Make sure to update the required fields
2627	* before using them.
2628	*
2629	* @remarks No-long-jump zone!!!
2630	*/
2631	static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2632	{
2633	int rc = VINF_SUCCESS;
2634	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RSP)
2635	{
2636	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
2637	AssertRCReturn(rc, rc);
2638	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RSP;
2639	}
2640	return rc;
2641	}
2642
2643
2644	/**
2645	* Loads the guest's RFLAGS into the guest-state area in the VMCS.
2646	*
2647	* @returns VBox status code.
2648	* @param pVCpu Pointer to the VMCPU.
2649	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2650	* out-of-sync. Make sure to update the required fields
2651	* before using them.
2652	*
2653	* @remarks No-long-jump zone!!!
2654	*/
2655	static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2656	{
2657	int rc = VINF_SUCCESS;
2658	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RFLAGS)
2659	{
2660	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
2661	Let us assert it as such and use 32-bit VMWRITE. */
2662	Assert(!(pMixedCtx->rflags.u64 >> 32));
2663	X86EFLAGS uEFlags = pMixedCtx->eflags;
2664	uEFlags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
2665	uEFlags.u32 \|= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
2666
2667	/*
2668	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM exit.
2669	* Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
2670	*/
2671	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2672	{
2673	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
2674	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
2675	pVCpu->hm.s.vmx.RealMode.eflags.u32 = uEFlags.u32; /* Save the original eflags of the real-mode guest. */
2676	uEFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
2677	uEFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
2678	}
2679
2680	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, uEFlags.u32);
2681	AssertRCReturn(rc, rc);
2682
2683	Log(("Load: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", uEFlags.u32));
2684	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RFLAGS;
2685	}
2686	return rc;
2687	}
2688
2689
2690	/**
2691	* Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
2692	*
2693	* @returns VBox status code.
2694	* @param pVCpu Pointer to the VMCPU.
2695	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2696	* out-of-sync. Make sure to update the required fields
2697	* before using them.
2698	*
2699	* @remarks No-long-jump zone!!!
2700	*/
2701	DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2702	{
2703	int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
2704	AssertRCReturn(rc, rc);
2705	rc = hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
2706	AssertRCReturn(rc, rc);
2707	rc = hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
2708	AssertRCReturn(rc, rc);
2709	return rc;
2710	}
2711
2712
2713	/**
2714	* Loads the guest control registers (CR0, CR3, CR4) into the guest-state area
2715	* in the VMCS.
2716	*
2717	* @returns VBox status code.
2718	* @param pVM Pointer to the VM.
2719	* @param pVCpu Pointer to the VMCPU.
2720	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2721	* out-of-sync. Make sure to update the required fields
2722	* before using them.
2723	*
2724	* @remarks No-long-jump zone!!!
2725	*/
2726	static int hmR0VmxLoadGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pCtx)
2727	{
2728	int rc = VINF_SUCCESS;
2729	PVM pVM = pVCpu->CTX_SUFF(pVM);
2730
2731	/*
2732	* Guest CR0.
2733	* Guest FPU.
2734	*/
2735	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR0)
2736	{
2737	Assert(!(pCtx->cr0 >> 32));
2738	uint32_t u32GuestCR0 = pCtx->cr0;
2739
2740	/* The guest's view (read access) of its CR0 is unblemished. */
2741	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
2742	AssertRCReturn(rc, rc);
2743	Log(("Load: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", u32GuestCR0));
2744
2745	/* Setup VT-x's view of the guest CR0. */
2746	/* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
2747	if (pVM->hm.s.fNestedPaging)
2748	{
2749	if (CPUMIsGuestPagingEnabledEx(pCtx))
2750	{
2751	/* The guest has paging enabled, let it access CR3 without causing a VM exit if supported. */
2752	pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2753	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
2754	}
2755	else
2756	{
2757	/* The guest doesn't have paging enabled, make CR3 access to cause VM exits to update our shadow. */
2758	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2759	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2760	}
2761
2762	/* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
2763	if (pVM->hm.s.vmx.fUnrestrictedGuest)
2764	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2765
2766	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
2767	AssertRCReturn(rc, rc);
2768	}
2769	else
2770	u32GuestCR0 \|= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a VM-exit. */
2771
2772	/*
2773	* Guest FPU bits.
2774	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
2775	* CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
2776	*/
2777	u32GuestCR0 \|= X86_CR0_NE;
2778	bool fInterceptNM = false;
2779	if (CPUMIsGuestFPUStateActive(pVCpu))
2780	{
2781	fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
2782	/* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
2783	We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
2784	}
2785	else
2786	{
2787	fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
2788	u32GuestCR0 \|= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
2789	\| X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
2790	}
2791
2792	/* Catch floating point exceptions if we need to report them to the guest in a different way. */
2793	bool fInterceptMF = false;
2794	if (!(pCtx->cr0 & X86_CR0_NE))
2795	fInterceptMF = true;
2796
2797	/* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
2798	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2799	{
2800	Assert(PDMVmmDevHeapIsEnabled(pVM));
2801	Assert(pVM->hm.s.vmx.pRealModeTSS);
2802	pVCpu->hm.s.vmx.u32XcptBitmap \|= HMVMX_REAL_MODE_XCPT_MASK;
2803	fInterceptNM = true;
2804	fInterceptMF = true;
2805	}
2806	else
2807	pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
2808
2809	if (fInterceptNM)
2810	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_NM);
2811	else
2812	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
2813
2814	if (fInterceptMF)
2815	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_MF);
2816	else
2817	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
2818
2819	/* Additional intercepts for debugging, define these yourself explicitly. */
2820	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
2821	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_BP)
2822	\| RT_BIT(X86_XCPT_DB)
2823	\| RT_BIT(X86_XCPT_DE)
2824	\| RT_BIT(X86_XCPT_NM)
2825	\| RT_BIT(X86_XCPT_UD)
2826	\| RT_BIT(X86_XCPT_NP)
2827	\| RT_BIT(X86_XCPT_SS)
2828	\| RT_BIT(X86_XCPT_GP)
2829	\| RT_BIT(X86_XCPT_PF)
2830	\| RT_BIT(X86_XCPT_MF);
2831	#elif defined(HMVMX_ALWAYS_TRAP_PF)
2832	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
2833	#endif
2834
2835	Assert(pVM->hm.s.fNestedPaging \|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
2836
2837	/* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
2838	uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.msr.vmx_cr0_fixed0 & pVM->hm.s.vmx.msr.vmx_cr0_fixed1);
2839	uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.msr.vmx_cr0_fixed0 \| pVM->hm.s.vmx.msr.vmx_cr0_fixed1);
2840	if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
2841	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
2842	else
2843	Assert((uSetCR0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
2844
2845	u32GuestCR0 \|= uSetCR0;
2846	u32GuestCR0 &= uZapCR0;
2847	u32GuestCR0 &= ~(X86_CR0_CD \| X86_CR0_NW); /* Always enable caching. */
2848
2849	/* Write VT-x's view of the guest CR0 into the VMCS and update the exception bitmap. */
2850	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
2851	AssertRCReturn(rc, rc);
2852	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
2853	AssertRCReturn(rc, rc);
2854	Log(("Load: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", u32GuestCR0, uSetCR0, uZapCR0));
2855
2856	/*
2857	* CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
2858	* by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
2859	* we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
2860	*/
2861	uint32_t u32CR0Mask = 0;
2862	u32CR0Mask = X86_CR0_PE
2863	\| X86_CR0_NE
2864	\| X86_CR0_WP
2865	\| X86_CR0_PG
2866	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
2867	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
2868	\| X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
2869	if (pVM->hm.s.vmx.fUnrestrictedGuest)
2870	u32CR0Mask &= ~X86_CR0_PE;
2871	if (pVM->hm.s.fNestedPaging)
2872	u32CR0Mask &= ~X86_CR0_WP;
2873
2874	/* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
2875	if (fInterceptNM)
2876	u32CR0Mask \|= (X86_CR0_TS \| X86_CR0_MP);
2877	else
2878	u32CR0Mask &= ~(X86_CR0_TS \| X86_CR0_MP);
2879
2880	/* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
2881	pVCpu->hm.s.vmx.cr0_mask = u32CR0Mask;
2882	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
2883	AssertRCReturn(rc, rc);
2884
2885	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR0;
2886	}
2887
2888	/*
2889	* Guest CR2.
2890	* It's always loaded in the assembler code. Nothing to do here.
2891	*/
2892
2893	/*
2894	* Guest CR3.
2895	*/
2896	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR3)
2897	{
2898	RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
2899	if (pVM->hm.s.fNestedPaging)
2900	{
2901	pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
2902
2903	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
2904	Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
2905	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
2906	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
2907
2908	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
2909	pVCpu->hm.s.vmx.HCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
2910	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
2911
2912	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
2913	AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
2914	&& ((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x3f) == 0, /* Bits 6:11 MBZ. */
2915	("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
2916
2917	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
2918	AssertRCReturn(rc, rc);
2919	Log(("Load: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
2920
2921	if ( pVM->hm.s.vmx.fUnrestrictedGuest
2922	\|\| CPUMIsGuestPagingEnabledEx(pCtx))
2923	{
2924	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
2925	if (CPUMIsGuestInPAEModeEx(pCtx))
2926	{
2927	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]); AssertRCReturn(rc, rc);
2928	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
2929	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
2930	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
2931	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
2932	}
2933
2934	/* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
2935	have Unrestricted Execution to handle the guest when it's not using paging. */
2936	GCPhysGuestCR3 = pCtx->cr3;
2937	}
2938	else
2939	{
2940	/*
2941	* The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
2942	* directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
2943	* EPT takes care of translating it to host-physical addresses.
2944	*/
2945	RTGCPHYS GCPhys;
2946	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
2947	Assert(PDMVmmDevHeapIsEnabled(pVM));
2948
2949	/* We obtain it here every time as the guest could have relocated this PCI region. */
2950	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
2951	AssertRCReturn(rc, rc);
2952
2953	GCPhysGuestCR3 = GCPhys;
2954	}
2955
2956	Log(("Load: VMX_VMCS_GUEST_CR3=%#RGv (GstN)\n", GCPhysGuestCR3));
2957	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
2958	}
2959	else
2960	{
2961	/* Non-nested paging case, just use the hypervisor's CR3. */
2962	RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
2963
2964	Log(("Load: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", HCPhysGuestCR3));
2965	rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
2966	}
2967	AssertRCReturn(rc, rc);
2968
2969	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR3;
2970	}
2971
2972	/*
2973	* Guest CR4.
2974	*/
2975	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR4)
2976	{
2977	Assert(!(pCtx->cr4 >> 32));
2978	uint32_t u32GuestCR4 = pCtx->cr4;
2979
2980	/* The guest's view of its CR4 is unblemished. */
2981	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
2982	AssertRCReturn(rc, rc);
2983	Log(("Load: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", u32GuestCR4));
2984
2985	/* Setup VT-x's view of the guest CR4. */
2986	/*
2987	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
2988	* interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
2989	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
2990	*/
2991	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2992	{
2993	Assert(pVM->hm.s.vmx.pRealModeTSS);
2994	Assert(PDMVmmDevHeapIsEnabled(pVM));
2995	u32GuestCR4 &= ~X86_CR4_VME;
2996	}
2997
2998	if (pVM->hm.s.fNestedPaging)
2999	{
3000	if ( !CPUMIsGuestPagingEnabledEx(pCtx)
3001	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
3002	{
3003	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
3004	u32GuestCR4 \|= X86_CR4_PSE;
3005	/* Our identity mapping is a 32 bits page directory. */
3006	u32GuestCR4 &= ~X86_CR4_PAE;
3007	}
3008	/* else use guest CR4.*/
3009	}
3010	else
3011	{
3012	/*
3013	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
3014	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
3015	*/
3016	switch (pVCpu->hm.s.enmShadowMode)
3017	{
3018	case PGMMODE_REAL: /* Real-mode. */
3019	case PGMMODE_PROTECTED: /* Protected mode without paging. */
3020	case PGMMODE_32_BIT: /* 32-bit paging. */
3021	{
3022	u32GuestCR4 &= ~X86_CR4_PAE;
3023	break;
3024	}
3025
3026	case PGMMODE_PAE: /* PAE paging. */
3027	case PGMMODE_PAE_NX: /* PAE paging with NX. */
3028	{
3029	u32GuestCR4 \|= X86_CR4_PAE;
3030	break;
3031	}
3032
3033	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
3034	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
3035	#ifdef VBOX_ENABLE_64_BITS_GUESTS
3036	break;
3037	#endif
3038	default:
3039	AssertFailed();
3040	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3041	}
3042	}
3043
3044	/* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
3045	uint64_t uSetCR4 = (pVM->hm.s.vmx.msr.vmx_cr4_fixed0 & pVM->hm.s.vmx.msr.vmx_cr4_fixed1);
3046	uint64_t uZapCR4 = (pVM->hm.s.vmx.msr.vmx_cr4_fixed0 \| pVM->hm.s.vmx.msr.vmx_cr4_fixed1);
3047	u32GuestCR4 \|= uSetCR4;
3048	u32GuestCR4 &= uZapCR4;
3049
3050	/* Write VT-x's view of the guest CR4 into the VMCS. */
3051	Log(("Load: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", u32GuestCR4, uSetCR4, uZapCR4));
3052	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
3053	AssertRCReturn(rc, rc);
3054
3055	/* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM exit. */
3056	uint32_t u32CR4Mask = 0;
3057	u32CR4Mask = X86_CR4_VME
3058	\| X86_CR4_PAE
3059	\| X86_CR4_PGE
3060	\| X86_CR4_PSE
3061	\| X86_CR4_VMXE;
3062	pVCpu->hm.s.vmx.cr4_mask = u32CR4Mask;
3063	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
3064	AssertRCReturn(rc, rc);
3065
3066	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR4;
3067	}
3068	return rc;
3069	}
3070
3071
3072	/**
3073	* Loads the guest debug registers into the guest-state area in the VMCS.
3074	* This also sets up whether #DB and MOV DRx accesses cause VM exits.
3075	*
3076	* @returns VBox status code.
3077	* @param pVCpu Pointer to the VMCPU.
3078	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3079	* out-of-sync. Make sure to update the required fields
3080	* before using them.
3081	*
3082	* @remarks No-long-jump zone!!!
3083	*/
3084	static int hmR0VmxLoadGuestDebugRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3085	{
3086	if (!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_DEBUG))
3087	return VINF_SUCCESS;
3088
3089	#ifdef VBOX_STRICT
3090	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
3091	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
3092	{
3093	Assert(!(pMixedCtx->dr[7] >> 32)); /* upper 32 bits are reserved (MBZ). */
3094	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
3095	Assert((pMixedCtx->dr[7] & 0xd800) == 0); /* bits 15, 14, 12, 11 are reserved (MBZ). */
3096	Assert((pMixedCtx->dr[7] & 0x400) == 0x400); /* bit 10 is reserved (MB1). */
3097	}
3098	#endif
3099
3100	int rc = VERR_INTERNAL_ERROR_5;
3101	PVM pVM = pVCpu->CTX_SUFF(pVM);
3102	bool fInterceptDB = false;
3103	bool fInterceptMovDRx = false;
3104	if (DBGFIsStepping(pVCpu))
3105	{
3106	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
3107	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
3108	{
3109	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
3110	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3111	AssertRCReturn(rc, rc);
3112	Assert(fInterceptDB == false);
3113	}
3114	else
3115	{
3116	fInterceptDB = true;
3117	pMixedCtx->eflags.u32 \|= X86_EFL_TF;
3118	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RFLAGS;
3119	}
3120	}
3121
3122	if (CPUMGetHyperDR7(pVCpu) & (X86_DR7_ENABLED_MASK \| X86_DR7_GD))
3123	{
3124	if (!CPUMIsHyperDebugStateActive(pVCpu))
3125	{
3126	rc = CPUMR0LoadHyperDebugState(pVM, pVCpu, pMixedCtx, true /* include DR6 */);
3127	AssertRC(rc);
3128	}
3129	Assert(CPUMIsHyperDebugStateActive(pVCpu));
3130	fInterceptMovDRx = true;
3131	}
3132	else if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD))
3133	{
3134	if (!CPUMIsGuestDebugStateActive(pVCpu))
3135	{
3136	rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pMixedCtx, true /* include DR6 */);
3137	AssertRC(rc);
3138	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
3139	}
3140	Assert(CPUMIsGuestDebugStateActive(pVCpu));
3141	Assert(fInterceptMovDRx == false);
3142	}
3143	else if (!CPUMIsGuestDebugStateActive(pVCpu))
3144	{
3145	/* For the first time we would need to intercept MOV DRx accesses even when the guest debug registers aren't loaded. */
3146	fInterceptMovDRx = true;
3147	}
3148
3149	/* Update the exception bitmap regarding intercepting #DB generated by the guest. */
3150	if (fInterceptDB)
3151	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_DB);
3152	else if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3153	{
3154	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3155	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
3156	#endif
3157	}
3158
3159	/* Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions. */
3160	if (fInterceptMovDRx)
3161	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
3162	else
3163	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
3164
3165	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3166	AssertRCReturn(rc, rc);
3167	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3168	AssertRCReturn(rc, rc);
3169
3170	/* The guest's view of its DR7 is unblemished. Use 32-bit write as upper 32-bits MBZ as asserted above. */
3171	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
3172	AssertRCReturn(rc, rc);
3173
3174	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_DEBUG;
3175	return rc;
3176	}
3177
3178
3179	#ifdef VBOX_STRICT
3180	/**
3181	* Strict function to validate segment registers.
3182	*
3183	* @remarks Requires CR0.
3184	*/
3185	static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3186	{
3187	/* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
3188	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0);
3189	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS);
3190
3191	/* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
3192	* only updates the VMCS bits with the unusable bit and doesn't change the guest-context value. */
3193	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
3194	&& ( !CPUMIsGuestInRealModeEx(pCtx)
3195	&& !CPUMIsGuestInV86ModeEx(pCtx)))
3196	{
3197	/* Protected mode checks */
3198	/* CS */
3199	Assert(pCtx->cs.Attr.n.u1Present);
3200	Assert(!(pCtx->cs.Attr.u & 0xf00));
3201	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
3202	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
3203	\|\| !(pCtx->cs.Attr.n.u1Granularity));
3204	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
3205	\|\| (pCtx->cs.Attr.n.u1Granularity));
3206	/* CS cannot be loaded with NULL in protected mode. */
3207	Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & HMVMX_SEL_UNUSABLE)); /** @todo is this really true even for 64-bit CS?!? */
3208	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
3209	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
3210	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
3211	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
3212	else
3213	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
3214	/* SS */
3215	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3216	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
3217	Assert(!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR0));
3218	if ( !(pCtx->cr0 & X86_CR0_PE)
3219	\|\| pCtx->cs.Attr.n.u4Type == 3)
3220	{
3221	Assert(!pCtx->ss.Attr.n.u2Dpl);
3222	}
3223	if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & HMVMX_SEL_UNUSABLE))
3224	{
3225	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3226	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
3227	Assert(pCtx->ss.Attr.n.u1Present);
3228	Assert(!(pCtx->ss.Attr.u & 0xf00));
3229	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
3230	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
3231	\|\| !(pCtx->ss.Attr.n.u1Granularity));
3232	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
3233	\|\| (pCtx->ss.Attr.n.u1Granularity));
3234	}
3235	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
3236	if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & HMVMX_SEL_UNUSABLE))
3237	{
3238	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3239	Assert(pCtx->ds.Attr.n.u1Present);
3240	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
3241	Assert(!(pCtx->ds.Attr.u & 0xf00));
3242	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
3243	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
3244	\|\| !(pCtx->ds.Attr.n.u1Granularity));
3245	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
3246	\|\| (pCtx->ds.Attr.n.u1Granularity));
3247	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3248	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
3249	}
3250	if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & HMVMX_SEL_UNUSABLE))
3251	{
3252	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3253	Assert(pCtx->es.Attr.n.u1Present);
3254	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
3255	Assert(!(pCtx->es.Attr.u & 0xf00));
3256	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
3257	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
3258	\|\| !(pCtx->es.Attr.n.u1Granularity));
3259	Assert( !(pCtx->es.u32Limit & 0xfff00000)
3260	\|\| (pCtx->es.Attr.n.u1Granularity));
3261	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3262	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
3263	}
3264	if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & HMVMX_SEL_UNUSABLE))
3265	{
3266	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3267	Assert(pCtx->fs.Attr.n.u1Present);
3268	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
3269	Assert(!(pCtx->fs.Attr.u & 0xf00));
3270	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
3271	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
3272	\|\| !(pCtx->fs.Attr.n.u1Granularity));
3273	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
3274	\|\| (pCtx->fs.Attr.n.u1Granularity));
3275	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3276	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3277	}
3278	if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & HMVMX_SEL_UNUSABLE))
3279	{
3280	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3281	Assert(pCtx->gs.Attr.n.u1Present);
3282	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
3283	Assert(!(pCtx->gs.Attr.u & 0xf00));
3284	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
3285	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
3286	\|\| !(pCtx->gs.Attr.n.u1Granularity));
3287	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
3288	\|\| (pCtx->gs.Attr.n.u1Granularity));
3289	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3290	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3291	}
3292	/* 64-bit capable CPUs. */
3293	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3294	Assert(!(pCtx->cs.u64Base >> 32));
3295	Assert(!pCtx->ss.Attr.u \|\| !(pCtx->ss.u64Base >> 32));
3296	Assert(!pCtx->ds.Attr.u \|\| !(pCtx->ds.u64Base >> 32));
3297	Assert(!pCtx->es.Attr.u \|\| !(pCtx->es.u64Base >> 32));
3298	# endif
3299	}
3300	else if ( CPUMIsGuestInV86ModeEx(pCtx)
3301	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
3302	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
3303	{
3304	/* Real and v86 mode checks. */
3305	/* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
3306	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
3307	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3308	{
3309	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
3310	}
3311	else
3312	{
3313	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
3314	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
3315	}
3316
3317	/* CS */
3318	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
3319	Assert(pCtx->cs.u32Limit == 0xffff);
3320	Assert(u32CSAttr == 0xf3);
3321	/* SS */
3322	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
3323	Assert(pCtx->ss.u32Limit == 0xffff);
3324	Assert(u32SSAttr == 0xf3);
3325	/* DS */
3326	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
3327	Assert(pCtx->ds.u32Limit == 0xffff);
3328	Assert(u32DSAttr == 0xf3);
3329	/* ES */
3330	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
3331	Assert(pCtx->es.u32Limit == 0xffff);
3332	Assert(u32ESAttr == 0xf3);
3333	/* FS */
3334	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
3335	Assert(pCtx->fs.u32Limit == 0xffff);
3336	Assert(u32FSAttr == 0xf3);
3337	/* GS */
3338	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
3339	Assert(pCtx->gs.u32Limit == 0xffff);
3340	Assert(u32GSAttr == 0xf3);
3341	/* 64-bit capable CPUs. */
3342	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3343	Assert(!(pCtx->cs.u64Base >> 32));
3344	Assert(!u32SSAttr \|\| !(pCtx->ss.u64Base >> 32));
3345	Assert(!u32DSAttr \|\| !(pCtx->ds.u64Base >> 32));
3346	Assert(!u32ESAttr \|\| !(pCtx->es.u64Base >> 32));
3347	# endif
3348	}
3349	}
3350	#endif /* VBOX_STRICT */
3351
3352
3353	/**
3354	* Writes a guest segment register into the guest-state area in the VMCS.
3355	*
3356	* @returns VBox status code.
3357	* @param pVCpu Pointer to the VMCPU.
3358	* @param idxSel Index of the selector in the VMCS.
3359	* @param idxLimit Index of the segment limit in the VMCS.
3360	* @param idxBase Index of the segment base in the VMCS.
3361	* @param idxAccess Index of the access rights of the segment in the VMCS.
3362	* @param pSelReg Pointer to the segment selector.
3363	* @param pCtx Pointer to the guest-CPU context.
3364	*
3365	* @remarks No-long-jump zone!!!
3366	*/
3367	static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
3368	uint32_t idxAccess, PCPUMSELREG pSelReg, PCPUMCTX pCtx)
3369	{
3370	int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
3371	AssertRCReturn(rc, rc);
3372	rc = VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
3373	AssertRCReturn(rc, rc);
3374	rc = VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
3375	AssertRCReturn(rc, rc);
3376
3377	uint32_t u32Access = pSelReg->Attr.u;
3378	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3379	{
3380	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
3381	u32Access = 0xf3;
3382	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3383	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3384	}
3385	else
3386	{
3387	/*
3388	* The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
3389	* real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
3390	* protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
3391	* loaded in protected-mode have their attribute as 0.
3392	*/
3393	if (!u32Access)
3394	u32Access = HMVMX_SEL_UNUSABLE;
3395	}
3396
3397	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
3398	AssertMsg((u32Access & HMVMX_SEL_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
3399	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
3400
3401	rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
3402	AssertRCReturn(rc, rc);
3403	return rc;
3404	}
3405
3406
3407	/**
3408	* Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
3409	* into the guest-state area in the VMCS.
3410	*
3411	* @returns VBox status code.
3412	* @param pVM Pointer to the VM.
3413	* @param pVCPU Pointer to the VMCPU.
3414	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3415	* out-of-sync. Make sure to update the required fields
3416	* before using them.
3417	*
3418	* @remarks Requires CR0 (strict builds validation).
3419	* @remarks No-long-jump zone!!!
3420	*/
3421	static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3422	{
3423	int rc = VERR_INTERNAL_ERROR_5;
3424	PVM pVM = pVCpu->CTX_SUFF(pVM);
3425
3426	/*
3427	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
3428	*/
3429	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SEGMENT_REGS)
3430	{
3431	/* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
3432	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3433	{
3434	pVCpu->hm.s.vmx.RealMode.uAttrCS.u = pMixedCtx->cs.Attr.u;
3435	pVCpu->hm.s.vmx.RealMode.uAttrSS.u = pMixedCtx->ss.Attr.u;
3436	pVCpu->hm.s.vmx.RealMode.uAttrDS.u = pMixedCtx->ds.Attr.u;
3437	pVCpu->hm.s.vmx.RealMode.uAttrES.u = pMixedCtx->es.Attr.u;
3438	pVCpu->hm.s.vmx.RealMode.uAttrFS.u = pMixedCtx->fs.Attr.u;
3439	pVCpu->hm.s.vmx.RealMode.uAttrGS.u = pMixedCtx->gs.Attr.u;
3440	}
3441
3442	#ifdef VBOX_WITH_REM
3443	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
3444	{
3445	Assert(pVM->hm.s.vmx.pRealModeTSS);
3446	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
3447	if ( pVCpu->hm.s.vmx.fWasInRealMode
3448	&& PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
3449	{
3450	/* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
3451	in real-mode (e.g. OpenBSD 4.0) */
3452	REMFlushTBs(pVM);
3453	Log(("Load: Switch to protected mode detected!\n"));
3454	pVCpu->hm.s.vmx.fWasInRealMode = false;
3455	}
3456	}
3457	#endif
3458	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
3459	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs, pMixedCtx);
3460	AssertRCReturn(rc, rc);
3461	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
3462	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss, pMixedCtx);
3463	AssertRCReturn(rc, rc);
3464	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
3465	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds, pMixedCtx);
3466	AssertRCReturn(rc, rc);
3467	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
3468	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es, pMixedCtx);
3469	AssertRCReturn(rc, rc);
3470	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
3471	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs, pMixedCtx);
3472	AssertRCReturn(rc, rc);
3473	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
3474	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs, pMixedCtx);
3475	AssertRCReturn(rc, rc);
3476
3477	Log(("Load: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pMixedCtx->cs.Sel, pMixedCtx->cs.u64Base,
3478	pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
3479	#ifdef VBOX_STRICT
3480	hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
3481	#endif
3482	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SEGMENT_REGS;
3483	}
3484
3485	/*
3486	* Guest TR.
3487	*/
3488	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_TR)
3489	{
3490	/*
3491	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
3492	* using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
3493	* See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
3494	*/
3495	uint16_t u16Sel = 0;
3496	uint32_t u32Limit = 0;
3497	uint64_t u64Base = 0;
3498	uint32_t u32AccessRights = 0;
3499
3500	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3501	{
3502	u16Sel = pMixedCtx->tr.Sel;
3503	u32Limit = pMixedCtx->tr.u32Limit;
3504	u64Base = pMixedCtx->tr.u64Base;
3505	u32AccessRights = pMixedCtx->tr.Attr.u;
3506	}
3507	else
3508	{
3509	Assert(pVM->hm.s.vmx.pRealModeTSS);
3510	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
3511
3512	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
3513	RTGCPHYS GCPhys;
3514	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
3515	AssertRCReturn(rc, rc);
3516
3517	X86DESCATTR DescAttr;
3518	DescAttr.u = 0;
3519	DescAttr.n.u1Present = 1;
3520	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
3521
3522	u16Sel = 0;
3523	u32Limit = HM_VTX_TSS_SIZE;
3524	u64Base = GCPhys; /* in real-mode phys = virt. */
3525	u32AccessRights = DescAttr.u;
3526	}
3527
3528	/* Validate. */
3529	Assert(!(u16Sel & RT_BIT(2)));
3530	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
3531	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
3532	AssertMsg(!(u32AccessRights & HMVMX_SEL_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
3533	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
3534	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
3535	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
3536	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
3537	Assert( (u32Limit & 0xfff) == 0xfff
3538	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
3539	Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
3540	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
3541
3542	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_TR, u16Sel); AssertRCReturn(rc, rc);
3543	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit); AssertRCReturn(rc, rc);
3544	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base); AssertRCReturn(rc, rc);
3545	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights); AssertRCReturn(rc, rc);
3546
3547	Log(("Load: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", u64Base));
3548	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_TR;
3549	}
3550
3551	/*
3552	* Guest GDTR.
3553	*/
3554	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_GDTR)
3555	{
3556	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt); AssertRCReturn(rc, rc);
3557	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt); AssertRCReturn(rc, rc);
3558
3559	Assert(!(pMixedCtx->gdtr.cbGdt & UINT64_C(0xffff0000))); /* Bits 31:16 MBZ. */
3560	Log(("Load: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pMixedCtx->gdtr.pGdt));
3561	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_GDTR;
3562	}
3563
3564	/*
3565	* Guest LDTR.
3566	*/
3567	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_LDTR)
3568	{
3569	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
3570	uint32_t u32Access = 0;
3571	if (!pMixedCtx->ldtr.Attr.u)
3572	u32Access = HMVMX_SEL_UNUSABLE;
3573	else
3574	u32Access = pMixedCtx->ldtr.Attr.u;
3575
3576	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_LDTR, pMixedCtx->ldtr.Sel); AssertRCReturn(rc, rc);
3577	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit); AssertRCReturn(rc, rc);
3578	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base); AssertRCReturn(rc, rc);
3579	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access); AssertRCReturn(rc, rc);
3580
3581	/* Validate. */
3582	if (!(u32Access & HMVMX_SEL_UNUSABLE))
3583	{
3584	Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
3585	Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
3586	Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
3587	Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
3588	Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
3589	Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
3590	Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
3591	\|\| !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
3592	Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
3593	\|\| pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
3594	}
3595
3596	Log(("Load: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pMixedCtx->ldtr.u64Base));
3597	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_LDTR;
3598	}
3599
3600	/*
3601	* Guest IDTR.
3602	*/
3603	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_IDTR)
3604	{
3605	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt); AssertRCReturn(rc, rc);
3606	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt); AssertRCReturn(rc, rc);
3607
3608	Assert(!(pMixedCtx->idtr.cbIdt & UINT64_C(0xffff0000))); /* Bits 31:16 MBZ. */
3609	Log(("Load: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pMixedCtx->idtr.pIdt));
3610	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_IDTR;
3611	}
3612
3613	return VINF_SUCCESS;
3614	}
3615
3616
3617	/**
3618	* Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
3619	* areas. These MSRs will automatically be loaded to the host CPU on every
3620	* successful VM entry and stored from the host CPU on every successful VM exit.
3621	* Also loads the sysenter MSRs into the guest-state area in the VMCS.
3622	*
3623	* @returns VBox status code.
3624	* @param pVCpu Pointer to the VMCPU.
3625	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3626	* out-of-sync. Make sure to update the required fields
3627	* before using them.
3628	*
3629	* @remarks No-long-jump zone!!!
3630	*/
3631	static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3632	{
3633	AssertPtr(pVCpu);
3634	AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
3635
3636	/*
3637	* MSRs covered by Auto-load/store: EFER, LSTAR, STAR, SF_MASK, TSC_AUX (RDTSCP).
3638	*/
3639	int rc = VINF_SUCCESS;
3640	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
3641	{
3642	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
3643	PVM pVM = pVCpu->CTX_SUFF(pVM);
3644	PVMXMSR pGuestMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvGuestMsr;
3645	uint32_t cGuestMsrs = 0;
3646
3647	/* See Intel spec. 4.1.4 "Enumeration of Paging Features by CPUID". */
3648	const bool fSupportsNX = CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_NX);
3649	const bool fSupportsLongMode = CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_LONG_MODE);
3650	if (fSupportsNX \|\| fSupportsLongMode)
3651	{
3652	/** @todo support save IA32_EFER, i.e.
3653	* VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR, in which case the
3654	* guest EFER need not be part of the VM-entry MSR-load area. */
3655	pGuestMsr->u32IndexMSR = MSR_K6_EFER;
3656	pGuestMsr->u32Reserved = 0;
3657	pGuestMsr->u64Value = pMixedCtx->msrEFER;
3658	/* VT-x will complain if only MSR_K6_EFER_LME is set. See Intel spec. 26.4 "Loading MSRs" for details. */
3659	if (!CPUMIsGuestInLongModeEx(pMixedCtx))
3660	pGuestMsr->u64Value &= ~(MSR_K6_EFER_LMA \| MSR_K6_EFER_LME);
3661	pGuestMsr++; cGuestMsrs++;
3662	if (fSupportsLongMode)
3663	{
3664	pGuestMsr->u32IndexMSR = MSR_K8_LSTAR;
3665	pGuestMsr->u32Reserved = 0;
3666	pGuestMsr->u64Value = pMixedCtx->msrLSTAR; /* 64 bits mode syscall rip */
3667	pGuestMsr++; cGuestMsrs++;
3668	pGuestMsr->u32IndexMSR = MSR_K6_STAR;
3669	pGuestMsr->u32Reserved = 0;
3670	pGuestMsr->u64Value = pMixedCtx->msrSTAR; /* legacy syscall eip, cs & ss */
3671	pGuestMsr++; cGuestMsrs++;
3672	pGuestMsr->u32IndexMSR = MSR_K8_SF_MASK;
3673	pGuestMsr->u32Reserved = 0;
3674	pGuestMsr->u64Value = pMixedCtx->msrSFMASK; /* syscall flag mask */
3675	pGuestMsr++; cGuestMsrs++;
3676	pGuestMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
3677	pGuestMsr->u32Reserved = 0;
3678	pGuestMsr->u64Value = pMixedCtx->msrKERNELGSBASE; /* swapgs exchange value */
3679	pGuestMsr++; cGuestMsrs++;
3680	}
3681	}
3682
3683	/*
3684	* RDTSCP requires the TSC_AUX MSR. Host and guest share the physical MSR. So we have to
3685	* load the guest's copy if the guest can execute RDTSCP without causing VM-exits.
3686	*/
3687	if ( CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_RDTSCP)
3688	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP))
3689	{
3690	pGuestMsr->u32IndexMSR = MSR_K8_TSC_AUX;
3691	pGuestMsr->u32Reserved = 0;
3692	rc = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &pGuestMsr->u64Value);
3693	AssertRCReturn(rc, rc);
3694	pGuestMsr++; cGuestMsrs++;
3695	}
3696
3697	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
3698	if (cGuestMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc))
3699	{
3700	LogRel(("CPU autoload/store MSR count in VMCS exceeded cGuestMsrs=%u.\n", cGuestMsrs));
3701	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3702	}
3703
3704	/* Update the VCPU's copy of the guest MSR count. */
3705	pVCpu->hm.s.vmx.cGuestMsrs = cGuestMsrs;
3706	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cGuestMsrs); AssertRCReturn(rc, rc);
3707	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cGuestMsrs); AssertRCReturn(rc, rc);
3708	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
3709
3710	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_AUTO_MSRS;
3711	}
3712
3713	/*
3714	* Guest Sysenter MSRs.
3715	* These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
3716	* VM-exits on WRMSRs for these MSRs.
3717	*/
3718	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
3719	{
3720	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
3721	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_CS_MSR;
3722	}
3723	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
3724	{
3725	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
3726	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR;
3727	}
3728	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
3729	{
3730	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
3731	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR;
3732	}
3733
3734	return rc;
3735	}
3736
3737
3738	/**
3739	* Loads the guest activity state into the guest-state area in the VMCS.
3740	*
3741	* @returns VBox status code.
3742	* @param pVCpu Pointer to the VMCPU.
3743	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3744	* out-of-sync. Make sure to update the required fields
3745	* before using them.
3746	*
3747	* @remarks No-long-jump zone!!!
3748	*/
3749	static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pCtx)
3750	{
3751	/** @todo See if we can make use of other states, e.g.
3752	* VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
3753	int rc = VINF_SUCCESS;
3754	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_ACTIVITY_STATE)
3755	{
3756	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
3757	AssertRCReturn(rc, rc);
3758	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_ACTIVITY_STATE;
3759	}
3760	return rc;
3761	}
3762
3763
3764	/**
3765	* Sets up the appropriate function to run guest code.
3766	*
3767	* @returns VBox status code.
3768	* @param pVCpu Pointer to the VMCPU.
3769	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3770	* out-of-sync. Make sure to update the required fields
3771	* before using them.
3772	*
3773	* @remarks No-long-jump zone!!!
3774	*/
3775	static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3776	{
3777	if (CPUMIsGuestInLongModeEx(pMixedCtx))
3778	{
3779	#ifndef VBOX_ENABLE_64_BITS_GUESTS
3780	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3781	#endif
3782	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
3783	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3784	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
3785	pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
3786	#else
3787	/* 64-bit host or hybrid host. */
3788	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
3789	#endif
3790	}
3791	else
3792	{
3793	/* Guest is not in long mode, use the 32-bit handler. */
3794	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
3795	}
3796	Assert(pVCpu->hm.s.vmx.pfnStartVM);
3797	return VINF_SUCCESS;
3798	}
3799
3800
3801	/**
3802	* Wrapper for running the guest code in VT-x.
3803	*
3804	* @returns VBox strict status code.
3805	* @param pVM Pointer to the VM.
3806	* @param pVCpu Pointer to the VMCPU.
3807	* @param pCtx Pointer to the guest-CPU context.
3808	*
3809	* @remarks No-long-jump zone!!!
3810	*/
3811	DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3812	{
3813	/*
3814	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
3815	* using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
3816	* Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
3817	*/
3818	#ifdef VBOX_WITH_KERNEL_USING_XMM
3819	return HMR0VMXStartVMWrapXMM(pVCpu->hm.s.fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
3820	#else
3821	return pVCpu->hm.s.vmx.pfnStartVM(pVCpu->hm.s.fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
3822	#endif
3823	}
3824
3825
3826	/**
3827	* Report world-switch error and dump some useful debug info.
3828	*
3829	* @param pVM Pointer to the VM.
3830	* @param pVCpu Pointer to the VMCPU.
3831	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
3832	* @param pCtx Pointer to the guest-CPU context.
3833	* @param pVmxTransient Pointer to the VMX transient structure (only
3834	* exitReason updated).
3835	*/
3836	static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
3837	{
3838	Assert(pVM);
3839	Assert(pVCpu);
3840	Assert(pCtx);
3841	Assert(pVmxTransient);
3842	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3843
3844	Log(("VM-entry failure: %Rrc\n", rcVMRun));
3845	switch (rcVMRun)
3846	{
3847	case VERR_VMX_INVALID_VMXON_PTR:
3848	AssertFailed();
3849	break;
3850	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
3851	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
3852	{
3853	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.lasterror.u32ExitReason);
3854	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.lasterror.u32InstrError);
3855	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
3856	AssertRC(rc);
3857
3858	#ifdef VBOX_STRICT
3859	Log(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.lasterror.u32ExitReason,
3860	pVmxTransient->uExitReason));
3861	Log(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
3862	Log(("InstrError %#RX32\n", pVCpu->hm.s.vmx.lasterror.u32InstrError));
3863	if (pVCpu->hm.s.vmx.lasterror.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
3864	Log(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.lasterror.u32InstrError]));
3865	else
3866	Log(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
3867
3868	/* VMX control bits. */
3869	uint32_t u32Val;
3870	uint64_t u64Val;
3871	HMVMXHCUINTREG uHCReg;
3872	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
3873	Log(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
3874	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
3875	Log(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
3876	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
3877	Log(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
3878	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
3879	Log(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
3880	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
3881	Log(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
3882	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
3883	Log(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
3884	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
3885	Log(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
3886	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
3887	Log(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
3888	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
3889	Log(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
3890	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
3891	Log(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
3892	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
3893	Log(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
3894	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
3895	Log(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
3896	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
3897	Log(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
3898	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
3899	Log(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
3900	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
3901	Log(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
3902	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
3903	Log(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
3904	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
3905	Log(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
3906	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
3907	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
3908	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
3909	Log(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
3910	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
3911	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
3912	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
3913	Log(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
3914
3915	/* Guest bits. */
3916	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
3917	Log(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
3918	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
3919	Log(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
3920	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
3921	Log(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
3922	rc = VMXReadVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, &u32Val); AssertRC(rc);
3923	Log(("VMX_VMCS16_GUEST_FIELD_VPID %u\n", u32Val));
3924
3925	/* Host bits. */
3926	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
3927	Log(("Host CR0 %#RHr\n", uHCReg));
3928	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
3929	Log(("Host CR3 %#RHr\n", uHCReg));
3930	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
3931	Log(("Host CR4 %#RHr\n", uHCReg));
3932
3933	RTGDTR HostGdtr;
3934	PCX86DESCHC pDesc;
3935	ASMGetGDTR(&HostGdtr);
3936	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_CS, &u32Val); AssertRC(rc);
3937	Log(("Host CS %#08x\n", u32Val));
3938	if (u32Val < HostGdtr.cbGdt)
3939	{
3940	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3941	HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
3942	}
3943
3944	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_DS, &u32Val); AssertRC(rc);
3945	Log(("Host DS %#08x\n", u32Val));
3946	if (u32Val < HostGdtr.cbGdt)
3947	{
3948	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3949	HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
3950	}
3951
3952	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_ES, &u32Val); AssertRC(rc);
3953	Log(("Host ES %#08x\n", u32Val));
3954	if (u32Val < HostGdtr.cbGdt)
3955	{
3956	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3957	HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
3958	}
3959
3960	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_FS, &u32Val); AssertRC(rc);
3961	Log(("Host FS %#08x\n", u32Val));
3962	if (u32Val < HostGdtr.cbGdt)
3963	{
3964	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3965	HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
3966	}
3967
3968	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_GS, &u32Val); AssertRC(rc);
3969	Log(("Host GS %#08x\n", u32Val));
3970	if (u32Val < HostGdtr.cbGdt)
3971	{
3972	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3973	HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
3974	}
3975
3976	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_SS, &u32Val); AssertRC(rc);
3977	Log(("Host SS %#08x\n", u32Val));
3978	if (u32Val < HostGdtr.cbGdt)
3979	{
3980	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3981	HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
3982	}
3983
3984	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_TR, &u32Val); AssertRC(rc);
3985	Log(("Host TR %#08x\n", u32Val));
3986	if (u32Val < HostGdtr.cbGdt)
3987	{
3988	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3989	HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
3990	}
3991
3992	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
3993	Log(("Host TR Base %#RHv\n", uHCReg));
3994	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
3995	Log(("Host GDTR Base %#RHv\n", uHCReg));
3996	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
3997	Log(("Host IDTR Base %#RHv\n", uHCReg));
3998	rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
3999	Log(("Host SYSENTER CS %#08x\n", u32Val));
4000	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
4001	Log(("Host SYSENTER EIP %#RHv\n", uHCReg));
4002	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
4003	Log(("Host SYSENTER ESP %#RHv\n", uHCReg));
4004	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
4005	Log(("Host RSP %#RHv\n", uHCReg));
4006	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
4007	Log(("Host RIP %#RHv\n", uHCReg));
4008	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4009	if (HMVMX_IS_64BIT_HOST_MODE())
4010	{
4011	Log(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
4012	Log(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
4013	Log(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
4014	Log(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
4015	Log(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
4016	Log(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
4017	}
4018	# endif
4019	#endif /* VBOX_STRICT */
4020	break;
4021	}
4022
4023	default:
4024	/* Impossible */
4025	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
4026	break;
4027	}
4028	NOREF(pVM);
4029	}
4030
4031
4032	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4033	#ifndef VMX_USE_CACHED_VMCS_ACCESSES
4034	# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
4035	#endif
4036	#ifdef VBOX_STRICT
4037	static bool hmR0VmxIsValidWriteField(uint32_t idxField)
4038	{
4039	switch (idxField)
4040	{
4041	case VMX_VMCS_GUEST_RIP:
4042	case VMX_VMCS_GUEST_RSP:
4043	case VMX_VMCS_GUEST_SYSENTER_EIP:
4044	case VMX_VMCS_GUEST_SYSENTER_ESP:
4045	case VMX_VMCS_GUEST_GDTR_BASE:
4046	case VMX_VMCS_GUEST_IDTR_BASE:
4047	case VMX_VMCS_GUEST_CS_BASE:
4048	case VMX_VMCS_GUEST_DS_BASE:
4049	case VMX_VMCS_GUEST_ES_BASE:
4050	case VMX_VMCS_GUEST_FS_BASE:
4051	case VMX_VMCS_GUEST_GS_BASE:
4052	case VMX_VMCS_GUEST_SS_BASE:
4053	case VMX_VMCS_GUEST_LDTR_BASE:
4054	case VMX_VMCS_GUEST_TR_BASE:
4055	case VMX_VMCS_GUEST_CR3:
4056	return true;
4057	}
4058	return false;
4059	}
4060
4061	static bool hmR0VmxIsValidReadField(uint32_t idxField)
4062	{
4063	switch (idxField)
4064	{
4065	/* Read-only fields. */
4066	case VMX_VMCS_RO_EXIT_QUALIFICATION:
4067	return true;
4068	}
4069	/* Remaining readable fields should also be writable. */
4070	return hmR0VmxIsValidWriteField(idxField);
4071	}
4072	#endif /* VBOX_STRICT */
4073
4074
4075	/**
4076	* Executes the specified handler in 64-bit mode.
4077	*
4078	* @returns VBox status code.
4079	* @param pVM Pointer to the VM.
4080	* @param pVCpu Pointer to the VMCPU.
4081	* @param pCtx Pointer to the guest CPU context.
4082	* @param enmOp The operation to perform.
4083	* @param cbParam Number of parameters.
4084	* @param paParam Array of 32-bit parameters.
4085	*/
4086	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp, uint32_t cbParam,
4087	uint32_t *paParam)
4088	{
4089	int rc, rc2;
4090	PHMGLOBLCPUINFO pCpu;
4091	RTHCPHYS HCPhysCpuPage;
4092	RTCCUINTREG uOldEFlags;
4093
4094	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
4095	Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
4096	Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
4097	Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
4098
4099	#ifdef VBOX_STRICT
4100	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
4101	Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
4102
4103	for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
4104	Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
4105	#endif
4106
4107	/* Disable interrupts. */
4108	uOldEFlags = ASMIntDisableFlags();
4109
4110	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
4111	RTCPUID idHostCpu = RTMpCpuId();
4112	CPUMR0SetLApic(pVM, idHostCpu);
4113	#endif
4114
4115	pCpu = HMR0GetCurrentCpu();
4116	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4117
4118	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
4119	VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
4120
4121	/* Leave VMX Root Mode. */
4122	VMXDisable();
4123
4124	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4125
4126	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
4127	CPUMSetHyperEIP(pVCpu, enmOp);
4128	for (int i = (int)cbParam - 1; i >= 0; i--)
4129	CPUMPushHyper(pVCpu, paParam[i]);
4130
4131	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
4132
4133	/* Call the switcher. */
4134	rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
4135	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
4136
4137	/** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
4138	/* Make sure the VMX instructions don't cause #UD faults. */
4139	ASMSetCR4(ASMGetCR4() \| X86_CR4_VMXE);
4140
4141	/* Re-enter VMX Root Mode */
4142	rc2 = VMXEnable(HCPhysCpuPage);
4143	if (RT_FAILURE(rc2))
4144	{
4145	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4146	ASMSetFlags(uOldEFlags);
4147	return rc2;
4148	}
4149
4150	rc2 = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
4151	AssertRC(rc2);
4152	Assert(!(ASMGetFlags() & X86_EFL_IF));
4153	ASMSetFlags(uOldEFlags);
4154	return rc;
4155	}
4156
4157
4158	/**
4159	* Prepares for and executes VMLAUNCH (64 bits guests) for 32-bit hosts
4160	* supporting 64-bit guests.
4161	*
4162	* @returns VBox status code.
4163	* @param fResume Whether to VMLAUNCH or VMRESUME.
4164	* @param pCtx Pointer to the guest-CPU context.
4165	* @param pCache Pointer to the VMCS cache.
4166	* @param pVM Pointer to the VM.
4167	* @param pVCpu Pointer to the VMCPU.
4168	*/
4169	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
4170	{
4171	uint32_t aParam[6];
4172	PHMGLOBLCPUINFO pCpu = NULL;
4173	RTHCPHYS HCPhysCpuPage = 0;
4174	int rc = VERR_INTERNAL_ERROR_5;
4175
4176	pCpu = HMR0GetCurrentCpu();
4177	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4178
4179	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4180	pCache->uPos = 1;
4181	pCache->interPD = PGMGetInterPaeCR3(pVM);
4182	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
4183	#endif
4184
4185	#ifdef VBOX_STRICT
4186	pCache->TestIn.HCPhysCpuPage = 0;
4187	pCache->TestIn.HCPhysVmcs = 0;
4188	pCache->TestIn.pCache = 0;
4189	pCache->TestOut.HCPhysVmcs = 0;
4190	pCache->TestOut.pCache = 0;
4191	pCache->TestOut.pCtx = 0;
4192	pCache->TestOut.eflags = 0;
4193	#endif
4194
4195	aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
4196	aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
4197	aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
4198	aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
4199	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
4200	aParam[5] = 0;
4201
4202	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4203	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
4204	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
4205	#endif
4206	rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, 6, &aParam[0]);
4207
4208	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4209	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
4210	Assert(pCtx->dr[4] == 10);
4211	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
4212	#endif
4213
4214	#ifdef VBOX_STRICT
4215	AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
4216	AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4217	pVCpu->hm.s.vmx.HCPhysVmcs));
4218	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4219	pCache->TestOut.HCPhysVmcs));
4220	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
4221	pCache->TestOut.pCache));
4222	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
4223	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
4224	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
4225	pCache->TestOut.pCtx));
4226	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
4227	#endif
4228	return rc;
4229	}
4230
4231
4232	/**
4233	* Initialize the VMCS-Read cache. The VMCS cache is used for 32-bit hosts
4234	* running 64-bit guests (except 32-bit Darwin which runs with 64-bit paging in
4235	* 32-bit mode) for 64-bit fields that cannot be accessed in 32-bit mode. Some
4236	* 64-bit fields -can- be accessed (those that have a 32-bit FULL & HIGH part).
4237	*
4238	* @returns VBox status code.
4239	* @param pVM Pointer to the VM.
4240	* @param pVCpu Pointer to the VMCPU.
4241	*/
4242	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
4243	{
4244	#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
4245	{ \
4246	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
4247	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
4248	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
4249	++cReadFields; \
4250	}
4251
4252	AssertPtr(pVM);
4253	AssertPtr(pVCpu);
4254	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4255	uint32_t cReadFields = 0;
4256
4257	/*
4258	* Don't remove the #if 0'd fields in this code. They're listed here for consistency
4259	* and serve to indicate exceptions to the rules.
4260	*/
4261
4262	/* Guest-natural selector base fields. */
4263	#if 0
4264	/* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
4265	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
4266	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
4267	#endif
4268	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
4269	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
4270	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
4271	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
4272	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
4273	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
4274	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
4275	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
4276	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
4277	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
4278	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
4279	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
4280	#if 0
4281	/* Unused natural width guest-state fields. */
4282	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
4283	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
4284	#endif
4285	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
4286	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
4287
4288	/* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
4289	#if 0
4290	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
4291	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
4292	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
4293	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
4294	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
4295	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
4296	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
4297	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
4298	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
4299	#endif
4300
4301	/* Natural width guest-state fields. */
4302	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
4303	#if 0
4304	/* Currently unused field. */
4305	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
4306	#endif
4307
4308	if (pVM->hm.s.fNestedPaging)
4309	{
4310	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
4311	AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
4312	VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
4313	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
4314	}
4315	else
4316	{
4317	AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
4318	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
4319	}
4320
4321	#undef VMXLOCAL_INIT_READ_CACHE_FIELD
4322	return VINF_SUCCESS;
4323	}
4324
4325
4326	/**
4327	* Writes a field into the VMCS. This can either directly invoke a VMWRITE or
4328	* queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
4329	* darwin, running 64-bit guests).
4330	*
4331	* @returns VBox status code.
4332	* @param pVCpu Pointer to the VMCPU.
4333	* @param idxField The VMCS field encoding.
4334	* @param u64Val 16, 32 or 64 bits value.
4335	*/
4336	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4337	{
4338	int rc;
4339	switch (idxField)
4340	{
4341	/*
4342	* These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
4343	*/
4344	/* 64-bit Control fields. */
4345	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
4346	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
4347	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
4348	case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
4349	case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
4350	case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
4351	case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
4352	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
4353	case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
4354	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
4355	case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
4356	case VMX_VMCS64_CTRL_EPTP_FULL:
4357	case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
4358	/* 64-bit Guest-state fields. */
4359	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
4360	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
4361	case VMX_VMCS64_GUEST_PAT_FULL:
4362	case VMX_VMCS64_GUEST_EFER_FULL:
4363	case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
4364	case VMX_VMCS64_GUEST_PDPTE0_FULL:
4365	case VMX_VMCS64_GUEST_PDPTE1_FULL:
4366	case VMX_VMCS64_GUEST_PDPTE2_FULL:
4367	case VMX_VMCS64_GUEST_PDPTE3_FULL:
4368	/* 64-bit Host-state fields. */
4369	case VMX_VMCS64_HOST_FIELD_PAT_FULL:
4370	case VMX_VMCS64_HOST_FIELD_EFER_FULL:
4371	case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
4372	{
4373	rc = VMXWriteVmcs32(idxField, u64Val);
4374	rc \|= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
4375	break;
4376	}
4377
4378	/*
4379	* These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
4380	* values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
4381	*/
4382	/* Natural-width Guest-state fields. */
4383	case VMX_VMCS_GUEST_CR3:
4384	case VMX_VMCS_GUEST_ES_BASE:
4385	case VMX_VMCS_GUEST_CS_BASE:
4386	case VMX_VMCS_GUEST_SS_BASE:
4387	case VMX_VMCS_GUEST_DS_BASE:
4388	case VMX_VMCS_GUEST_FS_BASE:
4389	case VMX_VMCS_GUEST_GS_BASE:
4390	case VMX_VMCS_GUEST_LDTR_BASE:
4391	case VMX_VMCS_GUEST_TR_BASE:
4392	case VMX_VMCS_GUEST_GDTR_BASE:
4393	case VMX_VMCS_GUEST_IDTR_BASE:
4394	case VMX_VMCS_GUEST_RSP:
4395	case VMX_VMCS_GUEST_RIP:
4396	case VMX_VMCS_GUEST_SYSENTER_ESP:
4397	case VMX_VMCS_GUEST_SYSENTER_EIP:
4398	{
4399	if (!(u64Val >> 32))
4400	{
4401	/* If this field is 64-bit, VT-x will zero out the top bits. */
4402	rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
4403	}
4404	else
4405	{
4406	/* Assert that only the 32->64 switcher case should ever come here. */
4407	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
4408	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
4409	}
4410	break;
4411	}
4412
4413	default:
4414	{
4415	AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
4416	rc = VERR_INVALID_PARAMETER;
4417	break;
4418	}
4419	}
4420	AssertRCReturn(rc, rc);
4421	return rc;
4422	}
4423
4424
4425	/**
4426	* Queue up a VMWRITE by using the VMCS write cache. This is only used on 32-bit
4427	* hosts (except darwin) for 64-bit guests.
4428	*
4429	* @param pVCpu Pointer to the VMCPU.
4430	* @param idxField The VMCS field encoding.
4431	* @param u64Val 16, 32 or 64 bits value.
4432	*/
4433	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4434	{
4435	AssertPtr(pVCpu);
4436	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4437
4438	AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
4439	("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
4440
4441	/* Make sure there are no duplicates. */
4442	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
4443	{
4444	if (pCache->Write.aField[i] == idxField)
4445	{
4446	pCache->Write.aFieldVal[i] = u64Val;
4447	return VINF_SUCCESS;
4448	}
4449	}
4450
4451	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
4452	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
4453	pCache->Write.cValidEntries++;
4454	return VINF_SUCCESS;
4455	}
4456
4457	/* Enable later when the assembly code uses these as callbacks. */
4458	#if 0
4459	/*
4460	* Loads the VMCS write-cache into the CPU (by executing VMWRITEs).
4461	*
4462	* @param pVCpu Pointer to the VMCPU.
4463	* @param pCache Pointer to the VMCS cache.
4464	*
4465	* @remarks No-long-jump zone!!!
4466	*/
4467	VMMR0DECL(void) VMXWriteCachedVmcsLoad(PVMCPU pVCpu, PVMCSCACHE pCache)
4468	{
4469	AssertPtr(pCache);
4470	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
4471	{
4472	int rc = VMXWriteVmcs64(pCache->Write.aField[i], pCache->Write.aFieldVal[i]);
4473	AssertRC(rc);
4474	}
4475	pCache->Write.cValidEntries = 0;
4476	}
4477
4478
4479	/**
4480	* Stores the VMCS read-cache from the CPU (by executing VMREADs).
4481	*
4482	* @param pVCpu Pointer to the VMCPU.
4483	* @param pCache Pointer to the VMCS cache.
4484	*
4485	* @remarks No-long-jump zone!!!
4486	*/
4487	VMMR0DECL(void) VMXReadCachedVmcsStore(PVMCPU pVCpu, PVMCSCACHE pCache)
4488	{
4489	AssertPtr(pCache);
4490	for (uint32_t i = 0; i < pCache->Read.cValidEntries; i++)
4491	{
4492	int rc = VMXReadVmcs64(pCache->Read.aField[i], &pCache->Read.aFieldVal[i]);
4493	AssertRC(rc);
4494	}
4495	}
4496	#endif
4497	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
4498
4499
4500	/**
4501	* Sets up the usage of TSC-offsetting and updates the VMCS. If offsetting is
4502	* not possible, cause VM-exits on RDTSC(P)s. Also sets up the VMX preemption
4503	* timer.
4504	*
4505	* @returns VBox status code.
4506	* @param pVCpu Pointer to the VMCPU.
4507	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4508	* out-of-sync. Make sure to update the required fields
4509	* before using them.
4510	* @remarks No-long-jump zone!!!
4511	*/
4512	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4513	{
4514	int rc = VERR_INTERNAL_ERROR_5;
4515	bool fOffsettedTsc = false;
4516	PVM pVM = pVCpu->CTX_SUFF(pVM);
4517	if (pVM->hm.s.vmx.fUsePreemptTimer)
4518	{
4519	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &pVCpu->hm.s.vmx.u64TSCOffset);
4520
4521	/* Make sure the returned values have sane upper and lower boundaries. */
4522	uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
4523	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
4524	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
4525	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
4526
4527	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
4528	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
4529	}
4530	else
4531	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset);
4532
4533	if (fOffsettedTsc)
4534	{
4535	uint64_t u64CurTSC = ASMReadTSC();
4536	if (u64CurTSC + pVCpu->hm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
4537	{
4538	/* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
4539	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
4540
4541	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
4542	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4543	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
4544	}
4545	else
4546	{
4547	/* VM-exit on RDTSC(P) as we would otherwise pass decreasing TSC values to the guest. */
4548	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
4549	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4550	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscInterceptOverFlow);
4551	}
4552	}
4553	else
4554	{
4555	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
4556	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
4557	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4558	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
4559	}
4560	}
4561
4562
4563	/**
4564	* Determines if an exception is a contributory exception. Contributory
4565	* exceptions are ones which can cause double-faults. Page-fault is
4566	* intentionally not included here as it's a conditional contributory exception.
4567	*
4568	* @returns true if the exception is contributory, false otherwise.
4569	* @param uVector The exception vector.
4570	*/
4571	DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
4572	{
4573	switch (uVector)
4574	{
4575	case X86_XCPT_GP:
4576	case X86_XCPT_SS:
4577	case X86_XCPT_NP:
4578	case X86_XCPT_TS:
4579	case X86_XCPT_DE:
4580	return true;
4581	default:
4582	break;
4583	}
4584	return false;
4585	}
4586
4587
4588	/**
4589	* Sets an event as a pending event to be injected into the guest.
4590	*
4591	* @param pVCpu Pointer to the VMCPU.
4592	* @param u32IntrInfo The VM-entry interruption-information field.
4593	* @param cbInstr The VM-entry instruction length in bytes (for software
4594	* interrupts, exceptions and privileged software
4595	* exceptions).
4596	* @param u32ErrCode The VM-entry exception error code.
4597	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
4598	* page-fault.
4599	*/
4600	DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntrInfo, uint32_t cbInstr, uint32_t u32ErrCode,
4601	RTGCUINTPTR GCPtrFaultAddress)
4602	{
4603	Assert(!pVCpu->hm.s.Event.fPending);
4604	pVCpu->hm.s.Event.fPending = true;
4605	pVCpu->hm.s.Event.u64IntrInfo = u32IntrInfo;
4606	pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
4607	pVCpu->hm.s.Event.cbInstr = cbInstr;
4608	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
4609	}
4610
4611
4612	/**
4613	* Sets a double-fault (#DF) exception as pending-for-injection into the VM.
4614	*
4615	* @param pVCpu Pointer to the VMCPU.
4616	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4617	* out-of-sync. Make sure to update the required fields
4618	* before using them.
4619	*/
4620	DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4621	{
4622	uint32_t u32IntrInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
4623	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
4624	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
4625	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
4626	}
4627
4628
4629	/**
4630	* Handle a condition that occurred while delivering an event through the guest
4631	* IDT.
4632	*
4633	* @returns VBox status code (informational error codes included).
4634	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
4635	* @retval VINF_VMX_DOUBLE_FAULT if a #DF condition was detected and we ought to
4636	* continue execution of the guest which will delivery the #DF.
4637	* @retval VINF_EM_RESET if we detected a triple-fault condition.
4638	*
4639	* @param pVCpu Pointer to the VMCPU.
4640	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4641	* out-of-sync. Make sure to update the required fields
4642	* before using them.
4643	* @param pVmxTransient Pointer to the VMX transient structure.
4644	*
4645	* @remarks No-long-jump zone!!!
4646	*/
4647	static int hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
4648	{
4649	int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
4650	AssertRC(rc);
4651	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
4652	{
4653	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
4654	AssertRCReturn(rc, rc);
4655
4656	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
4657	uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntrInfo);
4658	uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
4659
4660	typedef enum
4661	{
4662	VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
4663	VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
4664	VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
4665	VMXREFLECTXCPT_NONE /* Nothing to reflect. */
4666	} VMXREFLECTXCPT;
4667
4668	/* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
4669	VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
4670	if (uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
4671	{
4672	enmReflect = VMXREFLECTXCPT_XCPT;
4673	#ifdef VBOX_STRICT
4674	if ( hmR0VmxIsContributoryXcpt(uIdtVector)
4675	&& uExitVector == X86_XCPT_PF)
4676	{
4677	Log(("IDT: Contributory #PF uCR2=%#RX64\n", pMixedCtx->cr2));
4678	}
4679	#endif
4680	if ( uExitVector == X86_XCPT_PF
4681	&& uIdtVector == X86_XCPT_PF)
4682	{
4683	pVmxTransient->fVectoringPF = true;
4684	Log(("IDT: Vectoring #PF uCR2=%#RX64\n", pMixedCtx->cr2));
4685	}
4686	else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
4687	&& hmR0VmxIsContributoryXcpt(uExitVector)
4688	&& ( hmR0VmxIsContributoryXcpt(uIdtVector)
4689	\|\| uIdtVector == X86_XCPT_PF))
4690	{
4691	enmReflect = VMXREFLECTXCPT_DF;
4692	}
4693	else if (uIdtVector == X86_XCPT_DF)
4694	enmReflect = VMXREFLECTXCPT_TF;
4695	}
4696	else if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
4697	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
4698	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
4699	{
4700	/*
4701	* Ignore software interrupts (INT n), software exceptions (#BP, #OF) and privileged software exception
4702	* (whatever they are) as they reoccur when restarting the instruction.
4703	*/
4704	enmReflect = VMXREFLECTXCPT_XCPT;
4705	}
4706
4707	switch (enmReflect)
4708	{
4709	case VMXREFLECTXCPT_XCPT:
4710	{
4711	uint32_t u32ErrCode = 0;
4712	if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntrInfo))
4713	{
4714	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
4715	AssertRCReturn(rc, rc);
4716	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
4717	}
4718
4719	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
4720	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INTR_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
4721	0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
4722	rc = VINF_SUCCESS;
4723	Log(("IDT: Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->hm.s.Event.u64IntrInfo,
4724	pVCpu->hm.s.Event.u32ErrCode));
4725	break;
4726	}
4727
4728	case VMXREFLECTXCPT_DF:
4729	{
4730	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
4731	rc = VINF_VMX_DOUBLE_FAULT;
4732	Log(("IDT: Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->hm.s.Event.u64IntrInfo,
4733	uIdtVector, uExitVector));
4734	break;
4735	}
4736
4737	case VMXREFLECTXCPT_TF:
4738	{
4739	Log(("IDT: Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", uIdtVector, uExitVector));
4740	rc = VINF_EM_RESET;
4741	break;
4742	}
4743
4744	default:
4745	Assert(rc == VINF_SUCCESS);
4746	break;
4747	}
4748	}
4749	Assert(rc == VINF_SUCCESS \|\| rc == VINF_VMX_DOUBLE_FAULT \|\| rc == VINF_EM_RESET);
4750	return rc;
4751	}
4752
4753
4754	/**
4755	* Saves the guest's CR0 register from the VMCS into the guest-CPU context.
4756	*
4757	* @returns VBox status code.
4758	* @param pVCpu Pointer to the VMCPU.
4759	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4760	* out-of-sync. Make sure to update the required fields
4761	* before using them.
4762	*
4763	* @remarks No-long-jump zone!!!
4764	*/
4765	static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4766	{
4767	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0))
4768	{
4769	uint32_t uVal = 0;
4770	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
4771	AssertRCReturn(rc, rc);
4772	uint32_t uShadow = 0;
4773	rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
4774	AssertRCReturn(rc, rc);
4775
4776	uVal = (uShadow & pVCpu->hm.s.vmx.cr0_mask) \| (uVal & ~pVCpu->hm.s.vmx.cr0_mask);
4777	CPUMSetGuestCR0(pVCpu, uVal);
4778	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR0;
4779	}
4780	return VINF_SUCCESS;
4781	}
4782
4783
4784	/**
4785	* Saves the guest's CR4 register from the VMCS into the guest-CPU context.
4786	*
4787	* @returns VBox status code.
4788	* @param pVCpu Pointer to the VMCPU.
4789	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4790	* out-of-sync. Make sure to update the required fields
4791	* before using them.
4792	*
4793	* @remarks No-long-jump zone!!!
4794	*/
4795	static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4796	{
4797	int rc = VINF_SUCCESS;
4798	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR4))
4799	{
4800	uint32_t uVal = 0;
4801	uint32_t uShadow = 0;
4802	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
4803	AssertRCReturn(rc, rc);
4804	rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
4805	AssertRCReturn(rc, rc);
4806
4807	uVal = (uShadow & pVCpu->hm.s.vmx.cr4_mask) \| (uVal & ~pVCpu->hm.s.vmx.cr4_mask);
4808	CPUMSetGuestCR4(pVCpu, uVal);
4809	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR4;
4810	}
4811	return rc;
4812	}
4813
4814
4815	/**
4816	* Saves the guest's RIP register from the VMCS into the guest-CPU context.
4817	*
4818	* @returns VBox status code.
4819	* @param pVCpu Pointer to the VMCPU.
4820	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4821	* out-of-sync. Make sure to update the required fields
4822	* before using them.
4823	*
4824	* @remarks No-long-jump zone!!!
4825	*/
4826	static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4827	{
4828	int rc = VINF_SUCCESS;
4829	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RIP))
4830	{
4831	uint64_t u64Val = 0;
4832	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
4833	AssertRCReturn(rc, rc);
4834
4835	pMixedCtx->rip = u64Val;
4836	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RIP;
4837	}
4838	return rc;
4839	}
4840
4841
4842	/**
4843	* Saves the guest's RSP register from the VMCS into the guest-CPU context.
4844	*
4845	* @returns VBox status code.
4846	* @param pVCpu Pointer to the VMCPU.
4847	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4848	* out-of-sync. Make sure to update the required fields
4849	* before using them.
4850	*
4851	* @remarks No-long-jump zone!!!
4852	*/
4853	static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4854	{
4855	int rc = VINF_SUCCESS;
4856	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RSP))
4857	{
4858	uint64_t u64Val = 0;
4859	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
4860	AssertRCReturn(rc, rc);
4861
4862	pMixedCtx->rsp = u64Val;
4863	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RSP;
4864	}
4865	return rc;
4866	}
4867
4868
4869	/**
4870	* Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
4871	*
4872	* @returns VBox status code.
4873	* @param pVCpu Pointer to the VMCPU.
4874	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4875	* out-of-sync. Make sure to update the required fields
4876	* before using them.
4877	*
4878	* @remarks No-long-jump zone!!!
4879	*/
4880	static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4881	{
4882	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS))
4883	{
4884	uint32_t uVal = 0;
4885	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
4886	AssertRCReturn(rc, rc);
4887
4888	pMixedCtx->eflags.u32 = uVal;
4889	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
4890	{
4891	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4892	Log(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
4893
4894	pMixedCtx->eflags.Bits.u1VM = 0;
4895	pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.eflags.Bits.u2IOPL;
4896	}
4897
4898	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RFLAGS;
4899	}
4900	return VINF_SUCCESS;
4901	}
4902
4903
4904	/**
4905	* Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
4906	* guest-CPU context.
4907	*/
4908	DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4909	{
4910	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
4911	rc \|= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
4912	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
4913	return rc;
4914	}
4915
4916
4917	/**
4918	* Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
4919	* from the guest-state area in the VMCS.
4920	*
4921	* @param pVCpu Pointer to the VMCPU.
4922	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4923	* out-of-sync. Make sure to update the required fields
4924	* before using them.
4925	*
4926	* @remarks No-long-jump zone!!!
4927	*/
4928	static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4929	{
4930	uint32_t uIntrState = 0;
4931	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
4932	AssertRC(rc);
4933
4934	if (!uIntrState)
4935	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
4936	else
4937	{
4938	Assert( uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
4939	\|\| uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
4940	rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
4941	AssertRC(rc);
4942	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
4943	AssertRC(rc);
4944
4945	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
4946	Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
4947	}
4948	}
4949
4950
4951	/**
4952	* Saves the guest's activity state.
4953	*
4954	* @returns VBox status code.
4955	* @param pVCpu Pointer to the VMCPU.
4956	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4957	* out-of-sync. Make sure to update the required fields
4958	* before using them.
4959	*
4960	* @remarks No-long-jump zone!!!
4961	*/
4962	static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4963	{
4964	/* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
4965	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_ACTIVITY_STATE;
4966	return VINF_SUCCESS;
4967	}
4968
4969
4970	/**
4971	* Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
4972	* the current VMCS into the guest-CPU context.
4973	*
4974	* @returns VBox status code.
4975	* @param pVCpu Pointer to the VMCPU.
4976	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4977	* out-of-sync. Make sure to update the required fields
4978	* before using them.
4979	*
4980	* @remarks No-long-jump zone!!!
4981	*/
4982	static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4983	{
4984	int rc = VINF_SUCCESS;
4985	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
4986	{
4987	uint32_t u32Val = 0;
4988	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
4989	pMixedCtx->SysEnter.cs = u32Val;
4990	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR;
4991	}
4992
4993	uint64_t u64Val = 0;
4994	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
4995	{
4996	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
4997	pMixedCtx->SysEnter.eip = u64Val;
4998	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR;
4999	}
5000	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
5001	{
5002	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
5003	pMixedCtx->SysEnter.esp = u64Val;
5004	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR;
5005	}
5006	return rc;
5007	}
5008
5009
5010	/**
5011	* Saves the guest FS_BASE MSRs from the current VMCS into the guest-CPU
5012	* context.
5013	*
5014	* @returns VBox status code.
5015	* @param pVCpu Pointer to the VMCPU.
5016	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5017	* out-of-sync. Make sure to update the required fields
5018	* before using them.
5019	*
5020	* @remarks No-long-jump zone!!!
5021	*/
5022	static int hmR0VmxSaveGuestFSBaseMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5023	{
5024	int rc = VINF_SUCCESS;
5025	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_FS_BASE_MSR))
5026	{
5027	uint64_t u64Val = 0;
5028	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_FS_BASE, &u64Val); AssertRCReturn(rc, rc);
5029	pMixedCtx->fs.u64Base = u64Val;
5030	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_FS_BASE_MSR;
5031	}
5032	return rc;
5033	}
5034
5035
5036	/**
5037	* Saves the guest GS_BASE MSRs from the current VMCS into the guest-CPU
5038	* context.
5039	*
5040	* @returns VBox status code.
5041	* @param pVCpu Pointer to the VMCPU.
5042	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5043	* out-of-sync. Make sure to update the required fields
5044	* before using them.
5045	*
5046	* @remarks No-long-jump zone!!!
5047	*/
5048	static int hmR0VmxSaveGuestGSBaseMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5049	{
5050	int rc = VINF_SUCCESS;
5051	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_GS_BASE_MSR))
5052	{
5053	uint64_t u64Val = 0;
5054	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GS_BASE, &u64Val); AssertRCReturn(rc, rc);
5055	pMixedCtx->gs.u64Base = u64Val;
5056	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_GS_BASE_MSR;
5057	}
5058	return rc;
5059	}
5060
5061
5062	/**
5063	* Saves the auto load/store'd guest MSRs from the current VMCS into the
5064	* guest-CPU context. Currently these are LSTAR, STAR, SFMASK, KERNEL-GS BASE
5065	* and TSC_AUX.
5066	*
5067	* @returns VBox status code.
5068	* @param pVCpu Pointer to the VMCPU.
5069	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5070	* out-of-sync. Make sure to update the required fields
5071	* before using them.
5072	*
5073	* @remarks No-long-jump zone!!!
5074	*/
5075	static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5076	{
5077	if (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS)
5078	return VINF_SUCCESS;
5079
5080	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
5081	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cGuestMsrs; i++)
5082	{
5083	PVMXMSR pMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvGuestMsr;
5084	pMsr += i;
5085	switch (pMsr->u32IndexMSR)
5086	{
5087	case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
5088	case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
5089	case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
5090	case MSR_K8_TSC_AUX: CPUMSetGuestMsr(pVCpu, MSR_K8_TSC_AUX, pMsr->u64Value); break;
5091	case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
5092	case MSR_K6_EFER: /* EFER can't be changed without causing a VM-exit. */ break;
5093	default:
5094	{
5095	AssertFailed();
5096	return VERR_HM_UNEXPECTED_LD_ST_MSR;
5097	}
5098	}
5099	}
5100	#endif
5101
5102	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS;
5103	return VINF_SUCCESS;
5104	}
5105
5106
5107	/**
5108	* Saves the guest control registers from the current VMCS into the guest-CPU
5109	* context.
5110	*
5111	* @returns VBox status code.
5112	* @param pVCpu Pointer to the VMCPU.
5113	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5114	* out-of-sync. Make sure to update the required fields
5115	* before using them.
5116	*
5117	* @remarks No-long-jump zone!!!
5118	*/
5119	static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5120	{
5121	/* Guest CR0. Guest FPU. */
5122	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5123	AssertRCReturn(rc, rc);
5124
5125	/* Guest CR4. */
5126	rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
5127	AssertRCReturn(rc, rc);
5128
5129	/* Guest CR2 - updated always during the world-switch or in #PF. */
5130	/* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
5131	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR3))
5132	{
5133	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0);
5134	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR4);
5135
5136	PVM pVM = pVCpu->CTX_SUFF(pVM);
5137	if ( pVM->hm.s.vmx.fUnrestrictedGuest
5138	\|\| ( pVM->hm.s.fNestedPaging
5139	&& CPUMIsGuestPagingEnabledEx(pMixedCtx)))
5140	{
5141	uint64_t u64Val = 0;
5142	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
5143	if (pMixedCtx->cr3 != u64Val)
5144	{
5145	CPUMSetGuestCR3(pVCpu, u64Val);
5146	if (VMMRZCallRing3IsEnabled(pVCpu))
5147	{
5148	PGMUpdateCR3(pVCpu, u64Val);
5149	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5150	}
5151	else
5152	{
5153	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
5154	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
5155	}
5156	}
5157
5158	/* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
5159	if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
5160	{
5161	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
5162	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
5163	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
5164	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
5165
5166	if (VMMRZCallRing3IsEnabled(pVCpu))
5167	{
5168	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5169	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5170	}
5171	else
5172	{
5173	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
5174	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
5175	}
5176	}
5177	}
5178
5179	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR3;
5180	}
5181
5182	/*
5183	* Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
5184	* -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
5185	* -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
5186	*
5187	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
5188	*/
5189	if (VMMRZCallRing3IsEnabled(pVCpu))
5190	{
5191	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
5192	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
5193
5194	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
5195	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5196
5197	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5198	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5199	}
5200
5201	return rc;
5202	}
5203
5204
5205	/**
5206	* Reads a guest segment register from the current VMCS into the guest-CPU
5207	* context.
5208	*
5209	* @returns VBox status code.
5210	* @param pVCpu Pointer to the VMCPU.
5211	* @param idxSel Index of the selector in the VMCS.
5212	* @param idxLimit Index of the segment limit in the VMCS.
5213	* @param idxBase Index of the segment base in the VMCS.
5214	* @param idxAccess Index of the access rights of the segment in the VMCS.
5215	* @param pSelReg Pointer to the segment selector.
5216	*
5217	* @remarks No-long-jump zone!!!
5218	* @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
5219	* macro as that takes care of whether to read from the VMCS cache or
5220	* not.
5221	*/
5222	DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
5223	PCPUMSELREG pSelReg)
5224	{
5225	uint32_t u32Val = 0;
5226	int rc = VMXReadVmcs32(idxSel, &u32Val);
5227	AssertRCReturn(rc, rc);
5228	pSelReg->Sel = (uint16_t)u32Val;
5229	pSelReg->ValidSel = (uint16_t)u32Val;
5230	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
5231
5232	rc = VMXReadVmcs32(idxLimit, &u32Val);
5233	AssertRCReturn(rc, rc);
5234	pSelReg->u32Limit = u32Val;
5235
5236	uint64_t u64Val = 0;
5237	rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
5238	AssertRCReturn(rc, rc);
5239	pSelReg->u64Base = u64Val;
5240
5241	rc = VMXReadVmcs32(idxAccess, &u32Val);
5242	AssertRCReturn(rc, rc);
5243	pSelReg->Attr.u = u32Val;
5244
5245	/*
5246	* If VT-x marks the segment as unusable, the rest of the attributes are undefined with certain exceptions (some bits in
5247	* CS, SS). Regardless, we have to clear the bits here and only retain the unusable bit because the unusable bit is specific
5248	* to VT-x, everyone else relies on the attribute being zero and have no clue what the unusable bit is.
5249	*
5250	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
5251	*/
5252	if (pSelReg->Attr.u & HMVMX_SEL_UNUSABLE)
5253	{
5254	Assert(idxSel != VMX_VMCS16_GUEST_FIELD_TR); /* TR is the only selector that can never be unusable. */
5255	pSelReg->Attr.u = HMVMX_SEL_UNUSABLE;
5256	}
5257	return VINF_SUCCESS;
5258	}
5259
5260
5261	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
5262	#define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5263	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5264	VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5265	#else
5266	#define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5267	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5268	VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5269	#endif
5270
5271
5272	/**
5273	* Saves the guest segment registers from the current VMCS into the guest-CPU
5274	* context.
5275	*
5276	* @returns VBox status code.
5277	* @param pVCpu Pointer to the VMCPU.
5278	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5279	* out-of-sync. Make sure to update the required fields
5280	* before using them.
5281	*
5282	* @remarks No-long-jump zone!!!
5283	*/
5284	static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5285	{
5286	/* Guest segment registers. */
5287	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SEGMENT_REGS))
5288	{
5289	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); AssertRCReturn(rc, rc);
5290	rc = VMXLOCAL_READ_SEG(CS, cs); AssertRCReturn(rc, rc);
5291	rc = VMXLOCAL_READ_SEG(SS, ss); AssertRCReturn(rc, rc);
5292	rc = VMXLOCAL_READ_SEG(DS, ds); AssertRCReturn(rc, rc);
5293	rc = VMXLOCAL_READ_SEG(ES, es); AssertRCReturn(rc, rc);
5294	rc = VMXLOCAL_READ_SEG(FS, fs); AssertRCReturn(rc, rc);
5295	rc = VMXLOCAL_READ_SEG(GS, gs); AssertRCReturn(rc, rc);
5296
5297	/* Restore segment attributes for real-on-v86 mode hack. */
5298	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5299	{
5300	pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrCS.u;
5301	pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrSS.u;
5302	pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrDS.u;
5303	pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrES.u;
5304	pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrFS.u;
5305	pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrGS.u;
5306	}
5307	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SEGMENT_REGS;
5308	}
5309
5310	return VINF_SUCCESS;
5311	}
5312
5313
5314	/**
5315	* Saves the guest descriptor table registers and task register from the current
5316	* VMCS into the guest-CPU context.
5317	*
5318	* @returns VBox status code.
5319	* @param pVCpu Pointer to the VMCPU.
5320	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5321	* out-of-sync. Make sure to update the required fields
5322	* before using them.
5323	*
5324	* @remarks No-long-jump zone!!!
5325	*/
5326	static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5327	{
5328	int rc = VINF_SUCCESS;
5329
5330	/* Guest LDTR. */
5331	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_LDTR))
5332	{
5333	rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
5334	AssertRCReturn(rc, rc);
5335	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_LDTR;
5336	}
5337
5338	/* Guest GDTR. */
5339	uint64_t u64Val = 0;
5340	uint32_t u32Val = 0;
5341	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_GDTR))
5342	{
5343	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
5344	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5345	pMixedCtx->gdtr.pGdt = u64Val;
5346	pMixedCtx->gdtr.cbGdt = u32Val;
5347	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_GDTR;
5348	}
5349
5350	/* Guest IDTR. */
5351	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_IDTR))
5352	{
5353	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
5354	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5355	pMixedCtx->idtr.pIdt = u64Val;
5356	pMixedCtx->idtr.cbIdt = u32Val;
5357	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_IDTR;
5358	}
5359
5360	/* Guest TR. */
5361	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_TR))
5362	{
5363	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5364	AssertRCReturn(rc, rc);
5365
5366	/* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
5367	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5368	{
5369	rc = VMXLOCAL_READ_SEG(TR, tr);
5370	AssertRCReturn(rc, rc);
5371	}
5372	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_TR;
5373	}
5374	return rc;
5375	}
5376
5377	#undef VMXLOCAL_READ_SEG
5378
5379
5380	/**
5381	* Saves the guest debug registers from the current VMCS into the guest-CPU
5382	* context.
5383	*
5384	* @returns VBox status code.
5385	* @param pVCpu Pointer to the VMCPU.
5386	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5387	* out-of-sync. Make sure to update the required fields
5388	* before using them.
5389	*
5390	* @remarks No-long-jump zone!!!
5391	*/
5392	static int hmR0VmxSaveGuestDebugRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5393	{
5394	int rc = VINF_SUCCESS;
5395	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_DEBUG))
5396	{
5397	/* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
5398	uint32_t u32Val;
5399	rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
5400	pMixedCtx->dr[7] = u32Val;
5401
5402	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_DEBUG;
5403	}
5404	return rc;
5405	}
5406
5407
5408	/**
5409	* Saves the guest APIC state from the currentl VMCS into the guest-CPU context.
5410	*
5411	* @returns VBox status code.
5412	* @param pVCpu Pointer to the VMCPU.
5413	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5414	* out-of-sync. Make sure to update the required fields
5415	* before using them.
5416	*
5417	* @remarks No-long-jump zone!!!
5418	*/
5419	static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5420	{
5421	/* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
5422	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_APIC_STATE;
5423	return VINF_SUCCESS;
5424	}
5425
5426
5427	/**
5428	* Saves the entire guest state from the currently active VMCS into the
5429	* guest-CPU context. This essentially VMREADs all guest-data.
5430	*
5431	* @returns VBox status code.
5432	* @param pVCpu Pointer to the VMCPU.
5433	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5434	* out-of-sync. Make sure to update the required fields
5435	* before using them.
5436	*/
5437	static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5438	{
5439	Assert(pVCpu);
5440	Assert(pMixedCtx);
5441
5442	if (pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL)
5443	return VINF_SUCCESS;
5444
5445	/* Though we can longjmp to ring-3 due to log-flushes here and get recalled again on the ring-3 callback path,
5446	there is no real need to. */
5447	if (VMMRZCallRing3IsEnabled(pVCpu))
5448	VMMR0LogFlushDisable(pVCpu);
5449	else
5450	Assert(VMMR0IsLogFlushDisabled(pVCpu));
5451	LogFunc(("\n"));
5452
5453	int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
5454	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5455
5456	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
5457	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5458
5459	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
5460	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5461
5462	rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
5463	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5464
5465	rc = hmR0VmxSaveGuestDebugRegs(pVCpu, pMixedCtx);
5466	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDebugRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5467
5468	rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
5469	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5470
5471	rc = hmR0VmxSaveGuestFSBaseMsr(pVCpu, pMixedCtx);
5472	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestFSBaseMsr failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5473
5474	rc = hmR0VmxSaveGuestGSBaseMsr(pVCpu, pMixedCtx);
5475	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestGSBaseMsr failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5476
5477	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
5478	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5479
5480	rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
5481	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5482
5483	rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
5484	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDebugRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5485
5486	AssertMsg(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL,
5487	("Missed guest state bits while saving state; residue %RX32\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
5488
5489	if (VMMRZCallRing3IsEnabled(pVCpu))
5490	VMMR0LogFlushEnable(pVCpu);
5491
5492	return rc;
5493	}
5494
5495
5496	/**
5497	* Check per-VM and per-VCPU force flag actions that require us to go back to
5498	* ring-3 for one reason or another.
5499	*
5500	* @returns VBox status code (information status code included).
5501	* @retval VINF_SUCCESS if we don't have any actions that require going back to
5502	* ring-3.
5503	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
5504	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
5505	* interrupts)
5506	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
5507	* all EMTs to be in ring-3.
5508	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
5509	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
5510	* to the EM loop.
5511	*
5512	* @param pVM Pointer to the VM.
5513	* @param pVCpu Pointer to the VMCPU.
5514	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5515	* out-of-sync. Make sure to update the required fields
5516	* before using them.
5517	*/
5518	static int hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5519	{
5520	Assert(VMMRZCallRing3IsEnabled(pVCpu));
5521
5522	int rc = VERR_INTERNAL_ERROR_5;
5523	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK \| VM_FF_REQUEST \| VM_FF_PGM_POOL_FLUSH_PENDING \| VM_FF_PDM_DMA)
5524	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK \| VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL
5525	\| VMCPU_FF_REQUEST \| VMCPU_FF_HM_UPDATE_CR3 \| VMCPU_FF_HM_UPDATE_PAE_PDPES))
5526	{
5527	/* We need the control registers now, make sure the guest-CPU context is updated. */
5528	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
5529	AssertRCReturn(rc, rc);
5530
5531	/* Pending HM CR3 sync. */
5532	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
5533	{
5534	rc = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
5535	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3);
5536	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5537	}
5538
5539	/* Pending HM PAE PDPEs. */
5540	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
5541	{
5542	rc = PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5543	AssertRC(rc);
5544	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5545	}
5546
5547	/* Pending PGM C3 sync. */
5548	if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
5549	{
5550	rc = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4, VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
5551	if (rc != VINF_SUCCESS)
5552	{
5553	AssertRC(rc);
5554	Log(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc=%d\n", rc));
5555	return rc;
5556	}
5557	}
5558
5559	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
5560	/* -XXX- what was that about single stepping? */
5561	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
5562	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
5563	{
5564	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
5565	rc = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
5566	Log(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc));
5567	return rc;
5568	}
5569
5570	/* Pending VM request packets, such as hardware interrupts. */
5571	if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
5572	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
5573	{
5574	Log(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
5575	return VINF_EM_PENDING_REQUEST;
5576	}
5577
5578	/* Pending PGM pool flushes. */
5579	if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
5580	{
5581	Log(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
5582	return VINF_PGM_POOL_FLUSH_PENDING;
5583	}
5584
5585	/* Pending DMA requests. */
5586	if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
5587	{
5588	Log(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
5589	return VINF_EM_RAW_TO_R3;
5590	}
5591	}
5592
5593	/* Paranoia. */
5594	Assert(rc != VERR_EM_INTERPRETER);
5595	return VINF_SUCCESS;
5596	}
5597
5598
5599	/**
5600	* Converts any TRPM trap into a pending VMX event. This is typically used when
5601	* entering from ring-3 (not longjmp returns).
5602	*
5603	* @param pVCpu Pointer to the VMCPU.
5604	*/
5605	static void hmR0VmxTRPMTrapToPendingEvent(PVMCPU pVCpu)
5606	{
5607	Assert(TRPMHasTrap(pVCpu));
5608	Assert(!pVCpu->hm.s.Event.fPending);
5609
5610	uint8_t uVector;
5611	TRPMEVENT enmTrpmEvent;
5612	RTGCUINT uErrCode;
5613	RTGCUINTPTR GCPtrFaultAddress;
5614	uint8_t cbInstr;
5615
5616	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
5617	AssertRC(rc);
5618
5619	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntrInfo. */
5620	uint32_t u32IntrInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
5621	if (enmTrpmEvent == TRPM_TRAP)
5622	{
5623	switch (uVector)
5624	{
5625	case X86_XCPT_BP:
5626	case X86_XCPT_OF:
5627	{
5628	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5629	break;
5630	}
5631
5632	case X86_XCPT_PF:
5633	case X86_XCPT_DF:
5634	case X86_XCPT_TS:
5635	case X86_XCPT_NP:
5636	case X86_XCPT_SS:
5637	case X86_XCPT_GP:
5638	case X86_XCPT_AC:
5639	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5640	/* no break! */
5641	default:
5642	{
5643	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5644	break;
5645	}
5646	}
5647	}
5648	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
5649	{
5650	if (uVector != X86_XCPT_NMI)
5651	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5652	else
5653	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5654	}
5655	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
5656	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5657	else
5658	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
5659
5660	rc = TRPMResetTrap(pVCpu);
5661	AssertRC(rc);
5662	Log(("TRPM->HM event: u32IntrInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
5663	u32IntrInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
5664	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, uErrCode, GCPtrFaultAddress);
5665	}
5666
5667
5668	/**
5669	* Converts any pending VMX event into a TRPM trap. Typically used when leaving
5670	* VT-x to execute any instruction.
5671	*
5672	* @param pvCpu Pointer to the VMCPU.
5673	*/
5674	static void hmR0VmxPendingEventToTRPMTrap(PVMCPU pVCpu)
5675	{
5676	Assert(pVCpu->hm.s.Event.fPending);
5677
5678	uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo);
5679	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntrInfo);
5680	bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntrInfo);
5681	uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
5682
5683	/* If a trap was already pending, we did something wrong! */
5684	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
5685
5686	TRPMEVENT enmTrapType;
5687	switch (uVectorType)
5688	{
5689	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
5690	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
5691	enmTrapType = TRPM_HARDWARE_INT;
5692	break;
5693	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
5694	enmTrapType = TRPM_SOFTWARE_INT;
5695	break;
5696	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
5697	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
5698	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
5699	enmTrapType = TRPM_TRAP;
5700	break;
5701	default:
5702	AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
5703	enmTrapType = TRPM_32BIT_HACK;
5704	break;
5705	}
5706
5707	Log(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
5708
5709	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
5710	AssertRC(rc);
5711
5712	if (fErrorCodeValid)
5713	TRPMSetErrorCode(pVCpu, uErrorCode);
5714	if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5715	&& uVector == X86_XCPT_PF)
5716	{
5717	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
5718	}
5719	else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5720	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5721	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
5722	{
5723	AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5724	\|\| (uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF),
5725	("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
5726	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
5727	}
5728	pVCpu->hm.s.Event.fPending = false;
5729	}
5730
5731
5732	/**
5733	* Does the necessary state syncing before doing a longjmp to ring-3.
5734	*
5735	* @param pVM Pointer to the VM.
5736	* @param pVCpu Pointer to the VMCPU.
5737	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5738	* out-of-sync. Make sure to update the required fields
5739	* before using them.
5740	* @param rcExit The reason for exiting to ring-3. Can be
5741	* VINF_VMM_UNKNOWN_RING3_CALL.
5742	*
5743	* @remarks No-long-jmp zone!!!
5744	*/
5745	static void hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
5746	{
5747	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
5748	Assert(VMMR0IsLogFlushDisabled(pVCpu));
5749
5750	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
5751	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL);
5752	AssertRC(rc);
5753
5754	/* Restore FPU state if necessary and resync on next R0 reentry .*/
5755	if (CPUMIsGuestFPUStateActive(pVCpu))
5756	{
5757	CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
5758	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
5759	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
5760	}
5761
5762	/* Restore debug registers if necessary and resync on next R0 reentry. */
5763	if (CPUMIsGuestDebugStateActive(pVCpu))
5764	{
5765	CPUMR0SaveGuestDebugState(pVM, pVCpu, pMixedCtx, true /* save DR6 */);
5766	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
5767	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
5768	}
5769	else if (CPUMIsHyperDebugStateActive(pVCpu))
5770	{
5771	CPUMR0LoadHostDebugState(pVM, pVCpu);
5772	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
5773	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
5774	}
5775
5776	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
5777	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
5778	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
5779	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
5780	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
5781	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
5782	}
5783
5784
5785	/**
5786	* An action requires us to go back to ring-3. This function does the necessary
5787	* steps before we can safely return to ring-3. This is not the same as longjmps
5788	* to ring-3, this is voluntary.
5789	*
5790	* @param pVM Pointer to the VM.
5791	* @param pVCpu Pointer to the VMCPU.
5792	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5793	* out-of-sync. Make sure to update the required fields
5794	* before using them.
5795	* @param rcExit The reason for exiting to ring-3. Can be
5796	* VINF_VMM_UNKNOWN_RING3_CALL.
5797	*/
5798	static void hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
5799	{
5800	Assert(pVM);
5801	Assert(pVCpu);
5802	Assert(pMixedCtx);
5803	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5804
5805	if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_GUEST_STATE))
5806	{
5807	/* We want to see what the guest-state was before VM-entry, don't resync here, as we won't continue guest execution. */
5808	return;
5809	}
5810	else if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
5811	{
5812	VMXGetActivateVMCS(&pVCpu->hm.s.vmx.lasterror.u64VMCSPhys);
5813	pVCpu->hm.s.vmx.lasterror.u32VMCSRevision = (uint32_t )pVCpu->hm.s.vmx.pvVmcs;
5814	pVCpu->hm.s.vmx.lasterror.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
5815	pVCpu->hm.s.vmx.lasterror.idCurrentCpu = RTMpCpuId();
5816	return;
5817	}
5818
5819	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
5820	VMMRZCallRing3Disable(pVCpu);
5821	Log(("hmR0VmxExitToRing3: rcExit=%d\n", rcExit));
5822
5823	/* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
5824	if (pVCpu->hm.s.Event.fPending)
5825	{
5826	hmR0VmxPendingEventToTRPMTrap(pVCpu);
5827	Assert(!pVCpu->hm.s.Event.fPending);
5828	}
5829
5830	/* Sync. the guest state. */
5831	hmR0VmxLongJmpToRing3(pVM, pVCpu, pMixedCtx, rcExit);
5832	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
5833
5834	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
5835	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
5836	\| CPUM_CHANGED_LDTR
5837	\| CPUM_CHANGED_GDTR
5838	\| CPUM_CHANGED_IDTR
5839	\| CPUM_CHANGED_TR
5840	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
5841
5842	/* On our way back from ring-3 the following needs to be done. */
5843	/** @todo This can change with preemption hooks. */
5844	if (rcExit == VINF_EM_RAW_INTERRUPT)
5845	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_HOST_CONTEXT;
5846	else
5847	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_ALL_GUEST;
5848
5849	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
5850	VMMRZCallRing3Enable(pVCpu);
5851	}
5852
5853
5854	/**
5855	* VMMRZCallRing3 callback wrapper which saves the guest state before we
5856	* longjump to ring-3 and possibly get preempted.
5857	*
5858	* @param pVCpu Pointer to the VMCPU.
5859	* @param enmOperation The operation causing the ring-3 longjump.
5860	* @param pvUser The user argument (pointer to the possibly
5861	* out-of-date guest-CPU context).
5862	*
5863	* @remarks Must never be called with @a enmOperation ==
5864	* VMMCALLRING3_VM_R0_ASSERTION.
5865	*/
5866	DECLCALLBACK(void) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
5867	{
5868	/* VMMRZCallRing3() already makes sure we never get called as a result of an longjmp due to an assertion, */
5869	Assert(pVCpu);
5870	Assert(pvUser);
5871	Assert(VMMRZCallRing3IsEnabled(pVCpu));
5872	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5873
5874	VMMRZCallRing3Disable(pVCpu);
5875	Assert(VMMR0IsLogFlushDisabled(pVCpu));
5876	Log(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3\n"));
5877	hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser, VINF_VMM_UNKNOWN_RING3_CALL);
5878	VMMRZCallRing3Enable(pVCpu);
5879	}
5880
5881
5882	/**
5883	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
5884	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
5885	*
5886	* @returns VBox status code.
5887	* @param pVCpu Pointer to the VMCPU.
5888	*/
5889	DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
5890	{
5891	if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
5892	{
5893	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
5894	{
5895	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
5896	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
5897	AssertRC(rc);
5898	}
5899	} /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
5900	}
5901
5902
5903	/**
5904	* Injects any pending events into the guest if the guest is in a state to
5905	* receive them.
5906	*
5907	* @returns VBox status code (informational status codes included).
5908	* @param pVCpu Pointer to the VMCPU.
5909	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5910	* out-of-sync. Make sure to update the required fields
5911	* before using them.
5912	*/
5913	static int hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5914	{
5915	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
5916	uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
5917	bool fBlockMovSS = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
5918	bool fBlockSti = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
5919
5920	Assert(!fBlockSti \|\| (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS));
5921	Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
5922	&& !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
5923	Assert(!fBlockSti \|\| pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
5924	Assert(!TRPMHasTrap(pVCpu));
5925
5926	int rc = VINF_SUCCESS;
5927	if (pVCpu->hm.s.Event.fPending) /* First, inject any pending HM events. */
5928	{
5929	uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo);
5930	bool fInject = true;
5931	if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
5932	{
5933	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
5934	AssertRCReturn(rc, rc);
5935	const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
5936	if ( fBlockInt
5937	\|\| fBlockSti
5938	\|\| fBlockMovSS)
5939	{
5940	fInject = false;
5941	}
5942	}
5943	else if ( uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
5944	&& ( fBlockMovSS
5945	\|\| fBlockSti))
5946	{
5947	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
5948	fInject = false;
5949	}
5950
5951	if (fInject)
5952	{
5953	Log(("Injecting pending event\n"));
5954	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntrInfo, pVCpu->hm.s.Event.cbInstr,
5955	pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, &uIntrState);
5956	AssertRCReturn(rc, rc);
5957	pVCpu->hm.s.Event.fPending = false;
5958	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntReinject);
5959	}
5960	else
5961	hmR0VmxSetIntWindowExitVmcs(pVCpu);
5962	} /** @todo SMI. SMIs take priority over NMIs. */
5963	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts . */
5964	{
5965	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
5966	if ( !fBlockMovSS
5967	&& !fBlockSti)
5968	{
5969	Log(("Injecting NMI\n"));
5970	uint32_t u32IntrInfo = X86_XCPT_NMI \| VMX_EXIT_INTERRUPTION_INFO_VALID;
5971	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5972	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */,
5973	0 /* GCPtrFaultAddress */, &uIntrState);
5974	AssertRCReturn(rc, rc);
5975	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
5976	}
5977	else
5978	hmR0VmxSetIntWindowExitVmcs(pVCpu);
5979	}
5980	else if (VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)))
5981	{
5982	/* Check if there are guest external interrupts (PIC/APIC) pending and inject them if the guest can receive them. */
5983	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
5984	AssertRCReturn(rc, rc);
5985	const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
5986	if ( !fBlockInt
5987	&& !fBlockSti
5988	&& !fBlockMovSS)
5989	{
5990	uint8_t u8Interrupt;
5991	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
5992	if (RT_SUCCESS(rc))
5993	{
5994	Log(("Injecting interrupt u8Interrupt=%#x\n", u8Interrupt));
5995	uint32_t u32IntrInfo = u8Interrupt \| VMX_EXIT_INTERRUPTION_INFO_VALID;
5996	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5997	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */,
5998	0 /* GCPtrFaultAddress */, &uIntrState);
5999	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
6000	}
6001	else
6002	{
6003	/** @todo Does this actually happen? If not turn it into an assertion. */
6004	Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)));
6005	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
6006	rc = VINF_SUCCESS;
6007	}
6008	}
6009	else
6010	hmR0VmxSetIntWindowExitVmcs(pVCpu);
6011	}
6012
6013	/*
6014	* Delivery pending debug exception if the guest is single-stepping. The interruptibility-state could have been changed by
6015	* hmR0VmxInjectEventVmcs() (e.g. real-on-v86 injecting software interrupts), re-evaluate it and set the BS bit.
6016	*/
6017	fBlockMovSS = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6018	fBlockSti = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
6019	int rc2 = VINF_SUCCESS;
6020	if ( fBlockSti
6021	\|\| fBlockMovSS)
6022	{
6023	if (!DBGFIsStepping(pVCpu))
6024	{
6025	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS);
6026	if (pMixedCtx->eflags.Bits.u1TF) /* We don't have any IA32_DEBUGCTL MSR for guests. Treat as all bits 0. */
6027	{
6028	/*
6029	* The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD, VMX_EXIT_MTF
6030	* VMX_EXIT_APIC_WRITE, VMX_EXIT_VIRTUALIZED_EOI. See Intel spec. 27.3.4 "Saving Non-Register State".
6031	*/
6032	rc2 = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
6033	AssertRCReturn(rc, rc);
6034	}
6035	}
6036	else
6037	{
6038	/* We are single-stepping in the hypervisor debugger, clear interrupt inhibition as setting the BS bit would mean
6039	delivering a #DB to the guest upon VM-entry when it shouldn't be. */
6040	uIntrState = 0;
6041	}
6042	}
6043
6044	/*
6045	* There's no need to clear the VM entry-interruption information field here if we're not injecting anything.
6046	* VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
6047	*/
6048	rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
6049	AssertRC(rc2);
6050
6051	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
6052	return rc;
6053	}
6054
6055
6056	/**
6057	* Sets an invalid-opcode (#UD) exception as pending-for-injection into the VM.
6058	*
6059	* @param pVCpu Pointer to the VMCPU.
6060	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6061	* out-of-sync. Make sure to update the required fields
6062	* before using them.
6063	*/
6064	DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6065	{
6066	uint32_t u32IntrInfo = X86_XCPT_UD \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6067	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6068	}
6069
6070
6071	/**
6072	* Injects a double-fault (#DF) exception into the VM.
6073	*
6074	* @returns VBox status code (informational status code included).
6075	* @param pVCpu Pointer to the VMCPU.
6076	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6077	* out-of-sync. Make sure to update the required fields
6078	* before using them.
6079	*/
6080	DECLINLINE(int) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t *puIntrState)
6081	{
6082	uint32_t u32IntrInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6083	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6084	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6085	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */,
6086	puIntrState);
6087	}
6088
6089
6090	/**
6091	* Sets a debug (#DB) exception as pending-for-injection into the VM.
6092	*
6093	* @param pVCpu Pointer to the VMCPU.
6094	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6095	* out-of-sync. Make sure to update the required fields
6096	* before using them.
6097	*/
6098	DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6099	{
6100	uint32_t u32IntrInfo = X86_XCPT_DB \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6101	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6102	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6103	}
6104
6105
6106	/**
6107	* Sets an overflow (#OF) exception as pending-for-injection into the VM.
6108	*
6109	* @param pVCpu Pointer to the VMCPU.
6110	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6111	* out-of-sync. Make sure to update the required fields
6112	* before using them.
6113	* @param cbInstr The value of RIP that is to be pushed on the guest
6114	* stack.
6115	*/
6116	DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
6117	{
6118	uint32_t u32IntrInfo = X86_XCPT_OF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6119	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6120	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
6121	}
6122
6123
6124	/**
6125	* Injects a general-protection (#GP) fault into the VM.
6126	*
6127	* @returns VBox status code (informational status code included).
6128	* @param pVCpu Pointer to the VMCPU.
6129	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6130	* out-of-sync. Make sure to update the required fields
6131	* before using them.
6132	* @param u32ErrorCode The error code associated with the #GP.
6133	*/
6134	DECLINLINE(int) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
6135	uint32_t *puIntrState)
6136	{
6137	uint32_t u32IntrInfo = X86_XCPT_GP \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6138	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6139	if (fErrorCodeValid)
6140	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6141	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, u32ErrorCode, 0 / GCPtrFaultAddress */,
6142	puIntrState);
6143	}
6144
6145
6146	/**
6147	* Sets a software interrupt (INTn) as pending-for-injection into the VM.
6148	*
6149	* @param pVCpu Pointer to the VMCPU.
6150	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6151	* out-of-sync. Make sure to update the required fields
6152	* before using them.
6153	* @param uVector The software interrupt vector number.
6154	* @param cbInstr The value of RIP that is to be pushed on the guest
6155	* stack.
6156	*/
6157	DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
6158	{
6159	uint32_t u32IntrInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6160	if ( uVector == X86_XCPT_BP
6161	\|\| uVector == X86_XCPT_OF)
6162	{
6163	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6164	}
6165	else
6166	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6167	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
6168	}
6169
6170
6171	/**
6172	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
6173	* stack.
6174	*
6175	* @returns VBox status code (information status code included).
6176	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
6177	* @param pVM Pointer to the VM.
6178	* @param pMixedCtx Pointer to the guest-CPU context.
6179	* @param uValue The value to push to the guest stack.
6180	*/
6181	DECLINLINE(int) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
6182	{
6183	/*
6184	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
6185	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
6186	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
6187	*/
6188	if (pMixedCtx->sp == 1)
6189	return VINF_EM_RESET;
6190	pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
6191	int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
6192	AssertRCReturn(rc, rc);
6193	return rc;
6194	}
6195
6196
6197	/**
6198	* Injects an event into the guest upon VM-entry by updating the relevant fields
6199	* in the VM-entry area in the VMCS.
6200	*
6201	* @returns VBox status code (informational error codes included).
6202	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
6203	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
6204	*
6205	* @param pVCpu Pointer to the VMCPU.
6206	* @param pMixedCtx Pointer to the guest-CPU context. The data may
6207	* be out-of-sync. Make sure to update the required
6208	* fields before using them.
6209	* @param u64IntrInfo The VM-entry interruption-information field.
6210	* @param cbInstr The VM-entry instruction length in bytes (for
6211	* software interrupts, exceptions and privileged
6212	* software exceptions).
6213	* @param u32ErrCode The VM-entry exception error code.
6214	* @param GCPtrFaultAddress The page-fault address for #PF exceptions.
6215	* @param puIntrState Pointer to the current guest interruptibility-state.
6216	* This interruptibility-state will be updated if
6217	* necessary. This cannot not be NULL.
6218	*
6219	* @remarks No-long-jump zone!!!
6220	* @remarks Requires CR0!
6221	*/
6222	static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntrInfo, uint32_t cbInstr,
6223	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState)
6224	{
6225	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
6226	AssertMsg(u64IntrInfo >> 32 == 0, ("%#RX64\n", u64IntrInfo));
6227	Assert(puIntrState);
6228	uint32_t u32IntrInfo = (uint32_t)u64IntrInfo;
6229
6230	const uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntrInfo);
6231	const uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo);
6232
6233	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
6234	Assert( uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
6235	\|\| !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
6236
6237	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
6238
6239	/* We require CR0 to check if the guest is in real-mode. */
6240	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6241	AssertRCReturn(rc, rc);
6242
6243	/*
6244	* Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
6245	* mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
6246	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
6247	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
6248	*/
6249	if (CPUMIsGuestInRealModeEx(pMixedCtx))
6250	{
6251	PVM pVM = pVCpu->CTX_SUFF(pVM);
6252	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
6253	{
6254	Assert(PDMVmmDevHeapIsEnabled(pVM));
6255	Assert(pVM->hm.s.vmx.pRealModeTSS);
6256
6257	/* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
6258	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6259	rc \|= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
6260	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6261	AssertRCReturn(rc, rc);
6262	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RIP);
6263
6264	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
6265	const size_t cbIdtEntry = 4;
6266	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
6267	{
6268	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
6269	if (uVector == X86_XCPT_DF)
6270	return VINF_EM_RESET;
6271	else if (uVector == X86_XCPT_GP)
6272	{
6273	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
6274	return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, puIntrState);
6275	}
6276
6277	/* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
6278	/* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
6279	return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid /, 0 / u32ErrCode */, puIntrState);
6280	}
6281
6282	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
6283	uint16_t uGuestIp = pMixedCtx->ip;
6284	if (VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
6285	{
6286	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
6287	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
6288	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6289	}
6290	else if (VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
6291	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6292
6293	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
6294	uint16_t offIdtEntry = 0;
6295	RTSEL selIdtEntry = 0;
6296	RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
6297	rc = PGMPhysSimpleReadGCPhys(pVM, &offIdtEntry, GCPhysIdtEntry, sizeof(offIdtEntry));
6298	rc \|= PGMPhysSimpleReadGCPhys(pVM, &selIdtEntry, GCPhysIdtEntry + 2, sizeof(selIdtEntry));
6299	AssertRCReturn(rc, rc);
6300
6301	/* Construct the stack frame for the interrupt/exception handler. */
6302	rc = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
6303	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
6304	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
6305	AssertRCReturn(rc, rc);
6306
6307	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
6308	if (rc == VINF_SUCCESS)
6309	{
6310	pMixedCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
6311	pMixedCtx->rip = offIdtEntry;
6312	pMixedCtx->cs.Sel = selIdtEntry;
6313	pMixedCtx->cs.u64Base = selIdtEntry << cbIdtEntry;
6314	if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
6315	&& uVector == X86_XCPT_PF)
6316	{
6317	pMixedCtx->cr2 = GCPtrFaultAddress;
6318	}
6319	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SEGMENT_REGS
6320	\| HM_CHANGED_GUEST_RIP
6321	\| HM_CHANGED_GUEST_RFLAGS
6322	\| HM_CHANGED_GUEST_RSP;
6323
6324	/* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
6325	if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
6326	{
6327	Assert( uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
6328	&& uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
6329	Log(("Clearing inhibition due to STI.\n"));
6330	*puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
6331	}
6332	Log(("Injecting real-mode: u32IntrInfo=%#x u32ErrCode=%#x instrlen=%#x\n", u32IntrInfo, u32ErrCode, cbInstr));
6333	}
6334	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
6335	return rc;
6336	}
6337	else
6338	{
6339	/*
6340	* For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
6341	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
6342	*/
6343	u32IntrInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6344	}
6345	}
6346
6347	/* Validate. */
6348	Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntrInfo)); /* Bit 31 (Valid bit) must be set by caller. */
6349	Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK(u32IntrInfo)); /* Bit 12 MBZ. */
6350	Assert(!(u32IntrInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
6351
6352	/* Inject. */
6353	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntrInfo);
6354	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntrInfo))
6355	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
6356	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
6357
6358	if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
6359	&& uVector == X86_XCPT_PF)
6360	{
6361	pMixedCtx->cr2 = GCPtrFaultAddress;
6362	}
6363	Log(("Injecting u32IntrInfo=%#x u32ErrCode=%#x cbInstr=%#x uCR2=%#RX64\n", u32IntrInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
6364
6365	AssertRCReturn(rc, rc);
6366	return rc;
6367	}
6368
6369
6370	/**
6371	* Enters the VT-x session.
6372	*
6373	* @returns VBox status code.
6374	* @param pVM Pointer to the VM.
6375	* @param pVCpu Pointer to the VMCPU.
6376	* @param pCpu Pointer to the CPU info struct.
6377	*/
6378	VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBLCPUINFO pCpu)
6379	{
6380	AssertPtr(pVM);
6381	AssertPtr(pVCpu);
6382	Assert(pVM->hm.s.vmx.fSupported);
6383	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6384	NOREF(pCpu);
6385
6386	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6387
6388	/* Make sure we're in VMX root mode. */
6389	RTCCUINTREG u32HostCR4 = ASMGetCR4();
6390	if (!(u32HostCR4 & X86_CR4_VMXE))
6391	{
6392	LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
6393	return VERR_VMX_X86_CR4_VMXE_CLEARED;
6394	}
6395
6396	/* Load the active VMCS as the current one. */
6397	int rc = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
6398	if (RT_FAILURE(rc))
6399	return rc;
6400
6401	/** @todo this will change with preemption hooks where can can VMRESUME as long
6402	* as we're no preempted. */
6403	pVCpu->hm.s.fResumeVM = false;
6404	return VINF_SUCCESS;
6405	}
6406
6407
6408	/**
6409	* Leaves the VT-x session.
6410	*
6411	* @returns VBox status code.
6412	* @param pVM Pointer to the VM.
6413	* @param pVCpu Pointer to the VMCPU.
6414	* @param pCtx Pointer to the guest-CPU context.
6415	*/
6416	VMMR0DECL(int) VMXR0Leave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
6417	{
6418	AssertPtr(pVCpu);
6419	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6420	NOREF(pVM);
6421	NOREF(pCtx);
6422
6423	/** @todo this will change with preemption hooks where we only VMCLEAR when
6424	* we are actually going to be preempted, not all the time like we
6425	* currently do. */
6426	/*
6427	* Sync the current VMCS (writes back internal data back into the VMCS region in memory)
6428	* and mark the VMCS launch-state as "clear".
6429	*/
6430	int rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
6431	return rc;
6432	}
6433
6434
6435	/**
6436	* Saves the host state in the VMCS host-state.
6437	* Sets up the VM-exit MSR-load area.
6438	*
6439	* The CPU state will be loaded from these fields on every successful VM-exit.
6440	*
6441	* @returns VBox status code.
6442	* @param pVM Pointer to the VM.
6443	* @param pVCpu Pointer to the VMCPU.
6444	*
6445	* @remarks No-long-jump zone!!!
6446	*/
6447	VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
6448	{
6449	AssertPtr(pVM);
6450	AssertPtr(pVCpu);
6451	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6452
6453	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6454
6455	/* Nothing to do if the host-state-changed flag isn't set. This will later be optimized when preemption hooks are in place. */
6456	if (!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT))
6457	return VINF_SUCCESS;
6458
6459	int rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
6460	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6461
6462	rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
6463	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6464
6465	rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
6466	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6467
6468	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_HOST_CONTEXT;
6469	return rc;
6470	}
6471
6472
6473	/**
6474	* Loads the guest state into the VMCS guest-state area. The CPU state will be
6475	* loaded from these fields on every successful VM-entry.
6476	*
6477	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas.
6478	* Sets up the VM-entry controls.
6479	* Sets up the appropriate VMX non-root function to execute guest code based on
6480	* the guest CPU mode.
6481	*
6482	* @returns VBox status code.
6483	* @param pVM Pointer to the VM.
6484	* @param pVCpu Pointer to the VMCPU.
6485	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6486	* out-of-sync. Make sure to update the required fields
6487	* before using them.
6488	*
6489	* @remarks No-long-jump zone!!!
6490	*/
6491	VMMR0DECL(int) VMXR0LoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6492	{
6493	AssertPtr(pVM);
6494	AssertPtr(pVCpu);
6495	AssertPtr(pMixedCtx);
6496	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6497
6498	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6499
6500	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
6501
6502	/* Determine real-on-v86 mode. */
6503	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
6504	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
6505	&& CPUMIsGuestInRealModeEx(pMixedCtx))
6506	{
6507	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
6508	}
6509
6510	/*
6511	* Load the guest-state into the VMCS.
6512	* Any ordering dependency among the sub-functions below must be explicitly stated using comments.
6513	* Ideally, assert that the cross-dependent bits are up to date at the point of using it.
6514	*/
6515	int rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
6516	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6517
6518	rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
6519	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6520
6521	rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
6522	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6523
6524	rc = hmR0VmxLoadGuestControlRegs(pVCpu, pMixedCtx);
6525	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestControlRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6526
6527	/* Must be done after CR0 is loaded (strict builds require CR0 for segment register validation checks). */
6528	rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
6529	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6530
6531	rc = hmR0VmxLoadGuestDebugRegs(pVCpu, pMixedCtx);
6532	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestDebugRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6533
6534	rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
6535	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6536
6537	rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
6538	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6539
6540	/* Must be done after hmR0VmxLoadGuestDebugRegs() as it may update eflags.TF for debugging purposes. */
6541	rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
6542	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestGprs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6543
6544	rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
6545	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6546
6547	AssertMsg(!pVCpu->hm.s.fContextUseFlags,
6548	("Missed updating flags while loading guest state. pVM=%p pVCpu=%p fContextUseFlags=%#RX32\n",
6549	pVM, pVCpu, pVCpu->hm.s.fContextUseFlags));
6550
6551	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
6552	return rc;
6553	}
6554
6555
6556	/**
6557	* Does the preparations before executing guest code in VT-x.
6558	*
6559	* This may cause longjmps to ring-3 and may even result in rescheduling to the
6560	* recompiler. We must be cautious what we do here regarding committing
6561	* guest-state information into the the VMCS assuming we assuredly execute the
6562	* guest in VT-x. If we fall back to the recompiler after updating VMCS and
6563	* clearing the common-state (TRPM/forceflags), we must undo those changes so
6564	* that the recompiler can (and should) use them when it resumes guest
6565	* execution. Otherwise such operations must be done when we can no longer
6566	* exit to ring-3.
6567	*
6568	* @returns VBox status code (informational status codes included).
6569	* @retval VINF_SUCCESS if we can proceed with running the guest.
6570	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a double-fault
6571	* into the guest.
6572	* @retval VINF_* scheduling changes, we have to go back to ring-3.
6573	*
6574	* @param pVM Pointer to the VM.
6575	* @param pVCpu Pointer to the VMCPU.
6576	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6577	* out-of-sync. Make sure to update the required fields
6578	* before using them.
6579	* @param pVmxTransient Pointer to the VMX transient structure.
6580	*
6581	* @remarks Called with preemption disabled.
6582	*/
6583	DECLINLINE(int) hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6584	{
6585	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6586
6587	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
6588	PGMRZDynMapFlushAutoSet(pVCpu);
6589	#endif
6590
6591	/* Check force flag actions that might require us to go back to ring-3. */
6592	int rc = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx);
6593	if (rc != VINF_SUCCESS)
6594	return rc;
6595
6596	/* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
6597	if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
6598	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
6599	{
6600	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
6601	RTGCPHYS GCPhysApicBase;
6602	GCPhysApicBase = pMixedCtx->msrApicBase;
6603	GCPhysApicBase &= PAGE_BASE_GC_MASK;
6604
6605	/* Unalias any existing mapping. */
6606	rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
6607	AssertRCReturn(rc, rc);
6608
6609	/* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
6610	Log(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGv\n", GCPhysApicBase));
6611	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
6612	AssertRCReturn(rc, rc);
6613
6614	pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
6615	}
6616
6617	#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
6618	/* We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.) */
6619	pVmxTransient->uEFlags = ASMIntDisableFlags();
6620	if (RTThreadPreemptIsPending(NIL_RTTHREAD))
6621	{
6622	ASMSetFlags(pVmxTransient->uEFlags);
6623	STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
6624	/* Don't use VINF_EM_RAW_INTERRUPT_HYPER as we can't assume the host does kernel preemption. Maybe some day? */
6625	return VINF_EM_RAW_INTERRUPT;
6626	}
6627	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
6628	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
6629	#endif
6630
6631	/*
6632	* Evaluates and injects any pending events, toggling force-flags and updating the guest-interruptibility
6633	* state (interrupt shadow) in the VMCS. This -can- potentially be reworked to be done before disabling
6634	* interrupts and handle returning to ring-3 afterwards, but requires very careful state restoration.
6635	*/
6636	/** @todo Rework event evaluation and injection to be completely separate. */
6637	if (TRPMHasTrap(pVCpu))
6638	hmR0VmxTRPMTrapToPendingEvent(pVCpu);
6639
6640	rc = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx);
6641	AssertRCReturn(rc, rc);
6642	return rc;
6643	}
6644
6645
6646	/**
6647	* Prepares to run guest code in VT-x and we've committed to doing so. This
6648	* means there is no backing out to ring-3 or anywhere else at this
6649	* point.
6650	*
6651	* @param pVM Pointer to the VM.
6652	* @param pVCpu Pointer to the VMCPU.
6653	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6654	* out-of-sync. Make sure to update the required fields
6655	* before using them.
6656	* @param pVmxTransient Pointer to the VMX transient structure.
6657	*
6658	* @remarks Called with preemption disabled.
6659	* @remarks No-long-jump zone!!!
6660	*/
6661	DECLINLINE(void) hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6662	{
6663	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6664	Assert(VMMR0IsLogFlushDisabled(pVCpu));
6665
6666	#ifndef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
6667	/** @todo I don't see the point of this, VMMR0EntryFast() already disables interrupts for the entire period. */
6668	pVmxTransient->uEFlags = ASMIntDisableFlags();
6669	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
6670	#endif
6671
6672	/* Load the required guest state bits (for guest-state changes in the inner execution loop). */
6673	Assert(!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT));
6674	Log4(("LoadFlags=%#RX32\n", pVCpu->hm.s.fContextUseFlags));
6675	#ifdef HMVMX_SYNC_FULL_GUEST_STATE
6676	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_ALL_GUEST;
6677	#endif
6678	int rc = VINF_SUCCESS;
6679	if (pVCpu->hm.s.fContextUseFlags == HM_CHANGED_GUEST_RIP)
6680	{
6681	rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
6682	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
6683	}
6684	else if (pVCpu->hm.s.fContextUseFlags)
6685	{
6686	rc = VMXR0LoadGuestState(pVM, pVCpu, pMixedCtx);
6687	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
6688	}
6689	AssertRC(rc);
6690	AssertMsg(!pVCpu->hm.s.fContextUseFlags, ("fContextUseFlags =%#x\n", pVCpu->hm.s.fContextUseFlags));
6691
6692	/* Cache the TPR-shadow for checking on every VM-exit if it might have changed. */
6693	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
6694	pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
6695
6696	if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
6697	\|\| HMR0GetCurrentCpu()->idCpu != pVCpu->hm.s.idLastCpu)
6698	{
6699	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu, pMixedCtx);
6700	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
6701	}
6702
6703	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB-shootdowns, set this across the world switch. */
6704	hmR0VmxFlushTaggedTlb(pVCpu); /* Invalidate the appropriate guest entries from the TLB. */
6705	Assert(HMR0GetCurrentCpu()->idCpu == pVCpu->hm.s.idLastCpu);
6706
6707	/*
6708	* TPR patching (only active for 32-bit guests on 64-bit capable CPUs) when the CPU does not supported virtualizing
6709	* APIC accesses feature (VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC).
6710	*/
6711	if (pVM->hm.s.fTPRPatchingActive)
6712	{
6713	Assert(!CPUMIsGuestInLongMode(pVCpu));
6714
6715	/* Need guest's LSTAR MSR (which is part of the auto load/store MSRs in the VMCS), ensure we have the updated one. */
6716	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6717	AssertRC(rc);
6718
6719	/* The patch code uses the LSTAR as it's not used by a guest in 32-bit mode implicitly (i.e. SYSCALL is 64-bit only). */
6720	pVmxTransient->u64LStarMsr = ASMRdMsr(MSR_K8_LSTAR);
6721	ASMWrMsr(MSR_K8_LSTAR, pMixedCtx->msrLSTAR); /* pMixedCtx->msrLSTAR contains the guest's TPR,
6722	see hmR0VmxLoadGuestApicState(). */
6723	}
6724
6725	#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
6726	/*
6727	* Save the current Host TSC_AUX and write the guest TSC_AUX to the host, so that
6728	* RDTSCPs (that don't cause exits) reads the guest MSR. See @bugref{3324}.
6729	*/
6730	if ( (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
6731	&& !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
6732	{
6733	pVCpu->hm.s.u64HostTscAux = ASMRdMsr(MSR_K8_TSC_AUX);
6734	uint64_t u64HostTscAux = 0;
6735	int rc2 = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &u64HostTscAux);
6736	AssertRC(rc2);
6737	ASMWrMsr(MSR_K8_TSC_AUX, u64HostTscAux);
6738	}
6739	#endif
6740
6741	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
6742	TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
6743	to start executing. */
6744	}
6745
6746
6747	/**
6748	* Performs some essential restoration of state after running guest code in
6749	* VT-x.
6750	*
6751	* @param pVM Pointer to the VM.
6752	* @param pVCpu Pointer to the VMCPU.
6753	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6754	* out-of-sync. Make sure to update the required fields
6755	* before using them.
6756	* @param pVmxTransient Pointer to the VMX transient structure.
6757	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
6758	*
6759	* @remarks Called with interrupts disabled.
6760	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
6761	* unconditionally when it is safe to do so.
6762	*/
6763	DECLINLINE(void) hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
6764	{
6765	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6766	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
6767
6768	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB-shootdowns. */
6769	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for TLB-shootdowns. */
6770	pVCpu->hm.s.vmx.fUpdatedGuestState = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
6771	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
6772	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
6773
6774	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
6775	{
6776	#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
6777	/* Restore host's TSC_AUX. */
6778	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
6779	ASMWrMsr(MSR_K8_TSC_AUX, pVCpu->hm.s.u64HostTscAux);
6780	#endif
6781	/** @todo Find a way to fix hardcoding a guestimate. */
6782	TMCpuTickSetLastSeen(pVCpu, ASMReadTSC()
6783	+ pVCpu->hm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
6784	}
6785
6786	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
6787	Assert(!(ASMGetFlags() & X86_EFL_IF));
6788	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
6789
6790	/* Restore the effects of TPR patching if any. */
6791	if (pVM->hm.s.fTPRPatchingActive)
6792	{
6793	int rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6794	AssertRC(rc);
6795	pMixedCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR); /* MSR_K8_LSTAR contains the guest TPR. */
6796	ASMWrMsr(MSR_K8_LSTAR, pVmxTransient->u64LStarMsr);
6797	}
6798
6799	ASMSetFlags(pVmxTransient->uEFlags); /* Enable interrupts. */
6800	pVCpu->hm.s.fResumeVM = true; /* Use VMRESUME instead of VMLAUNCH in the next run. */
6801
6802	/* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
6803	uint32_t uExitReason;
6804	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
6805	rc \|= hmR0VmxReadEntryIntrInfoVmcs(pVmxTransient);
6806	AssertRC(rc);
6807	pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
6808	pVmxTransient->fVMEntryFailed = !!VMX_ENTRY_INTERRUPTION_INFO_VALID(pVmxTransient->uEntryIntrInfo);
6809
6810	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pMixedCtx);
6811	VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
6812
6813	/* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_. /
6814	if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
6815	{
6816	Log(("VM-entry failure: rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", rcVMRun, pVmxTransient->fVMEntryFailed));
6817	return;
6818	}
6819
6820	if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
6821	{
6822	/* Update the guest interruptibility-state from the VMCS. */
6823	hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
6824	#if defined(HMVMX_SYNC_FULL_GUEST_STATE) \|\| defined(HMVMX_SAVE_FULL_GUEST_STATE)
6825	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6826	AssertRC(rc);
6827	#endif
6828	/*
6829	* If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
6830	* we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3 and which is
6831	* why it's done here as it's easier and no less efficient to deal with it here than making hmR0VmxSaveGuestState()
6832	* cope with longjmps safely (see VMCPU_FF_HM_UPDATE_CR3 handling).
6833	*/
6834	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
6835	&& pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
6836	{
6837	rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
6838	AssertRC(rc);
6839	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
6840	}
6841	}
6842	}
6843
6844
6845	/**
6846	* Runs the guest code using VT-x.
6847	*
6848	* @returns VBox status code.
6849	* @param pVM Pointer to the VM.
6850	* @param pVCpu Pointer to the VMCPU.
6851	* @param pCtx Pointer to the guest-CPU context.
6852	*
6853	* @remarks Called with preemption disabled.
6854	*/
6855	VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
6856	{
6857	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6858	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6859
6860	VMXTRANSIENT VmxTransient;
6861	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
6862	int rc = VERR_INTERNAL_ERROR_5;
6863	uint32_t cLoops = 0;
6864
6865	for (;; cLoops++)
6866	{
6867	Assert(!HMR0SuspendPending());
6868	AssertMsg(pVCpu->hm.s.idEnteredCpu == RTMpCpuId(),
6869	("Illegal migration! Entered on CPU %u Current %u cLoops=%u\n", (unsigned)pVCpu->hm.s.idEnteredCpu,
6870	(unsigned)RTMpCpuId(), cLoops));
6871
6872	/* Preparatory work for running guest code, this may return to ring-3 for some last minute updates. */
6873	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
6874	rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
6875	if (rc != VINF_SUCCESS)
6876	break;
6877
6878	/*
6879	* No longjmps to ring-3 from this point on!!!
6880	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
6881	* This also disables flushing of the R0-logger instance (if any).
6882	*/
6883	VMMRZCallRing3Disable(pVCpu);
6884	VMMRZCallRing3RemoveNotification(pVCpu);
6885	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
6886
6887	rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
6888	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
6889
6890	/*
6891	* Restore any residual host-state and save any bits shared between host and guest into the guest-CPU state.
6892	* This will also re-enable longjmps to ring-3 when it has reached a safe point!!!
6893	*/
6894	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
6895	if (RT_UNLIKELY(rc != VINF_SUCCESS)) /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
6896	{
6897	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
6898	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
6899	return rc;
6900	}
6901
6902	/* Handle the VM-exit. */
6903	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
6904	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
6905	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
6906	HMVMX_START_EXIT_DISPATCH_PROF();
6907	#ifdef HMVMX_USE_FUNCTION_TABLE
6908	rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
6909	#else
6910	rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
6911	#endif
6912	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
6913	if (rc != VINF_SUCCESS)
6914	break;
6915	else if (cLoops > pVM->hm.s.cMaxResumeLoops)
6916	{
6917	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
6918	rc = VINF_EM_RAW_INTERRUPT;
6919	break;
6920	}
6921	}
6922
6923	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
6924	if (rc == VERR_EM_INTERPRETER)
6925	rc = VINF_EM_RAW_EMULATE_INSTR;
6926	else if (rc == VINF_EM_RESET)
6927	rc = VINF_EM_TRIPLE_FAULT;
6928	hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rc);
6929	return rc;
6930	}
6931
6932
6933	#ifndef HMVMX_USE_FUNCTION_TABLE
6934	DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
6935	{
6936	int rc;
6937	switch (rcReason)
6938	{
6939	case VMX_EXIT_EPT_MISCONFIG: rc = hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient); break;
6940	case VMX_EXIT_EPT_VIOLATION: rc = hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient); break;
6941	case VMX_EXIT_IO_INSTR: rc = hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient); break;
6942	case VMX_EXIT_CPUID: rc = hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient); break;
6943	case VMX_EXIT_RDTSC: rc = hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient); break;
6944	case VMX_EXIT_RDTSCP: rc = hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient); break;
6945	case VMX_EXIT_APIC_ACCESS: rc = hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient); break;
6946	case VMX_EXIT_XCPT_NMI: rc = hmR0VmxExitXcptNmi(pVCpu, pMixedCtx, pVmxTransient); break;
6947	case VMX_EXIT_MOV_CRX: rc = hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient); break;
6948	case VMX_EXIT_EXT_INT: rc = hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient); break;
6949	case VMX_EXIT_INT_WINDOW: rc = hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient); break;
6950	case VMX_EXIT_MWAIT: rc = hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient); break;
6951	case VMX_EXIT_MONITOR: rc = hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient); break;
6952	case VMX_EXIT_TASK_SWITCH: rc = hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient); break;
6953	case VMX_EXIT_PREEMPT_TIMER: rc = hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient); break;
6954	case VMX_EXIT_RDMSR: rc = hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient); break;
6955	case VMX_EXIT_WRMSR: rc = hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient); break;
6956	case VMX_EXIT_MOV_DRX: rc = hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient); break;
6957	case VMX_EXIT_TPR_BELOW_THRESHOLD: rc = hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient); break;
6958	case VMX_EXIT_HLT: rc = hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient); break;
6959	case VMX_EXIT_INVD: rc = hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient); break;
6960	case VMX_EXIT_INVLPG: rc = hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient); break;
6961	case VMX_EXIT_RSM: rc = hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient); break;
6962	case VMX_EXIT_MTF: rc = hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient); break;
6963	case VMX_EXIT_PAUSE: rc = hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient); break;
6964	case VMX_EXIT_XDTR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); break;
6965	case VMX_EXIT_TR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); break;
6966	case VMX_EXIT_WBINVD: rc = hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient); break;
6967	case VMX_EXIT_XSETBV: rc = hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient); break;
6968	case VMX_EXIT_RDRAND: rc = hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient); break;
6969	case VMX_EXIT_INVPCID: rc = hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient); break;
6970	case VMX_EXIT_GETSEC: rc = hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient); break;
6971	case VMX_EXIT_RDPMC: rc = hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient); break;
6972
6973	case VMX_EXIT_TRIPLE_FAULT: rc = hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient); break;
6974	case VMX_EXIT_NMI_WINDOW: rc = hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient); break;
6975	case VMX_EXIT_INIT_SIGNAL: rc = hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient); break;
6976	case VMX_EXIT_SIPI: rc = hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient); break;
6977	case VMX_EXIT_IO_SMI: rc = hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient); break;
6978	case VMX_EXIT_SMI: rc = hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient); break;
6979	case VMX_EXIT_ERR_MSR_LOAD: rc = hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient); break;
6980	case VMX_EXIT_ERR_INVALID_GUEST_STATE: rc = hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient); break;
6981	case VMX_EXIT_ERR_MACHINE_CHECK: rc = hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient); break;
6982
6983	case VMX_EXIT_VMCALL:
6984	case VMX_EXIT_VMCLEAR:
6985	case VMX_EXIT_VMLAUNCH:
6986	case VMX_EXIT_VMPTRLD:
6987	case VMX_EXIT_VMPTRST:
6988	case VMX_EXIT_VMREAD:
6989	case VMX_EXIT_VMRESUME:
6990	case VMX_EXIT_VMWRITE:
6991	case VMX_EXIT_VMXOFF:
6992	case VMX_EXIT_VMXON:
6993	case VMX_EXIT_INVEPT:
6994	case VMX_EXIT_INVVPID:
6995	case VMX_EXIT_VMFUNC:
6996	rc = hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
6997	break;
6998	default:
6999	rc = hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
7000	break;
7001	}
7002	return rc;
7003	}
7004	#endif
7005
7006	#ifdef DEBUG
7007	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
7008	# define VMX_ASSERT_PREEMPT_CPUID_VAR() \
7009	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
7010	# define VMX_ASSERT_PREEMPT_CPUID() \
7011	do \
7012	{ \
7013	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
7014	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
7015	} while (0)
7016
7017	# define VMX_VALIDATE_EXIT_HANDLER_PARAMS() \
7018	do { \
7019	AssertPtr(pVCpu); \
7020	AssertPtr(pMixedCtx); \
7021	AssertPtr(pVmxTransient); \
7022	Assert(pVmxTransient->fVMEntryFailed == false); \
7023	Assert(ASMIntAreEnabled()); \
7024	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD)); \
7025	VMX_ASSERT_PREEMPT_CPUID_VAR(); \
7026	LogFunc(("vcpu[%u] vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv\n", \
7027	(unsigned)pVCpu->idCpu)); \
7028	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD)); \
7029	if (VMMR0IsLogFlushDisabled(pVCpu)) \
7030	VMX_ASSERT_PREEMPT_CPUID(); \
7031	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
7032	} while (0)
7033	# define VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
7034	do { \
7035	LogFunc(("\n")); \
7036	} while(0)
7037	#else /* Release builds */
7038	# define VMX_VALIDATE_EXIT_HANDLER_PARAMS() do { HMVMX_STOP_EXIT_DISPATCH_PROF(); } while(0)
7039	# define VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while(0)
7040	#endif
7041
7042
7043	/**
7044	* Advances the guest RIP after reading it from the VMCS.
7045	*
7046	* @returns VBox status code.
7047	* @param pVCpu Pointer to the VMCPU.
7048	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
7049	* out-of-sync. Make sure to update the required fields
7050	* before using them.
7051	* @param pVmxTransient Pointer to the VMX transient structure.
7052	*
7053	* @remarks No-long-jump zone!!!
7054	*/
7055	DECLINLINE(int) hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7056	{
7057	int rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7058	rc \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
7059	AssertRCReturn(rc, rc);
7060
7061	pMixedCtx->rip += pVmxTransient->cbInstr;
7062	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7063	return rc;
7064	}
7065
7066
7067	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
7068	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
7069	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
7070	/**
7071	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
7072	*/
7073	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7074	{
7075	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7076	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
7077	#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
7078	Assert(ASMIntAreEnabled());
7079	return VINF_SUCCESS;
7080	#else
7081	return VINF_EM_RAW_INTERRUPT;
7082	#endif
7083	}
7084
7085
7086	/**
7087	* VM-exit handler for exceptions and NMIs (VMX_EXIT_XCPT_NMI).
7088	*/
7089	HMVMX_EXIT_DECL hmR0VmxExitXcptNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7090	{
7091	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7092	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
7093
7094	int rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
7095	AssertRCReturn(rc, rc);
7096
7097	uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntrInfo);
7098	Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
7099	&& uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
7100
7101	if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7102	{
7103	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
7104	return VINF_EM_RAW_INTERRUPT;
7105	}
7106
7107	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
7108	rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
7109	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
7110	{
7111	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
7112	return VINF_SUCCESS;
7113	}
7114	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
7115	{
7116	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
7117	return rc;
7118	}
7119
7120	uint32_t uExitIntrInfo = pVmxTransient->uExitIntrInfo;
7121	uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntrInfo);
7122	switch (uIntrType)
7123	{
7124	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
7125	Assert(uVector == X86_XCPT_DB \|\| uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
7126	/* no break */
7127	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
7128	{
7129	switch (uVector)
7130	{
7131	case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
7132	case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
7133	case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
7134	case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
7135	case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
7136	case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
7137	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
7138	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
7139	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7140	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
7141	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7142	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
7143	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7144	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
7145	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7146	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
7147	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7148	#endif
7149	default:
7150	{
7151	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7152	AssertRCReturn(rc, rc);
7153
7154	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
7155	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
7156	{
7157	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
7158	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
7159	rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7160	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
7161	AssertRCReturn(rc, rc);
7162	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntrInfo),
7163	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode,
7164	0 /* GCPtrFaultAddress */);
7165	AssertRCReturn(rc, rc);
7166	}
7167	else
7168	{
7169	AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
7170	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
7171	}
7172	break;
7173	}
7174	}
7175	break;
7176	}
7177
7178	case VMX_EXIT_INTERRUPTION_INFO_TYPE_DB_XCPT:
7179	default:
7180	{
7181	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_CODE;
7182	AssertMsgFailed(("Unexpected interruption code %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntrInfo)));
7183	break;
7184	}
7185	}
7186	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
7187	return rc;
7188	}
7189
7190
7191	/**
7192	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
7193	*/
7194	HMVMX_EXIT_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7195	{
7196	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7197
7198	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
7199	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7200	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7201	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7202	AssertRCReturn(rc, rc);
7203
7204	/* Deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and resume guest execution. */
7205	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
7206	return VINF_SUCCESS;
7207	}
7208
7209
7210	/**
7211	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
7212	*/
7213	HMVMX_EXIT_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7214	{
7215	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7216	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
7217	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7218	}
7219
7220
7221	/**
7222	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
7223	*/
7224	HMVMX_EXIT_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7225	{
7226	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7227	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
7228	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7229	}
7230
7231
7232	/**
7233	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
7234	*/
7235	HMVMX_EXIT_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7236	{
7237	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7238	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
7239	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7240	}
7241
7242
7243	/**
7244	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
7245	*/
7246	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7247	{
7248	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7249	PVM pVM = pVCpu->CTX_SUFF(pVM);
7250	int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7251	if (RT_LIKELY(rc == VINF_SUCCESS))
7252	{
7253	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7254	Assert(pVmxTransient->cbInstr == 2);
7255	}
7256	else
7257	{
7258	AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
7259	rc = VERR_EM_INTERPRETER;
7260	}
7261	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
7262	return rc;
7263	}
7264
7265
7266	/**
7267	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
7268	*/
7269	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7270	{
7271	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7272	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
7273	AssertRCReturn(rc, rc);
7274
7275	if (pMixedCtx->cr4 & X86_CR4_SMXE)
7276	return VINF_EM_RAW_EMULATE_INSTR;
7277
7278	AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
7279	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7280	}
7281
7282
7283	/**
7284	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
7285	*/
7286	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7287	{
7288	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7289	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
7290	AssertRCReturn(rc, rc);
7291
7292	PVM pVM = pVCpu->CTX_SUFF(pVM);
7293	rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7294	if (RT_LIKELY(rc == VINF_SUCCESS))
7295	{
7296	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7297	Assert(pVmxTransient->cbInstr == 2);
7298	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
7299	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
7300	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7301	}
7302	else
7303	{
7304	AssertMsgFailed(("hmR0VmxExitRdtsc: EMInterpretRdtsc failed with %Rrc\n", rc));
7305	rc = VERR_EM_INTERPRETER;
7306	}
7307	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
7308	return rc;
7309	}
7310
7311
7312	/**
7313	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
7314	*/
7315	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7316	{
7317	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7318	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
7319	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
7320	AssertRCReturn(rc, rc);
7321
7322	PVM pVM = pVCpu->CTX_SUFF(pVM);
7323	rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
7324	if (RT_LIKELY(rc == VINF_SUCCESS))
7325	{
7326	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7327	Assert(pVmxTransient->cbInstr == 3);
7328	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
7329	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
7330	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7331	}
7332	else
7333	{
7334	AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
7335	rc = VERR_EM_INTERPRETER;
7336	}
7337	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
7338	return rc;
7339	}
7340
7341
7342	/**
7343	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
7344	*/
7345	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7346	{
7347	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7348	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
7349	rc \|= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
7350	AssertRCReturn(rc, rc);
7351
7352	PVM pVM = pVCpu->CTX_SUFF(pVM);
7353	rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7354	if (RT_LIKELY(rc == VINF_SUCCESS))
7355	{
7356	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7357	Assert(pVmxTransient->cbInstr == 2);
7358	}
7359	else
7360	{
7361	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
7362	rc = VERR_EM_INTERPRETER;
7363	}
7364	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
7365	return rc;
7366	}
7367
7368
7369	/**
7370	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
7371	*/
7372	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7373	{
7374	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7375	PVM pVM = pVCpu->CTX_SUFF(pVM);
7376	Assert(!pVM->hm.s.fNestedPaging);
7377
7378	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
7379	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
7380	AssertRCReturn(rc, rc);
7381
7382	VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
7383	rc = VBOXSTRICTRC_VAL(rc2);
7384	if (RT_LIKELY(rc == VINF_SUCCESS))
7385	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7386	else
7387	{
7388	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
7389	pVmxTransient->uExitQualification, rc));
7390	}
7391	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
7392	return rc;
7393	}
7394
7395
7396	/**
7397	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
7398	*/
7399	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7400	{
7401	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7402	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7403	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7404	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7405	AssertRCReturn(rc, rc);
7406
7407	PVM pVM = pVCpu->CTX_SUFF(pVM);
7408	rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7409	if (RT_LIKELY(rc == VINF_SUCCESS))
7410	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7411	else
7412	{
7413	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
7414	rc = VERR_EM_INTERPRETER;
7415	}
7416	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
7417	return rc;
7418	}
7419
7420
7421	/**
7422	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
7423	*/
7424	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7425	{
7426	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7427	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7428	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7429	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7430	AssertRCReturn(rc, rc);
7431
7432	PVM pVM = pVCpu->CTX_SUFF(pVM);
7433	VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7434	rc = VBOXSTRICTRC_VAL(rc2);
7435	if (RT_LIKELY( rc == VINF_SUCCESS
7436	\|\| rc == VINF_EM_HALT))
7437	{
7438	int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7439	AssertRCReturn(rc3, rc3);
7440
7441	if ( rc == VINF_EM_HALT
7442	&& EMShouldContinueAfterHalt(pVCpu, pMixedCtx))
7443	{
7444	rc = VINF_SUCCESS;
7445	}
7446	}
7447	else
7448	{
7449	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
7450	rc = VERR_EM_INTERPRETER;
7451	}
7452	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_EM_HALT \|\| rc == VERR_EM_INTERPRETER,
7453	("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
7454	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
7455	return rc;
7456	}
7457
7458
7459	/**
7460	* VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
7461	*/
7462	HMVMX_EXIT_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7463	{
7464	/*
7465	* Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
7466	* get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
7467	* executing VMCALL in VMX root operation. If we get here something funny is going on.
7468	* See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
7469	*/
7470	AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7471	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7472	}
7473
7474
7475	/**
7476	* VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
7477	*/
7478	HMVMX_EXIT_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7479	{
7480	/*
7481	* This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
7482	* root operation. If we get there there is something funny going on.
7483	* See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
7484	*/
7485	AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7486	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7487	}
7488
7489
7490	/**
7491	* VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
7492	*/
7493	HMVMX_EXIT_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7494	{
7495	/* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
7496	AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7497	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7498	}
7499
7500
7501	/**
7502	* VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
7503	*/
7504	HMVMX_EXIT_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7505	{
7506	/*
7507	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
7508	* don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
7509	* See Intel spec. 25.3 "Other Causes of VM-exits".
7510	*/
7511	AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7512	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7513	}
7514
7515
7516	/**
7517	* VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
7518	* VM-exit.
7519	*/
7520	HMVMX_EXIT_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7521	{
7522	/*
7523	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM. See Intel spec. "33.14.1 Default Treatment of
7524	* SMI Delivery" and "29.3 VMX Instructions" for "VMXON". It is -NOT- blocked in VMX non-root operation so we can potentially
7525	* still get these exits. See Intel spec. "23.8 Restrictions on VMX operation".
7526	*/
7527	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7528	return VINF_SUCCESS; /** @todo r=ramshankar: correct?. */
7529	}
7530
7531
7532	/**
7533	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
7534	* VM-exit.
7535	*/
7536	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7537	{
7538	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7539	return VINF_EM_RESET;
7540	}
7541
7542
7543	/**
7544	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
7545	*/
7546	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7547	{
7548	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7549	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
7550	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
7551	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7552	AssertRCReturn(rc, rc);
7553
7554	pMixedCtx->rip++;
7555	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7556	if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
7557	rc = VINF_SUCCESS;
7558	else
7559	rc = VINF_EM_HALT;
7560
7561	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
7562	return rc;
7563	}
7564
7565
7566	/**
7567	* VM-exit handler for instructions that result in a #UD exception delivered to the guest.
7568	*/
7569	HMVMX_EXIT_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7570	{
7571	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7572	hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
7573	return VINF_SUCCESS;
7574	}
7575
7576
7577	/**
7578	* VM-exit handler for expiry of the VMX preemption timer.
7579	*/
7580	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7581	{
7582	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7583
7584	/* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
7585	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7586
7587	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
7588	PVM pVM = pVCpu->CTX_SUFF(pVM);
7589	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
7590	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
7591	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
7592	}
7593
7594
7595	/**
7596	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
7597	*/
7598	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7599	{
7600	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7601
7602	/* We expose XSETBV to the guest, fallback to the recompiler for emulation. */
7603	/** @todo check if XSETBV is supported by the recompiler. */
7604	return VERR_EM_INTERPRETER;
7605	}
7606
7607
7608	/**
7609	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
7610	*/
7611	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7612	{
7613	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7614
7615	/* The guest should not invalidate the host CPU's TLBs, fallback to recompiler. */
7616	/** @todo implement EMInterpretInvpcid() */
7617	return VERR_EM_INTERPRETER;
7618	}
7619
7620
7621	/**
7622	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
7623	* Error VM-exit.
7624	*/
7625	HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7626	{
7627	uint32_t uIntrState;
7628	HMVMXHCUINTREG uHCReg;
7629	uint64_t u64Val;
7630	uint32_t u32Val;
7631
7632	int rc = hmR0VmxReadEntryIntrInfoVmcs(pVmxTransient);
7633	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
7634	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVCpu, pVmxTransient);
7635	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
7636	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7637	AssertRCReturn(rc, rc);
7638
7639	Log(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntrInfo));
7640	Log(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
7641	Log(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
7642	Log(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
7643
7644	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
7645	Log(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
7646	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
7647	Log(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
7648	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
7649	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
7650	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
7651	Log(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
7652	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
7653	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
7654	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
7655	Log(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
7656
7657	PVM pVM = pVCpu->CTX_SUFF(pVM);
7658	HMDumpRegs(pVM, pVCpu, pMixedCtx);
7659
7660	return VERR_VMX_INVALID_GUEST_STATE;
7661	}
7662
7663
7664	/**
7665	* VM-exit handler for VM-entry failure due to an MSR-load
7666	* (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
7667	*/
7668	HMVMX_EXIT_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7669	{
7670	AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7671	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7672	}
7673
7674
7675	/**
7676	* VM-exit handler for VM-entry failure due to a machine-check event
7677	* (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
7678	*/
7679	HMVMX_EXIT_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7680	{
7681	AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7682	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7683	}
7684
7685
7686	/**
7687	* VM-exit handler for all undefined reasons. Should never ever happen.. in
7688	* theory.
7689	*/
7690	HMVMX_EXIT_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7691	{
7692	AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
7693	return VERR_VMX_UNDEFINED_EXIT_CODE;
7694	}
7695
7696
7697	/**
7698	* VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
7699	* (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
7700	* Conditional VM-exit.
7701	*/
7702	HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7703	{
7704	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7705
7706	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
7707	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
7708	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
7709	return VERR_EM_INTERPRETER;
7710	AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7711	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7712	}
7713
7714
7715	/**
7716	* VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
7717	*/
7718	HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7719	{
7720	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7721
7722	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
7723	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
7724	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
7725	return VERR_EM_INTERPRETER;
7726	AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7727	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7728	}
7729
7730
7731	/**
7732	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
7733	*/
7734	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7735	{
7736	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7737
7738	/* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
7739	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7740	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7741	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7742	AssertRCReturn(rc, rc);
7743
7744	PVM pVM = pVCpu->CTX_SUFF(pVM);
7745	rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7746	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER,
7747	("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
7748	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
7749
7750	if (RT_LIKELY(rc == VINF_SUCCESS))
7751	{
7752	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7753	Assert(pVmxTransient->cbInstr == 2);
7754	}
7755	return rc;
7756	}
7757
7758
7759	/**
7760	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
7761	*/
7762	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7763	{
7764	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7765	PVM pVM = pVCpu->CTX_SUFF(pVM);
7766	int rc = VINF_SUCCESS;
7767
7768	/* If TPR patching is active, LSTAR holds the guest TPR, writes to it must be propagated to the APIC. */
7769	if ( pVM->hm.s.fTPRPatchingActive
7770	&& pMixedCtx->ecx == MSR_K8_LSTAR)
7771	{
7772	Assert(!CPUMIsGuestInLongModeEx(pMixedCtx)); /* Requires EFER but it's always up-to-date. */
7773	if ((pMixedCtx->eax & 0xff) != pVmxTransient->u8GuestTpr)
7774	{
7775	rc = PDMApicSetTPR(pVCpu, pMixedCtx->eax & 0xff);
7776	AssertRC(rc);
7777	}
7778
7779	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7780	Assert(pVmxTransient->cbInstr == 2);
7781	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
7782	return VINF_SUCCESS;
7783	}
7784
7785	/* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
7786	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7787	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7788	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7789	AssertRCReturn(rc, rc);
7790	Log(("ecx=%#RX32\n", pMixedCtx->ecx));
7791
7792	rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7793	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
7794	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
7795
7796	if (RT_LIKELY(rc == VINF_SUCCESS))
7797	{
7798	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7799
7800	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
7801	if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
7802	&& pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
7803	{
7804	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_APIC_STATE);
7805	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
7806	}
7807	else if (pMixedCtx->ecx == MSR_K6_EFER) /* EFER is the only MSR we auto-load but don't allow write-passthrough. */
7808	{
7809	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
7810	AssertRCReturn(rc, rc);
7811	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_AUTO_MSRS;
7812	}
7813	else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
7814	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7815
7816	/* Update MSRs that are part of the VMCS when MSR-bitmaps are not supported. */
7817	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
7818	{
7819	switch (pMixedCtx->ecx)
7820	{
7821	case MSR_IA32_SYSENTER_CS: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_CS_MSR; break;
7822	case MSR_IA32_SYSENTER_EIP: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_EIP_MSR; break;
7823	case MSR_IA32_SYSENTER_ESP: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_ESP_MSR; break;
7824	case MSR_K8_FS_BASE: /* no break */
7825	case MSR_K8_GS_BASE: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SEGMENT_REGS; break;
7826	case MSR_K8_KERNEL_GS_BASE: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_AUTO_MSRS; break;
7827	}
7828	}
7829	#ifdef VBOX_STRICT
7830	else
7831	{
7832	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
7833	switch (pMixedCtx->ecx)
7834	{
7835	case MSR_IA32_SYSENTER_CS:
7836	case MSR_IA32_SYSENTER_EIP:
7837	case MSR_IA32_SYSENTER_ESP:
7838	case MSR_K8_FS_BASE:
7839	case MSR_K8_GS_BASE:
7840	{
7841	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
7842	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7843	}
7844
7845	case MSR_K8_LSTAR:
7846	case MSR_K6_STAR:
7847	case MSR_K8_SF_MASK:
7848	case MSR_K8_TSC_AUX:
7849	case MSR_K8_KERNEL_GS_BASE:
7850	{
7851	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
7852	pMixedCtx->ecx));
7853	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7854	}
7855	}
7856	}
7857	#endif /* VBOX_STRICT */
7858	}
7859	return rc;
7860	}
7861
7862
7863	/**
7864	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
7865	*/
7866	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7867	{
7868	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7869
7870	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT. */
7871	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
7872	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT)
7873	return VERR_EM_INTERPRETER;
7874	AssertMsgFailed(("Unexpected PAUSE exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7875	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7876	}
7877
7878
7879	/**
7880	* VM-exit handler for when the TPR value is lowered below the specified
7881	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
7882	*/
7883	HMVMX_EXIT_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7884	{
7885	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7886	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
7887
7888	/*
7889	* The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
7890	* the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and
7891	* resume guest execution.
7892	*/
7893	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
7894	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
7895	return VINF_SUCCESS;
7896	}
7897
7898
7899	/**
7900	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
7901	* VM-exit.
7902	*
7903	* @retval VINF_SUCCESS when guest execution can continue.
7904	* @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
7905	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
7906	* @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
7907	* recompiler.
7908	*/
7909	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7910	{
7911	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7912	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
7913	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
7914	AssertRCReturn(rc, rc);
7915
7916	const RTGCUINTPTR uExitQualification = pVmxTransient->uExitQualification;
7917	const uint32_t uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
7918	PVM pVM = pVCpu->CTX_SUFF(pVM);
7919	switch (uAccessType)
7920	{
7921	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
7922	{
7923	#if 0
7924	/* EMInterpretCRxWrite() references a lot of guest state (EFER, RFLAGS, Segment Registers, etc.) Sync entire state */
7925	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7926	#else
7927	rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
7928	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
7929	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7930	#endif
7931	AssertRCReturn(rc, rc);
7932
7933	rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
7934	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
7935	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
7936	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3);
7937
7938	switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
7939	{
7940	case 0: /* CR0 */
7941	Log(("CRX CR0 write rc=%d CR0=%#RX64\n", rc, pMixedCtx->cr0));
7942	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
7943	break;
7944	case 2: /* C2 **/
7945	/* Nothing to do here, CR2 it's not part of the VMCS. */
7946	break;
7947	case 3: /* CR3 */
7948	Assert(!pVM->hm.s.fNestedPaging \|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx));
7949	Log(("CRX CR3 write rc=%d CR3=%#RX64\n", rc, pMixedCtx->cr3));
7950	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR3;
7951	break;
7952	case 4: /* CR4 */
7953	Log(("CRX CR4 write rc=%d CR4=%#RX64\n", rc, pMixedCtx->cr4));
7954	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR4;
7955	break;
7956	case 8: /* CR8 */
7957	Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
7958	/* CR8 contains the APIC TPR. Was updated by EMInterpretCRxWrite(). */
7959	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
7960	break;
7961	default:
7962	AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
7963	break;
7964	}
7965
7966	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
7967	break;
7968	}
7969
7970	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
7971	{
7972	/* EMInterpretCRxRead() requires EFER MSR, CS. */
7973	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7974	AssertRCReturn(rc, rc);
7975	Assert( !pVM->hm.s.fNestedPaging
7976	\|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx)
7977	\|\| VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
7978
7979	/* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
7980	Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
7981	\|\| !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
7982
7983	rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
7984	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
7985	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
7986	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
7987	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
7988	Log(("CRX CR%d Read access rc=%d\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification), rc));
7989	break;
7990	}
7991
7992	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
7993	{
7994	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7995	AssertRCReturn(rc, rc);
7996	rc = EMInterpretCLTS(pVM, pVCpu);
7997	AssertRCReturn(rc, rc);
7998	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
7999	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
8000	Log(("CRX CLTS write rc=%d\n", rc));
8001	break;
8002	}
8003
8004	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
8005	{
8006	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8007	AssertRCReturn(rc, rc);
8008	rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
8009	if (RT_LIKELY(rc == VINF_SUCCESS))
8010	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
8011	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
8012	Log(("CRX LMSW write rc=%d\n", rc));
8013	break;
8014	}
8015
8016	default:
8017	{
8018	AssertMsgFailed(("Invalid access-type in Mov CRx exit qualification %#x\n", uAccessType));
8019	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
8020	}
8021	}
8022
8023	/* Validate possible error codes. */
8024	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_SYNC_CR3
8025	\|\| rc == VERR_VMX_UNEXPECTED_EXCEPTION);
8026	if (RT_SUCCESS(rc))
8027	{
8028	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8029	AssertRCReturn(rc2, rc2);
8030	}
8031
8032	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
8033	return rc;
8034	}
8035
8036
8037	/**
8038	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
8039	* VM-exit.
8040	*/
8041	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8042	{
8043	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8044	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
8045
8046	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8047	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8048	rc \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
8049	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
8050	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
8051	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
8052	/* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
8053	AssertRCReturn(rc, rc);
8054	Log(("CS:RIP=%04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
8055
8056	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
8057	uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
8058	uint32_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
8059	bool fIOWrite = (VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
8060	== VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
8061	bool fIOString = (VMX_EXIT_QUALIFICATION_IO_STRING(pVmxTransient->uExitQualification) == 1);
8062	Assert(uIOWidth == 0 \|\| uIOWidth == 1 \|\| uIOWidth == 3);
8063
8064	/* I/O operation lookup arrays. */
8065	static const uint32_t s_aIOSize[4] = { 1, 2, 0, 4 }; /* Size of the I/O Accesses. */
8066	static const uint32_t s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
8067
8068	const uint32_t cbSize = s_aIOSize[uIOWidth];
8069	const uint32_t cbInstr = pVmxTransient->cbInstr;
8070	PVM pVM = pVCpu->CTX_SUFF(pVM);
8071	if (fIOString)
8072	{
8073	/* INS/OUTS - I/O String instruction. */
8074	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
8075	/** @todo for now manually disassemble later optimize by getting the fields from
8076	* the VMCS. */
8077	/** @todo VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR contains the flat pointer
8078	* operand of the instruction. VMX_VMCS32_RO_EXIT_INSTR_INFO contains
8079	* segment prefix info. */
8080	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL);
8081	if (RT_SUCCESS(rc))
8082	{
8083	if (fIOWrite)
8084	{
8085	VBOXSTRICTRC rc2 = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
8086	(DISCPUMODE)pDis->uAddrMode, cbSize);
8087	rc = VBOXSTRICTRC_VAL(rc2);
8088	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
8089	}
8090	else
8091	{
8092	VBOXSTRICTRC rc2 = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
8093	(DISCPUMODE)pDis->uAddrMode, cbSize);
8094	rc = VBOXSTRICTRC_VAL(rc2);
8095	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
8096	}
8097	}
8098	else
8099	{
8100	AssertMsg(rc == VERR_EM_INTERPRETER, ("rc=%Rrc RIP %#RX64\n", rc, pMixedCtx->rip));
8101	rc = VINF_EM_RAW_EMULATE_INSTR;
8102	}
8103	}
8104	else
8105	{
8106	/* IN/OUT - I/O instruction. */
8107	const uint32_t uAndVal = s_aIOOpAnd[uIOWidth];
8108	Assert(!VMX_EXIT_QUALIFICATION_IO_REP(pVmxTransient->uExitQualification));
8109	if (fIOWrite)
8110	{
8111	VBOXSTRICTRC rc2 = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbSize);
8112	rc = VBOXSTRICTRC_VAL(rc2);
8113	if (rc == VINF_IOM_R3_IOPORT_WRITE)
8114	HMR0SavePendingIOPortWrite(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbSize);
8115	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
8116	}
8117	else
8118	{
8119	uint32_t u32Result = 0;
8120	VBOXSTRICTRC rc2 = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbSize);
8121	rc = VBOXSTRICTRC_VAL(rc2);
8122	if (IOM_SUCCESS(rc))
8123	{
8124	/* Save result of I/O IN instr. in AL/AX/EAX. */
8125	pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
8126	}
8127	else if (rc == VINF_IOM_R3_IOPORT_READ)
8128	HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbSize);
8129	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
8130	}
8131	}
8132
8133	if (IOM_SUCCESS(rc))
8134	{
8135	pMixedCtx->rip += cbInstr;
8136	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
8137	if (RT_LIKELY(rc == VINF_SUCCESS))
8138	{
8139	rc = hmR0VmxSaveGuestDebugRegs(pVCpu, pMixedCtx); /* For DR7. */
8140	AssertRCReturn(rc, rc);
8141
8142	/* If any IO breakpoints are armed, then we should check if a debug trap needs to be generated. */
8143	if (pMixedCtx->dr[7] & X86_DR7_ENABLED_MASK)
8144	{
8145	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
8146	for (unsigned i = 0; i < 4; i++)
8147	{
8148	uint32_t uBPLen = s_aIOSize[X86_DR7_GET_LEN(pMixedCtx->dr[7], i)];
8149	if ( ( uIOPort >= pMixedCtx->dr[i]
8150	&& uIOPort < pMixedCtx->dr[i] + uBPLen)
8151	&& (pMixedCtx->dr[7] & (X86_DR7_L(i) \| X86_DR7_G(i)))
8152	&& (pMixedCtx->dr[7] & X86_DR7_RW(i, X86_DR7_RW_IO)) == X86_DR7_RW(i, X86_DR7_RW_IO))
8153	{
8154	Assert(CPUMIsGuestDebugStateActive(pVCpu));
8155	uint64_t uDR6 = ASMGetDR6();
8156
8157	/* Clear all breakpoint status flags and set the one we just hit. */
8158	uDR6 &= ~(X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3);
8159	uDR6 \|= (uint64_t)RT_BIT(i);
8160
8161	/*
8162	* Note: AMD64 Architecture Programmer's Manual 13.1:
8163	* Bits 15:13 of the DR6 register is never cleared by the processor and must
8164	* be cleared by software after the contents have been read.
8165	*/
8166	ASMSetDR6(uDR6);
8167
8168	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
8169	pMixedCtx->dr[7] &= ~X86_DR7_GD;
8170
8171	/* Paranoia. */
8172	pMixedCtx->dr[7] &= UINT64_C(0xffffffff); /* Upper 32 bits MBZ. */
8173	pMixedCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* MBZ. */
8174	pMixedCtx->dr[7] \|= 0x400; /* MB1. */
8175
8176	/* Resync DR7 */
8177	/** @todo probably cheaper to just reload DR7, nothing else needs changing. */
8178	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
8179
8180	/* Set #DB to be injected into the VM and continue guest execution. */
8181	hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
8182	break;
8183	}
8184	}
8185	}
8186	}
8187	}
8188
8189	#ifdef DEBUG
8190	if (rc == VINF_IOM_R3_IOPORT_READ)
8191	Assert(!fIOWrite);
8192	else if (rc == VINF_IOM_R3_IOPORT_WRITE)
8193	Assert(fIOWrite);
8194	else
8195	{
8196	AssertMsg( RT_FAILURE(rc)
8197	\|\| rc == VINF_SUCCESS
8198	\|\| rc == VINF_EM_RAW_EMULATE_INSTR
8199	\|\| rc == VINF_EM_RAW_GUEST_TRAP
8200	\|\| rc == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", rc));
8201	}
8202	#endif
8203
8204	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
8205	return rc;
8206	}
8207
8208
8209	/**
8210	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
8211	* VM-exit.
8212	*/
8213	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8214	{
8215	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8216
8217	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
8218	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8219	AssertRCReturn(rc, rc);
8220	if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
8221	{
8222	rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
8223	AssertRCReturn(rc, rc);
8224	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
8225	{
8226	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
8227	/* Software interrupts and exceptions will be regenerated when the recompiler restarts the instruction. */
8228	if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
8229	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
8230	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
8231	{
8232	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
8233	bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
8234
8235	/* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
8236	Assert(!pVCpu->hm.s.Event.fPending);
8237	pVCpu->hm.s.Event.fPending = true;
8238	pVCpu->hm.s.Event.u64IntrInfo = pVmxTransient->uIdtVectoringInfo;
8239	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
8240	AssertRCReturn(rc, rc);
8241	if (fErrorCodeValid)
8242	pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
8243	else
8244	pVCpu->hm.s.Event.u32ErrCode = 0;
8245	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
8246	&& uVector == X86_XCPT_PF)
8247	{
8248	pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
8249	}
8250	Log(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
8251	}
8252	}
8253	}
8254	/** @todo Emulate task switch someday, currently just going back to ring-3 for
8255	* emulation. */
8256	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
8257	return VERR_EM_INTERPRETER;
8258	}
8259
8260
8261	/**
8262	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
8263	*/
8264	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8265	{
8266	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8267	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
8268	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
8269	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
8270	AssertRCReturn(rc, rc);
8271	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
8272	return VINF_EM_DBG_STOP;
8273	}
8274
8275
8276	/**
8277	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
8278	*/
8279	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8280	{
8281	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8282
8283	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8284	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
8285	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
8286	return VINF_SUCCESS;
8287	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8288	return rc;
8289
8290	#if 0
8291	/** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
8292	* just sync the whole thing. */
8293	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8294	#else
8295	/* Aggressive state sync. for now. */
8296	rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8297	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8298	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8299	#endif
8300	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8301	AssertRCReturn(rc, rc);
8302
8303	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
8304	uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
8305	switch (uAccessType)
8306	{
8307	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
8308	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
8309	{
8310	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8311	&& VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) == 0x80)
8312	{
8313	AssertMsgFailed(("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
8314	}
8315
8316	RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
8317	GCPhys &= PAGE_BASE_GC_MASK;
8318	GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
8319	PVM pVM = pVCpu->CTX_SUFF(pVM);
8320	Log(("ApicAccess uAccessType=%#x GCPhys=%#RGv Off=%#x\n", uAccessType, GCPhys,
8321	VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
8322
8323	VBOXSTRICTRC rc2 = IOMMMIOPhysHandler(pVM, pVCpu,
8324	(uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW,
8325	CPUMCTX2CORE(pMixedCtx), GCPhys);
8326	rc = VBOXSTRICTRC_VAL(rc2);
8327	Log(("ApicAccess rc=%d\n", rc));
8328	if ( rc == VINF_SUCCESS
8329	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8330	\|\| rc == VERR_PAGE_NOT_PRESENT)
8331	{
8332	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8333	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8334	rc = VINF_SUCCESS;
8335	}
8336	break;
8337	}
8338
8339	default:
8340	Log(("ApicAccess uAccessType=%#x\n", uAccessType));
8341	rc = VINF_EM_RAW_EMULATE_INSTR;
8342	break;
8343	}
8344
8345	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
8346	return rc;
8347	}
8348
8349
8350	/**
8351	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
8352	* VM-exit.
8353	*/
8354	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8355	{
8356	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8357
8358	/* We should -not- get this VM-exit if the guest is debugging. */
8359	if (CPUMIsGuestDebugStateActive(pVCpu))
8360	{
8361	AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
8362	return VERR_VMX_UNEXPECTED_EXIT_CODE;
8363	}
8364
8365	int rc = VERR_INTERNAL_ERROR_5;
8366	if ( !DBGFIsStepping(pVCpu)
8367	&& !CPUMIsHyperDebugStateActive(pVCpu))
8368	{
8369	/* Don't intercept MOV DRx. */
8370	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
8371	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
8372	AssertRCReturn(rc, rc);
8373
8374	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
8375	PVM pVM = pVCpu->CTX_SUFF(pVM);
8376	rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pMixedCtx, true /* include DR6 */);
8377	AssertRC(rc);
8378	Assert(CPUMIsGuestDebugStateActive(pVCpu));
8379
8380	#ifdef VBOX_WITH_STATISTICS
8381	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8382	AssertRCReturn(rc, rc);
8383	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
8384	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
8385	else
8386	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
8387	#endif
8388	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
8389	return VINF_SUCCESS;
8390	}
8391
8392	/*
8393	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date, see
8394	* hmR0VmxSaveGuestAutoLoadStoreMsrs(). Update only the segment registers from the CPU.
8395	*/
8396	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8397	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8398	AssertRCReturn(rc, rc);
8399
8400	PVM pVM = pVCpu->CTX_SUFF(pVM);
8401	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
8402	{
8403	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
8404	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
8405	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
8406	if (RT_SUCCESS(rc))
8407	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
8408	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
8409	}
8410	else
8411	{
8412	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
8413	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
8414	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
8415	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
8416	}
8417
8418	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
8419	if (RT_SUCCESS(rc))
8420	{
8421	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8422	AssertRCReturn(rc2, rc2);
8423	}
8424	return rc;
8425	}
8426
8427
8428	/**
8429	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
8430	* Conditional VM-exit.
8431	*/
8432	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8433	{
8434	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8435	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
8436
8437	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8438	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
8439	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
8440	return VINF_SUCCESS;
8441	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8442	return rc;
8443
8444	RTGCPHYS GCPhys = 0;
8445	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
8446
8447	#if 0
8448	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
8449	#else
8450	/* Aggressive state sync. for now. */
8451	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8452	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8453	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8454	#endif
8455	AssertRCReturn(rc, rc);
8456
8457	/*
8458	* If we succeed, resume guest execution.
8459	* If we fail in interpreting the instruction because we couldn't get the guest physical address
8460	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
8461	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
8462	* weird case. See @bugref{6043}.
8463	*/
8464	PVM pVM = pVCpu->CTX_SUFF(pVM);
8465	VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
8466	rc = VBOXSTRICTRC_VAL(rc2);
8467	Log(("EPT misconfig at %#RGv RIP=%#RX64 rc=%d\n", GCPhys, pMixedCtx->rip, rc));
8468	if ( rc == VINF_SUCCESS
8469	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8470	\|\| rc == VERR_PAGE_NOT_PRESENT)
8471	{
8472	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8473	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8474	return VINF_SUCCESS;
8475	}
8476	return rc;
8477	}
8478
8479
8480	/**
8481	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
8482	* VM-exit.
8483	*/
8484	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8485	{
8486	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8487	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
8488
8489	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8490	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
8491	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
8492	return VINF_SUCCESS;
8493	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8494	return rc;
8495
8496	RTGCPHYS GCPhys = 0;
8497	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
8498	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8499	#if 0
8500	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
8501	#else
8502	/* Aggressive state sync. for now. */
8503	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8504	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8505	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8506	#endif
8507	AssertRCReturn(rc, rc);
8508
8509	/* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
8510	AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
8511
8512	RTGCUINT uErrorCode = 0;
8513	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
8514	uErrorCode \|= X86_TRAP_PF_ID;
8515	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
8516	uErrorCode \|= X86_TRAP_PF_RW;
8517	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
8518	uErrorCode \|= X86_TRAP_PF_P;
8519
8520	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
8521
8522	Log(("EPT violation %#x at %#RX64 ErrorCode %#x CS:EIP=%04x:%#RX64\n", pVmxTransient->uExitQualification, GCPhys,
8523	uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
8524
8525	/* Handle the pagefault trap for the nested shadow table. */
8526	PVM pVM = pVCpu->CTX_SUFF(pVM);
8527	rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
8528	TRPMResetTrap(pVCpu);
8529
8530	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
8531	if ( rc == VINF_SUCCESS
8532	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8533	\|\| rc == VERR_PAGE_NOT_PRESENT)
8534	{
8535	/* Successfully synced our shadow page tables or emulation MMIO instruction. */
8536	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
8537	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8538	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8539	return VINF_SUCCESS;
8540	}
8541
8542	Log(("EPT return to ring-3 rc=%d\n"));
8543	return rc;
8544	}
8545
8546
8547	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8548	/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
8549	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8550	/**
8551	* VM-exit exception handler for #MF (Math Fault: floating point exception).
8552	*/
8553	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8554	{
8555	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8556	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
8557
8558	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8559	AssertRCReturn(rc, rc);
8560
8561	if (!(pMixedCtx->cr0 & X86_CR0_NE))
8562	{
8563	/* Old-style FPU error reporting needs some extra work. */
8564	/** @todo don't fall back to the recompiler, but do it manually. */
8565	return VERR_EM_INTERPRETER;
8566	}
8567	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8568	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8569	return rc;
8570	}
8571
8572
8573	/**
8574	* VM-exit exception handler for #BP (Breakpoint exception).
8575	*/
8576	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8577	{
8578	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8579	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
8580
8581	/** @todo Try optimize this by not saving the entire guest state unless
8582	* really needed. */
8583	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8584	AssertRCReturn(rc, rc);
8585
8586	PVM pVM = pVCpu->CTX_SUFF(pVM);
8587	rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8588	if (rc == VINF_EM_RAW_GUEST_TRAP)
8589	{
8590	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8591	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8592	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8593	AssertRCReturn(rc, rc);
8594
8595	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8596	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8597	}
8598
8599	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_EM_DBG_BREAKPOINT);
8600	return rc;
8601	}
8602
8603
8604	/**
8605	* VM-exit exception handler for #DB (Debug exception).
8606	*/
8607	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8608	{
8609	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8610	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
8611
8612	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8613	AssertRCReturn(rc, rc);
8614
8615	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
8616	uint64_t uDR6 = X86_DR6_INIT_VAL;
8617	uDR6 \|= (pVmxTransient->uExitQualification
8618	& (X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3 \| X86_DR6_BD \| X86_DR6_BS));
8619	PVM pVM = pVCpu->CTX_SUFF(pVM);
8620	rc = DBGFRZTrap01Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6);
8621	if (rc == VINF_EM_RAW_GUEST_TRAP)
8622	{
8623	/* DR6, DR7.GD and IA32_DEBUGCTL.LBR are not updated yet. See Intel spec. 27.1 "Architectural State before a VM-Exit". */
8624	pMixedCtx->dr[6] = uDR6;
8625
8626	if (CPUMIsGuestDebugStateActive(pVCpu))
8627	ASMSetDR6(pMixedCtx->dr[6]);
8628
8629	rc = hmR0VmxSaveGuestDebugRegs(pVCpu, pMixedCtx);
8630
8631	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
8632	pMixedCtx->dr[7] &= ~X86_DR7_GD;
8633
8634	/* Paranoia. */
8635	pMixedCtx->dr[7] &= UINT64_C(0xffffffff); /* upper 32 bits MBZ. */
8636	pMixedCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* must be zero */
8637	pMixedCtx->dr[7] \|= 0x400; /* must be one */
8638
8639	rc \|= VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
8640	AssertRCReturn(rc,rc);
8641
8642	int rc2 = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8643	rc2 \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8644	rc2 \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8645	AssertRCReturn(rc2, rc2);
8646	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8647	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8648	rc = VINF_SUCCESS;
8649	}
8650
8651	return rc;
8652	}
8653
8654
8655	/**
8656	* VM-exit exception handler for #NM (Device-not-available exception: floating
8657	* point exception).
8658	*/
8659	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8660	{
8661	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8662
8663	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
8664	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
8665	#endif
8666
8667	/* We require CR0 and EFER. EFER is always up-to-date. */
8668	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8669	AssertRCReturn(rc, rc);
8670
8671	/* Lazy FPU loading; load the guest-FPU state transparently and continue execution of the guest. */
8672	PVM pVM = pVCpu->CTX_SUFF(pVM);
8673	rc = CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8674	if (rc == VINF_SUCCESS)
8675	{
8676	Assert(CPUMIsGuestFPUStateActive(pVCpu));
8677	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
8678	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
8679	return VINF_SUCCESS;
8680	}
8681
8682	/* Forward #NM to the guest. */
8683	Assert(rc == VINF_EM_RAW_GUEST_TRAP);
8684	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8685	AssertRCReturn(rc, rc);
8686	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8687	pVmxTransient->cbInstr, 0 /* error code /, 0 / GCPtrFaultAddress */);
8688	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
8689	return rc;
8690	}
8691
8692
8693	/**
8694	* VM-exit exception handler for #GP (General-protection exception).
8695	*
8696	* @remarks Requires pVmxTransient->uExitIntrInfo to be up-to-date.
8697	*/
8698	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8699	{
8700	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8701	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
8702
8703	int rc = VERR_INTERNAL_ERROR_5;
8704	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8705	{
8706	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
8707	/* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
8708	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8709	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8710	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8711	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8712	AssertRCReturn(rc, rc);
8713	Log(("#GP Gst: RIP %#RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u\n", pMixedCtx->rip, pVmxTransient->uExitIntrErrorCode,
8714	pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu)));
8715	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8716	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8717	return rc;
8718	#else
8719	/* We don't intercept #GP. */
8720	AssertMsgFailed(("Unexpected VM-exit caused by #GP exception\n"));
8721	return VERR_VMX_UNEXPECTED_EXCEPTION;
8722	#endif
8723	}
8724
8725	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
8726	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
8727
8728	/* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
8729	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8730	AssertRCReturn(rc, rc);
8731
8732	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
8733	uint32_t cbOp = 0;
8734	PVM pVM = pVCpu->CTX_SUFF(pVM);
8735	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
8736	if (RT_SUCCESS(rc))
8737	{
8738	rc = VINF_SUCCESS;
8739	Assert(cbOp == pDis->cbInstr);
8740	Log(("#GP Disas OpCode=%u CS:EIP %04x:%#RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
8741	switch (pDis->pCurInstr->uOpcode)
8742	{
8743	case OP_CLI:
8744	pMixedCtx->eflags.Bits.u1IF = 0;
8745	pMixedCtx->rip += pDis->cbInstr;
8746	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS;
8747	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
8748	break;
8749
8750	case OP_STI:
8751	pMixedCtx->eflags.Bits.u1IF = 1;
8752	pMixedCtx->rip += pDis->cbInstr;
8753	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
8754	Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
8755	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS;
8756	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
8757	break;
8758
8759	case OP_HLT:
8760	rc = VINF_EM_HALT;
8761	pMixedCtx->rip += pDis->cbInstr;
8762	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
8763	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
8764	break;
8765
8766	case OP_POPF:
8767	{
8768	Log(("POPF CS:RIP %04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
8769	uint32_t cbParm = 0;
8770	uint32_t uMask = 0;
8771	if (pDis->fPrefix & DISPREFIX_OPSIZE)
8772	{
8773	cbParm = 4;
8774	uMask = 0xffffffff;
8775	}
8776	else
8777	{
8778	cbParm = 2;
8779	uMask = 0xffff;
8780	}
8781
8782	/* Get the stack pointer & pop the contents of the stack onto EFlags. */
8783	RTGCPTR GCPtrStack = 0;
8784	X86EFLAGS uEflags;
8785	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
8786	&GCPtrStack);
8787	if (RT_SUCCESS(rc))
8788	{
8789	Assert(sizeof(uEflags.u32) >= cbParm);
8790	uEflags.u32 = 0;
8791	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &uEflags.u32, cbParm);
8792	}
8793	if (RT_FAILURE(rc))
8794	{
8795	rc = VERR_EM_INTERPRETER;
8796	break;
8797	}
8798	Log(("POPF %x -> %#RX64 mask=%x RIP=%#RX64\n", uEflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
8799	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
8800	\| (uEflags.u32 & X86_EFL_POPF_BITS & uMask);
8801	/* The RF bit is always cleared by POPF; see Intel Instruction reference for POPF. */
8802	pMixedCtx->eflags.Bits.u1RF = 0;
8803	pMixedCtx->esp += cbParm;
8804	pMixedCtx->esp &= uMask;
8805	pMixedCtx->rip += pDis->cbInstr;
8806	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS;
8807	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
8808	break;
8809	}
8810
8811	case OP_PUSHF:
8812	{
8813	uint32_t cbParm = 0;
8814	uint32_t uMask = 0;
8815	if (pDis->fPrefix & DISPREFIX_OPSIZE)
8816	{
8817	cbParm = 4;
8818	uMask = 0xffffffff;
8819	}
8820	else
8821	{
8822	cbParm = 2;
8823	uMask = 0xffff;
8824	}
8825
8826	/* Get the stack pointer & push the contents of eflags onto the stack. */
8827	RTGCPTR GCPtrStack = 0;
8828	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
8829	SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
8830	if (RT_FAILURE(rc))
8831	{
8832	rc = VERR_EM_INTERPRETER;
8833	break;
8834	}
8835	X86EFLAGS uEflags;
8836	uEflags = pMixedCtx->eflags;
8837	/* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
8838	uEflags.Bits.u1RF = 0;
8839	uEflags.Bits.u1VM = 0;
8840
8841	rc = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &uEflags.u, cbParm);
8842	if (RT_FAILURE(rc))
8843	{
8844	rc = VERR_EM_INTERPRETER;
8845	break;
8846	}
8847	Log(("PUSHF %x -> %#RGv\n", uEflags.u, GCPtrStack));
8848	pMixedCtx->esp -= cbParm;
8849	pMixedCtx->esp &= uMask;
8850	pMixedCtx->rip += pDis->cbInstr;
8851	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP;
8852	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
8853	break;
8854	}
8855
8856	case OP_IRET:
8857	{
8858	/** @todo Handle 32-bit operand sizes and check stack limits. See Intel
8859	* instruction reference. */
8860	RTGCPTR GCPtrStack = 0;
8861	uint32_t uMask = 0xffff;
8862	uint16_t aIretFrame[3];
8863	if (pDis->fPrefix & (DISPREFIX_OPSIZE \| DISPREFIX_ADDRSIZE))
8864	{
8865	rc = VERR_EM_INTERPRETER;
8866	break;
8867	}
8868	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
8869	&GCPtrStack);
8870	if (RT_SUCCESS(rc))
8871	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
8872	if (RT_FAILURE(rc))
8873	{
8874	rc = VERR_EM_INTERPRETER;
8875	break;
8876	}
8877	pMixedCtx->eip = 0;
8878	pMixedCtx->ip = aIretFrame[0];
8879	pMixedCtx->cs.Sel = aIretFrame[1];
8880	pMixedCtx->cs.ValidSel = aIretFrame[1];
8881	pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
8882	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
8883	\| (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
8884	pMixedCtx->sp += sizeof(aIretFrame);
8885	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_SEGMENT_REGS \| HM_CHANGED_GUEST_RSP
8886	\| HM_CHANGED_GUEST_RFLAGS;
8887	Log(("IRET %#RX32 to %04x:%x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
8888	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
8889	break;
8890	}
8891
8892	case OP_INT:
8893	{
8894	uint16_t uVector = pDis->Param1.uValue & 0xff;
8895	hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
8896	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
8897	break;
8898	}
8899
8900	case OP_INTO:
8901	{
8902	if (pMixedCtx->eflags.Bits.u1OF)
8903	{
8904	hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
8905	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
8906	}
8907	break;
8908	}
8909
8910	default:
8911	{
8912	VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
8913	EMCODETYPE_SUPERVISOR);
8914	rc = VBOXSTRICTRC_VAL(rc2);
8915	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_ALL_GUEST;
8916	Log(("#GP rc=%Rrc\n", rc));
8917	break;
8918	}
8919	}
8920	}
8921	else
8922	rc = VERR_EM_INTERPRETER;
8923
8924	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_EM_HALT,
8925	("#GP Unexpected rc=%Rrc\n", rc));
8926	return rc;
8927	}
8928
8929
8930	/**
8931	* VM-exit exception handler wrapper for generic exceptions. Simply re-injects
8932	* the exception reported in the VMX transient structure back into the VM.
8933	*
8934	* @remarks Requires uExitIntrInfo in the VMX transient structure to be
8935	* up-to-date.
8936	*/
8937	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8938	{
8939	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8940
8941	/* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
8942	hmR0VmxCheckExitDueToEventDelivery(). */
8943	int rc = hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8944	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8945	AssertRCReturn(rc, rc);
8946	Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
8947	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8948	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8949	return VINF_SUCCESS;
8950	}
8951
8952
8953	/**
8954	* VM-exit exception handler for #PF (Page-fault exception).
8955	*/
8956	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8957	{
8958	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8959	PVM pVM = pVCpu->CTX_SUFF(pVM);
8960	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8961	rc \|= hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8962	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8963	AssertRCReturn(rc, rc);
8964
8965	#if defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) \|\| defined(HMVMX_ALWAYS_TRAP_PF)
8966	if (pVM->hm.s.fNestedPaging)
8967	{
8968	if (RT_LIKELY(!pVmxTransient->fVectoringPF))
8969	{
8970	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
8971	pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
8972	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8973	0 /* cbInstr */, pVmxTransient->uExitIntrErrorCode, pVmxTransient->uExitQualification);
8974	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
8975	}
8976	else
8977	{
8978	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
8979	pVCpu->hm.s.Event.fPending = false; /* A vectoring #PF. */
8980	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
8981	Log(("Pending #DF due to vectoring #PF. NP\n"));
8982	}
8983	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
8984	return rc;
8985	}
8986	#else
8987	Assert(!pVM->hm.s.fNestedPaging);
8988	#endif
8989
8990	#ifdef VBOX_HM_WITH_GUEST_PATCHING
8991	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8992	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8993	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8994	AssertRCReturn(rc, rc);
8995	/* Shortcut for APIC TPR access, only for 32-bit guests. */
8996	if ( pVM->hm.s.fTRPPatchingAllowed
8997	&& pVM->hm.s.pGuestPatchMem
8998	&& (pVmxTransient->uExitQualification & 0xfff) == 0x80 /* TPR offset */
8999	&& !(pVmxTransient->uExitIntrErrorCode & X86_TRAP_PF_P) /* Page not present */
9000	&& CPUMGetGuestCPL(pVCpu) == 0 /* Requires CR0, EFLAGS, segments. */
9001	&& !CPUMIsGuestInLongModeEx(pMixedCtx) /* Requires EFER. */
9002	&& pVM->hm.s.cPatches < RT_ELEMENTS(pVM->hm.s.aPatches))
9003	{
9004	RTGCPHYS GCPhys;
9005	RTGCPHYS GCPhysApicBase = (pMixedCtx->msrApicBase & PAGE_BASE_GC_MASK);
9006	rc = PGMGstGetPage(pVCpu, (RTGCPTR)pVmxTransient->uExitQualification, NULL /* pfFlags */, &GCPhys);
9007	if ( rc == VINF_SUCCESS
9008	&& GCPhys == GCPhysApicBase)
9009	{
9010	rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9011	AssertRCReturn(rc, rc);
9012
9013	/* Only attempt to patch the instruction once. */
9014	PHMTPRPATCH pPatch = (PHMTPRPATCH)RTAvloU32Get(&pVM->hm.s.PatchTree, (AVLOU32KEY)pMixedCtx->eip);
9015	if (!pPatch)
9016	return VINF_EM_HM_PATCH_TPR_INSTR;
9017	}
9018	}
9019	#endif
9020
9021	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9022	AssertRCReturn(rc, rc);
9023
9024	Log(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification, pMixedCtx->cs.Sel,
9025	pMixedCtx->rip, pVmxTransient->uExitIntrErrorCode, pMixedCtx->cr3));
9026
9027	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntrErrorCode);
9028	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntrErrorCode, CPUMCTX2CORE(pMixedCtx),
9029	(RTGCPTR)pVmxTransient->uExitQualification);
9030
9031	Log(("#PF: rc=%Rrc\n", rc));
9032	if (rc == VINF_SUCCESS)
9033	{
9034	/* Successfully synced shadow pages tables or emulated an MMIO instruction. */
9035	/** @todo this isn't quite right, what if guest does lgdt with some MMIO
9036	* memory? We don't update the whole state here... */
9037	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
9038	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
9039	TRPMResetTrap(pVCpu);
9040	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
9041	return rc;
9042	}
9043	else if (rc == VINF_EM_RAW_GUEST_TRAP)
9044	{
9045	if (!pVmxTransient->fVectoringPF)
9046	{
9047	/* It's a guest page fault and needs to be reflected to the guest. */
9048	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
9049	TRPMResetTrap(pVCpu);
9050	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
9051	pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
9052	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
9053	0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
9054	}
9055	else
9056	{
9057	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
9058	TRPMResetTrap(pVCpu);
9059	pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF for replace it with #DF. */
9060	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
9061	Log(("#PF: Pending #DF due to vectoring #PF\n"));
9062	}
9063
9064	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
9065	return VINF_SUCCESS;
9066	}
9067
9068	TRPMResetTrap(pVCpu);
9069	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
9070	return rc;
9071	}
9072

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 46150

Download in other formats: