HMVMXR0.cpp@ 70606

Last change on this file since 70606 was 70528, checked in by vboxsync, 7 years ago
VMM/HMVMXR0: Temporarily back out r120196 (needs other local commit to work which is under testing)
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 598.8 KB

Line
1	/* $Id: HMVMXR0.cpp 70528 2018-01-11 07:15:34Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012-2017 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*********************************************************************************************************************************
20	* Header Files *
21	*********************************************************************************************************************************/
22	#define LOG_GROUP LOG_GROUP_HM
23	#include <iprt/x86.h>
24	#include <iprt/asm-amd64-x86.h>
25	#include <iprt/thread.h>
26
27	#include <VBox/vmm/pdmapi.h>
28	#include <VBox/vmm/dbgf.h>
29	#include <VBox/vmm/iem.h>
30	#include <VBox/vmm/iom.h>
31	#include <VBox/vmm/selm.h>
32	#include <VBox/vmm/tm.h>
33	#include <VBox/vmm/gim.h>
34	#include <VBox/vmm/apic.h>
35	#ifdef VBOX_WITH_REM
36	# include <VBox/vmm/rem.h>
37	#endif
38	#include "HMInternal.h"
39	#include <VBox/vmm/vm.h>
40	#include "HMVMXR0.h"
41	#include "dtrace/VBoxVMM.h"
42
43	#define HMVMX_USE_IEM_EVENT_REFLECTION
44	#ifdef DEBUG_ramshankar
45	# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
46	# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
47	# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
48	# define HMVMX_ALWAYS_CHECK_GUEST_STATE
49	# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
50	# define HMVMX_ALWAYS_TRAP_PF
51	# define HMVMX_ALWAYS_SWAP_FPU_STATE
52	# define HMVMX_ALWAYS_FLUSH_TLB
53	# define HMVMX_ALWAYS_SWAP_EFER
54	#endif
55
56
57	/*********************************************************************************************************************************
58	* Defined Constants And Macros *
59	*********************************************************************************************************************************/
60	/** Use the function table. */
61	#define HMVMX_USE_FUNCTION_TABLE
62
63	/** Determine which tagged-TLB flush handler to use. */
64	#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
65	#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
66	#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
67	#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
68
69	/** @name Updated-guest-state flags.
70	* @{ */
71	#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
72	#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
73	#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
74	#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
75	#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
76	#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
77	#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
78	#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
79	#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
80	#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
81	#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
82	#define HMVMX_UPDATED_GUEST_DR7 RT_BIT(11)
83	#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(12)
84	#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(13)
85	#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(14)
86	#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(15)
87	#define HMVMX_UPDATED_GUEST_LAZY_MSRS RT_BIT(16)
88	#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(17)
89	#define HMVMX_UPDATED_GUEST_INTR_STATE RT_BIT(18)
90	#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(19)
91	#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
92	\| HMVMX_UPDATED_GUEST_RSP \
93	\| HMVMX_UPDATED_GUEST_RFLAGS \
94	\| HMVMX_UPDATED_GUEST_CR0 \
95	\| HMVMX_UPDATED_GUEST_CR3 \
96	\| HMVMX_UPDATED_GUEST_CR4 \
97	\| HMVMX_UPDATED_GUEST_GDTR \
98	\| HMVMX_UPDATED_GUEST_IDTR \
99	\| HMVMX_UPDATED_GUEST_LDTR \
100	\| HMVMX_UPDATED_GUEST_TR \
101	\| HMVMX_UPDATED_GUEST_SEGMENT_REGS \
102	\| HMVMX_UPDATED_GUEST_DR7 \
103	\| HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
104	\| HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
105	\| HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
106	\| HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
107	\| HMVMX_UPDATED_GUEST_LAZY_MSRS \
108	\| HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
109	\| HMVMX_UPDATED_GUEST_INTR_STATE \
110	\| HMVMX_UPDATED_GUEST_APIC_STATE)
111	/** @} */
112
113	/** @name
114	* Flags to skip redundant reads of some common VMCS fields that are not part of
115	* the guest-CPU state but are in the transient structure.
116	*/
117	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
118	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
119	#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
120	#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
121	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
122	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
123	#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO RT_BIT(6)
124	/** @} */
125
126	/** @name
127	* States of the VMCS.
128	*
129	* This does not reflect all possible VMCS states but currently only those
130	* needed for maintaining the VMCS consistently even when thread-context hooks
131	* are used. Maybe later this can be extended (i.e. Nested Virtualization).
132	*/
133	#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
134	#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
135	#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
136	/** @} */
137
138	/**
139	* Exception bitmap mask for real-mode guests (real-on-v86).
140	*
141	* We need to intercept all exceptions manually except:
142	* - \#NM, \#MF handled in hmR0VmxLoadSharedCR0().
143	* - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
144	* due to bugs in Intel CPUs.
145	* - \#PF need not be intercepted even in real-mode if we have Nested Paging
146	* support.
147	*/
148	#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: \| RT_BIT(X86_XCPT_DB) */ \| RT_BIT(X86_XCPT_NMI) \
149	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
150	\| RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ \| RT_BIT(X86_XCPT_DF) \
151	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
152	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
153	/* RT_BIT(X86_XCPT_MF) always: \| RT_BIT(X86_XCPT_AC) */ \| RT_BIT(X86_XCPT_MC) \
154	\| RT_BIT(X86_XCPT_XF))
155
156	/**
157	* Exception bitmap mask for all contributory exceptions.
158	*
159	* Page fault is deliberately excluded here as it's conditional as to whether
160	* it's contributory or benign. Page faults are handled separately.
161	*/
162	#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) \| RT_BIT(X86_XCPT_NP) \| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_TS) \
163	\| RT_BIT(X86_XCPT_DE))
164
165	/** Maximum VM-instruction error number. */
166	#define HMVMX_INSTR_ERROR_MAX 28
167
168	/** Profiling macro. */
169	#ifdef HM_PROFILE_EXIT_DISPATCH
170	# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
171	# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
172	#else
173	# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
174	# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
175	#endif
176
177	/** Assert that preemption is disabled or covered by thread-context hooks. */
178	#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
179	\|\| !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
180
181	/** Assert that we haven't migrated CPUs when thread-context hooks are not
182	* used. */
183	#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
184	\|\| pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
185	("Illegal migration! Entered on CPU %u Current %u\n", \
186	pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
187
188	/** Helper macro for VM-exit handlers called unexpectedly. */
189	#define HMVMX_RETURN_UNEXPECTED_EXIT() \
190	do { \
191	pVCpu->hm.s.u32HMError = pVmxTransient->uExitReason; \
192	return VERR_VMX_UNEXPECTED_EXIT; \
193	} while (0)
194
195
196	/*********************************************************************************************************************************
197	* Structures and Typedefs *
198	*********************************************************************************************************************************/
199	/**
200	* VMX transient state.
201	*
202	* A state structure for holding miscellaneous information across
203	* VMX non-root operation and restored after the transition.
204	*/
205	typedef struct VMXTRANSIENT
206	{
207	/** The host's rflags/eflags. */
208	RTCCUINTREG fEFlags;
209	#if HC_ARCH_BITS == 32
210	uint32_t u32Alignment0;
211	#endif
212	/** The guest's TPR value used for TPR shadowing. */
213	uint8_t u8GuestTpr;
214	/** Alignment. */
215	uint8_t abAlignment0[7];
216
217	/** The basic VM-exit reason. */
218	uint16_t uExitReason;
219	/** Alignment. */
220	uint16_t u16Alignment0;
221	/** The VM-exit interruption error code. */
222	uint32_t uExitIntErrorCode;
223	/** The VM-exit exit code qualification. */
224	uint64_t uExitQualification;
225
226	/** The VM-exit interruption-information field. */
227	uint32_t uExitIntInfo;
228	/** The VM-exit instruction-length field. */
229	uint32_t cbInstr;
230	/** The VM-exit instruction-information field. */
231	union
232	{
233	/** Plain unsigned int representation. */
234	uint32_t u;
235	/** INS and OUTS information. */
236	struct
237	{
238	uint32_t u7Reserved0 : 7;
239	/** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
240	uint32_t u3AddrSize : 3;
241	uint32_t u5Reserved1 : 5;
242	/** The segment register (X86_SREG_XXX). */
243	uint32_t iSegReg : 3;
244	uint32_t uReserved2 : 14;
245	} StrIo;
246	/** INVEPT, INVVPID, INVPCID information. */
247	struct
248	{
249	/** Scaling; 0=no scaling, 1=scale-by-2, 2=scale-by-4, 3=scale-by-8. */
250	uint32_t u2Scaling : 2;
251	uint32_t u5Reserved0 : 5;
252	/** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
253	uint32_t u3AddrSize : 3;
254	uint32_t u1Reserved0 : 1;
255	uint32_t u4Reserved0 : 4;
256	/** The segment register (X86_SREG_XXX). */
257	uint32_t iSegReg : 3;
258	/** The index register (X86_GREG_XXX). */
259	uint32_t iIdxReg : 4;
260	/** Set if index register is invalid. */
261	uint32_t fIdxRegValid : 1;
262	/** The base register (X86_GREG_XXX). */
263	uint32_t iBaseReg : 4;
264	/** Set if base register is invalid. */
265	uint32_t fBaseRegValid : 1;
266	/** Register 2 (X86_GREG_XXX). */
267	uint32_t iReg2 : 4;
268	} Inv;
269	} ExitInstrInfo;
270	/** Whether the VM-entry failed or not. */
271	bool fVMEntryFailed;
272	/** Alignment. */
273	uint8_t abAlignment1[3];
274
275	/** The VM-entry interruption-information field. */
276	uint32_t uEntryIntInfo;
277	/** The VM-entry exception error code field. */
278	uint32_t uEntryXcptErrorCode;
279	/** The VM-entry instruction length field. */
280	uint32_t cbEntryInstr;
281
282	/** IDT-vectoring information field. */
283	uint32_t uIdtVectoringInfo;
284	/** IDT-vectoring error code. */
285	uint32_t uIdtVectoringErrorCode;
286
287	/** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_. /
288	uint32_t fVmcsFieldsRead;
289
290	/** Whether the guest FPU was active at the time of VM-exit. */
291	bool fWasGuestFPUStateActive;
292	/** Whether the guest debug state was active at the time of VM-exit. */
293	bool fWasGuestDebugStateActive;
294	/** Whether the hyper debug state was active at the time of VM-exit. */
295	bool fWasHyperDebugStateActive;
296	/** Whether TSC-offsetting should be setup before VM-entry. */
297	bool fUpdateTscOffsettingAndPreemptTimer;
298	/** Whether the VM-exit was caused by a page-fault during delivery of a
299	* contributory exception or a page-fault. */
300	bool fVectoringDoublePF;
301	/** Whether the VM-exit was caused by a page-fault during delivery of an
302	* external interrupt or NMI. */
303	bool fVectoringPF;
304	} VMXTRANSIENT;
305	AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
306	AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
307	AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
308	AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestFPUStateActive, sizeof(uint64_t));
309	AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
310	/** Pointer to VMX transient state. */
311	typedef VMXTRANSIENT *PVMXTRANSIENT;
312
313
314	/**
315	* MSR-bitmap read permissions.
316	*/
317	typedef enum VMXMSREXITREAD
318	{
319	/** Reading this MSR causes a VM-exit. */
320	VMXMSREXIT_INTERCEPT_READ = 0xb,
321	/** Reading this MSR does not cause a VM-exit. */
322	VMXMSREXIT_PASSTHRU_READ
323	} VMXMSREXITREAD;
324	/** Pointer to MSR-bitmap read permissions. */
325	typedef VMXMSREXITREAD* PVMXMSREXITREAD;
326
327	/**
328	* MSR-bitmap write permissions.
329	*/
330	typedef enum VMXMSREXITWRITE
331	{
332	/** Writing to this MSR causes a VM-exit. */
333	VMXMSREXIT_INTERCEPT_WRITE = 0xd,
334	/** Writing to this MSR does not cause a VM-exit. */
335	VMXMSREXIT_PASSTHRU_WRITE
336	} VMXMSREXITWRITE;
337	/** Pointer to MSR-bitmap write permissions. */
338	typedef VMXMSREXITWRITE* PVMXMSREXITWRITE;
339
340
341	/**
342	* VMX VM-exit handler.
343	*
344	* @returns Strict VBox status code (i.e. informational status codes too).
345	* @param pVCpu The cross context virtual CPU structure.
346	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
347	* out-of-sync. Make sure to update the required
348	* fields before using them.
349	* @param pVmxTransient Pointer to the VMX-transient structure.
350	*/
351	#ifndef HMVMX_USE_FUNCTION_TABLE
352	typedef VBOXSTRICTRC FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
353	#else
354	typedef DECLCALLBACK(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
355	/** Pointer to VM-exit handler. */
356	typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
357	#endif
358
359	/**
360	* VMX VM-exit handler, non-strict status code.
361	*
362	* This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
363	*
364	* @returns VBox status code, no informational status code returned.
365	* @param pVCpu The cross context virtual CPU structure.
366	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
367	* out-of-sync. Make sure to update the required
368	* fields before using them.
369	* @param pVmxTransient Pointer to the VMX-transient structure.
370	*
371	* @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
372	* use of that status code will be replaced with VINF_EM_SOMETHING
373	* later when switching over to IEM.
374	*/
375	#ifndef HMVMX_USE_FUNCTION_TABLE
376	typedef int FNVMXEXITHANDLERNSRC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
377	#else
378	typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
379	#endif
380
381
382	/*********************************************************************************************************************************
383	* Internal Functions *
384	*********************************************************************************************************************************/
385	static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush);
386	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr);
387	static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu);
388	static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
389	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress,
390	bool fStepping, uint32_t *puIntState);
391	#if HC_ARCH_BITS == 32
392	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
393	#endif
394	#ifndef HMVMX_USE_FUNCTION_TABLE
395	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
396	# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
397	# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
398	#else
399	# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
400	# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
401	#endif
402
403
404	/** @name VM-exit handlers.
405	* @{
406	*/
407	static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
408	static FNVMXEXITHANDLER hmR0VmxExitExtInt;
409	static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
410	static FNVMXEXITHANDLERNSRC hmR0VmxExitInitSignal;
411	static FNVMXEXITHANDLERNSRC hmR0VmxExitSipi;
412	static FNVMXEXITHANDLERNSRC hmR0VmxExitIoSmi;
413	static FNVMXEXITHANDLERNSRC hmR0VmxExitSmi;
414	static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindow;
415	static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindow;
416	static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
417	static FNVMXEXITHANDLER hmR0VmxExitCpuid;
418	static FNVMXEXITHANDLER hmR0VmxExitGetsec;
419	static FNVMXEXITHANDLER hmR0VmxExitHlt;
420	static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
421	static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
422	static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
423	static FNVMXEXITHANDLER hmR0VmxExitVmcall;
424	static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
425	static FNVMXEXITHANDLERNSRC hmR0VmxExitRsm;
426	static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
427	static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
428	static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
429	static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
430	static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
431	static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
432	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
433	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMsrLoad;
434	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUndefined;
435	static FNVMXEXITHANDLER hmR0VmxExitMwait;
436	static FNVMXEXITHANDLER hmR0VmxExitMtf;
437	static FNVMXEXITHANDLER hmR0VmxExitMonitor;
438	static FNVMXEXITHANDLER hmR0VmxExitPause;
439	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMachineCheck;
440	static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThreshold;
441	static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
442	static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
443	static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
444	static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
445	static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
446	static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
447	static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
448	static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvd;
449	static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
450	static FNVMXEXITHANDLER hmR0VmxExitRdrand;
451	static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
452	/** @} */
453
454	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
455	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
456	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
457	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
458	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
459	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
460	static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
461	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
462	static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
463
464
465	/*********************************************************************************************************************************
466	* Global Variables *
467	*********************************************************************************************************************************/
468	#ifdef HMVMX_USE_FUNCTION_TABLE
469
470	/**
471	* VMX_EXIT dispatch table.
472	*/
473	static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
474	{
475	/* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
476	/* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
477	/* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
478	/* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
479	/* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
480	/* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
481	/* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
482	/* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
483	/* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
484	/* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
485	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
486	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
487	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
488	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
489	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
490	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
491	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
492	/* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
493	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
494	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
495	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
496	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
497	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
498	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
499	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
500	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
501	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
502	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
503	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
504	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
505	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
506	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
507	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
508	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
509	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
510	/* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
511	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
512	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
513	/* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
514	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
515	/* 40 UNDEFINED */ hmR0VmxExitPause,
516	/* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
517	/* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
518	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
519	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
520	/* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
521	/* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
522	/* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
523	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
524	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
525	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
526	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
527	/* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
528	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
529	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
530	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
531	/* 56 VMX_EXIT_APIC_WRITE */ hmR0VmxExitErrUndefined,
532	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
533	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
534	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD,
535	/* 60 VMX_EXIT_ENCLS */ hmR0VmxExitErrUndefined,
536	/* 61 VMX_EXIT_RDSEED / hmR0VmxExitErrUndefined, / only spurious exits, so undefined */
537	/* 62 VMX_EXIT_PML_FULL */ hmR0VmxExitErrUndefined,
538	/* 63 VMX_EXIT_XSAVES */ hmR0VmxExitSetPendingXcptUD,
539	/* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitSetPendingXcptUD,
540	};
541	#endif /* HMVMX_USE_FUNCTION_TABLE */
542
543	#ifdef VBOX_STRICT
544	static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
545	{
546	/* 0 */ "(Not Used)",
547	/* 1 */ "VMCALL executed in VMX root operation.",
548	/* 2 */ "VMCLEAR with invalid physical address.",
549	/* 3 */ "VMCLEAR with VMXON pointer.",
550	/* 4 */ "VMLAUNCH with non-clear VMCS.",
551	/* 5 */ "VMRESUME with non-launched VMCS.",
552	/* 6 */ "VMRESUME after VMXOFF",
553	/* 7 */ "VM-entry with invalid control fields.",
554	/* 8 */ "VM-entry with invalid host state fields.",
555	/* 9 */ "VMPTRLD with invalid physical address.",
556	/* 10 */ "VMPTRLD with VMXON pointer.",
557	/* 11 */ "VMPTRLD with incorrect revision identifier.",
558	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
559	/* 13 */ "VMWRITE to read-only VMCS component.",
560	/* 14 */ "(Not Used)",
561	/* 15 */ "VMXON executed in VMX root operation.",
562	/* 16 */ "VM-entry with invalid executive-VMCS pointer.",
563	/* 17 */ "VM-entry with non-launched executing VMCS.",
564	/* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
565	/* 19 */ "VMCALL with non-clear VMCS.",
566	/* 20 */ "VMCALL with invalid VM-exit control fields.",
567	/* 21 */ "(Not Used)",
568	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
569	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
570	/* 24 */ "VMCALL with invalid SMM-monitor features.",
571	/* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
572	/* 26 */ "VM-entry with events blocked by MOV SS.",
573	/* 27 */ "(Not Used)",
574	/* 28 */ "Invalid operand to INVEPT/INVVPID."
575	};
576	#endif /* VBOX_STRICT */
577
578
579
580	/**
581	* Updates the VM's last error record.
582	*
583	* If there was a VMX instruction error, reads the error data from the VMCS and
584	* updates VCPU's last error record as well.
585	*
586	* @param pVM The cross context VM structure.
587	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
588	* Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
589	* VERR_VMX_INVALID_VMCS_FIELD.
590	* @param rc The error code.
591	*/
592	static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
593	{
594	AssertPtr(pVM);
595	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
596	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
597	{
598	AssertPtrReturnVoid(pVCpu);
599	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
600	}
601	pVM->hm.s.lLastError = rc;
602	}
603
604
605	/**
606	* Reads the VM-entry interruption-information field from the VMCS into the VMX
607	* transient structure.
608	*
609	* @returns VBox status code.
610	* @param pVmxTransient Pointer to the VMX transient structure.
611	*
612	* @remarks No-long-jump zone!!!
613	*/
614	DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
615	{
616	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
617	AssertRCReturn(rc, rc);
618	return VINF_SUCCESS;
619	}
620
621
622	#ifdef VBOX_STRICT
623	/**
624	* Reads the VM-entry exception error code field from the VMCS into
625	* the VMX transient structure.
626	*
627	* @returns VBox status code.
628	* @param pVmxTransient Pointer to the VMX transient structure.
629	*
630	* @remarks No-long-jump zone!!!
631	*/
632	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
633	{
634	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
635	AssertRCReturn(rc, rc);
636	return VINF_SUCCESS;
637	}
638	#endif /* VBOX_STRICT */
639
640
641	#ifdef VBOX_STRICT
642	/**
643	* Reads the VM-entry exception error code field from the VMCS into
644	* the VMX transient structure.
645	*
646	* @returns VBox status code.
647	* @param pVmxTransient Pointer to the VMX transient structure.
648	*
649	* @remarks No-long-jump zone!!!
650	*/
651	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
652	{
653	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
654	AssertRCReturn(rc, rc);
655	return VINF_SUCCESS;
656	}
657	#endif /* VBOX_STRICT */
658
659
660	/**
661	* Reads the VM-exit interruption-information field from the VMCS into the VMX
662	* transient structure.
663	*
664	* @returns VBox status code.
665	* @param pVmxTransient Pointer to the VMX transient structure.
666	*/
667	DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
668	{
669	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
670	{
671	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
672	AssertRCReturn(rc, rc);
673	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
674	}
675	return VINF_SUCCESS;
676	}
677
678
679	/**
680	* Reads the VM-exit interruption error code from the VMCS into the VMX
681	* transient structure.
682	*
683	* @returns VBox status code.
684	* @param pVmxTransient Pointer to the VMX transient structure.
685	*/
686	DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
687	{
688	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
689	{
690	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
691	AssertRCReturn(rc, rc);
692	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
693	}
694	return VINF_SUCCESS;
695	}
696
697
698	/**
699	* Reads the VM-exit instruction length field from the VMCS into the VMX
700	* transient structure.
701	*
702	* @returns VBox status code.
703	* @param pVmxTransient Pointer to the VMX transient structure.
704	*/
705	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
706	{
707	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
708	{
709	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
710	AssertRCReturn(rc, rc);
711	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
712	}
713	return VINF_SUCCESS;
714	}
715
716
717	/**
718	* Reads the VM-exit instruction-information field from the VMCS into
719	* the VMX transient structure.
720	*
721	* @returns VBox status code.
722	* @param pVmxTransient Pointer to the VMX transient structure.
723	*/
724	DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
725	{
726	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO))
727	{
728	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
729	AssertRCReturn(rc, rc);
730	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO;
731	}
732	return VINF_SUCCESS;
733	}
734
735
736	/**
737	* Reads the exit code qualification from the VMCS into the VMX transient
738	* structure.
739	*
740	* @returns VBox status code.
741	* @param pVCpu The cross context virtual CPU structure of the
742	* calling EMT. (Required for the VMCS cache case.)
743	* @param pVmxTransient Pointer to the VMX transient structure.
744	*/
745	DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
746	{
747	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
748	{
749	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification); NOREF(pVCpu);
750	AssertRCReturn(rc, rc);
751	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
752	}
753	return VINF_SUCCESS;
754	}
755
756
757	/**
758	* Reads the IDT-vectoring information field from the VMCS into the VMX
759	* transient structure.
760	*
761	* @returns VBox status code.
762	* @param pVmxTransient Pointer to the VMX transient structure.
763	*
764	* @remarks No-long-jump zone!!!
765	*/
766	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
767	{
768	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
769	{
770	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
771	AssertRCReturn(rc, rc);
772	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
773	}
774	return VINF_SUCCESS;
775	}
776
777
778	/**
779	* Reads the IDT-vectoring error code from the VMCS into the VMX
780	* transient structure.
781	*
782	* @returns VBox status code.
783	* @param pVmxTransient Pointer to the VMX transient structure.
784	*/
785	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
786	{
787	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
788	{
789	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
790	AssertRCReturn(rc, rc);
791	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
792	}
793	return VINF_SUCCESS;
794	}
795
796
797	/**
798	* Enters VMX root mode operation on the current CPU.
799	*
800	* @returns VBox status code.
801	* @param pVM The cross context VM structure. Can be
802	* NULL, after a resume.
803	* @param HCPhysCpuPage Physical address of the VMXON region.
804	* @param pvCpuPage Pointer to the VMXON region.
805	*/
806	static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
807	{
808	Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
809	Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
810	Assert(pvCpuPage);
811	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
812
813	if (pVM)
814	{
815	/* Write the VMCS revision dword to the VMXON region. */
816	(uint32_t )pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
817	}
818
819	/* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
820	RTCCUINTREG fEFlags = ASMIntDisableFlags();
821
822	/* Enable the VMX bit in CR4 if necessary. */
823	RTCCUINTREG uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
824
825	/* Enter VMX root mode. */
826	int rc = VMXEnable(HCPhysCpuPage);
827	if (RT_FAILURE(rc))
828	{
829	if (!(uOldCr4 & X86_CR4_VMXE))
830	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
831
832	if (pVM)
833	pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
834	}
835
836	/* Restore interrupts. */
837	ASMSetFlags(fEFlags);
838	return rc;
839	}
840
841
842	/**
843	* Exits VMX root mode operation on the current CPU.
844	*
845	* @returns VBox status code.
846	*/
847	static int hmR0VmxLeaveRootMode(void)
848	{
849	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
850
851	/* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
852	RTCCUINTREG fEFlags = ASMIntDisableFlags();
853
854	/* If we're for some reason not in VMX root mode, then don't leave it. */
855	RTCCUINTREG uHostCR4 = ASMGetCR4();
856
857	int rc;
858	if (uHostCR4 & X86_CR4_VMXE)
859	{
860	/* Exit VMX root mode and clear the VMX bit in CR4. */
861	VMXDisable();
862	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
863	rc = VINF_SUCCESS;
864	}
865	else
866	rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
867
868	/* Restore interrupts. */
869	ASMSetFlags(fEFlags);
870	return rc;
871	}
872
873
874	/**
875	* Allocates and maps one physically contiguous page. The allocated page is
876	* zero'd out. (Used by various VT-x structures).
877	*
878	* @returns IPRT status code.
879	* @param pMemObj Pointer to the ring-0 memory object.
880	* @param ppVirt Where to store the virtual address of the
881	* allocation.
882	* @param pHCPhys Where to store the physical address of the
883	* allocation.
884	*/
885	DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
886	{
887	AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
888	AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
889	AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
890
891	int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
892	if (RT_FAILURE(rc))
893	return rc;
894	ppVirt = RTR0MemObjAddress(pMemObj);
895	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
896	ASMMemZero32(*ppVirt, PAGE_SIZE);
897	return VINF_SUCCESS;
898	}
899
900
901	/**
902	* Frees and unmaps an allocated physical page.
903	*
904	* @param pMemObj Pointer to the ring-0 memory object.
905	* @param ppVirt Where to re-initialize the virtual address of
906	* allocation as 0.
907	* @param pHCPhys Where to re-initialize the physical address of the
908	* allocation as 0.
909	*/
910	DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
911	{
912	AssertPtr(pMemObj);
913	AssertPtr(ppVirt);
914	AssertPtr(pHCPhys);
915	if (*pMemObj != NIL_RTR0MEMOBJ)
916	{
917	int rc = RTR0MemObjFree(pMemObj, true / fFreeMappings */);
918	AssertRC(rc);
919	*pMemObj = NIL_RTR0MEMOBJ;
920	*ppVirt = 0;
921	*pHCPhys = 0;
922	}
923	}
924
925
926	/**
927	* Worker function to free VT-x related structures.
928	*
929	* @returns IPRT status code.
930	* @param pVM The cross context VM structure.
931	*/
932	static void hmR0VmxStructsFree(PVM pVM)
933	{
934	for (VMCPUID i = 0; i < pVM->cCpus; i++)
935	{
936	PVMCPU pVCpu = &pVM->aCpus[i];
937	AssertPtr(pVCpu);
938
939	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
940	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
941
942	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
943	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
944
945	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
946	}
947
948	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
949	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
950	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
951	#endif
952	}
953
954
955	/**
956	* Worker function to allocate VT-x related VM structures.
957	*
958	* @returns IPRT status code.
959	* @param pVM The cross context VM structure.
960	*/
961	static int hmR0VmxStructsAlloc(PVM pVM)
962	{
963	/*
964	* Initialize members up-front so we can cleanup properly on allocation failure.
965	*/
966	#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
967	pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
968	pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
969	pVM->hm.s.vmx.HCPhys##a_Name = 0;
970
971	#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
972	pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
973	pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
974	pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
975
976	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
977	VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
978	#endif
979	VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
980
981	AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
982	for (VMCPUID i = 0; i < pVM->cCpus; i++)
983	{
984	PVMCPU pVCpu = &pVM->aCpus[i];
985	VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
986	VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
987	VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
988	VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
989	}
990	#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
991	#undef VMXLOCAL_INIT_VM_MEMOBJ
992
993	/* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
994	AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
995	(&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
996	VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
997
998	/*
999	* Allocate all the VT-x structures.
1000	*/
1001	int rc = VINF_SUCCESS;
1002	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1003	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
1004	if (RT_FAILURE(rc))
1005	goto cleanup;
1006	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
1007	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
1008	#endif
1009
1010	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
1011	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
1012	{
1013	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
1014	&pVM->hm.s.vmx.HCPhysApicAccess);
1015	if (RT_FAILURE(rc))
1016	goto cleanup;
1017	}
1018
1019	/*
1020	* Initialize per-VCPU VT-x structures.
1021	*/
1022	for (VMCPUID i = 0; i < pVM->cCpus; i++)
1023	{
1024	PVMCPU pVCpu = &pVM->aCpus[i];
1025	AssertPtr(pVCpu);
1026
1027	/* Allocate the VM control structure (VMCS). */
1028	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
1029	if (RT_FAILURE(rc))
1030	goto cleanup;
1031
1032	/* Get the allocated virtual-APIC page from the APIC device for transparent TPR accesses. */
1033	if ( PDMHasApic(pVM)
1034	&& (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW))
1035	{
1036	rc = APICGetApicPageForCpu(pVCpu, &pVCpu->hm.s.vmx.HCPhysVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
1037	NULL /* pR3Ptr /, NULL / pRCPtr */);
1038	if (RT_FAILURE(rc))
1039	goto cleanup;
1040	}
1041
1042	/*
1043	* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
1044	* transparent accesses of specific MSRs.
1045	*
1046	* If the condition for enabling MSR bitmaps changes here, don't forget to
1047	* update HMAreMsrBitmapsAvailable().
1048	*/
1049	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1050	{
1051	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
1052	&pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1053	if (RT_FAILURE(rc))
1054	goto cleanup;
1055	ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
1056	}
1057
1058	/* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
1059	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
1060	if (RT_FAILURE(rc))
1061	goto cleanup;
1062
1063	/* Allocate the VM-exit MSR-load page for the host MSRs. */
1064	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
1065	if (RT_FAILURE(rc))
1066	goto cleanup;
1067	}
1068
1069	return VINF_SUCCESS;
1070
1071	cleanup:
1072	hmR0VmxStructsFree(pVM);
1073	return rc;
1074	}
1075
1076
1077	/**
1078	* Does global VT-x initialization (called during module initialization).
1079	*
1080	* @returns VBox status code.
1081	*/
1082	VMMR0DECL(int) VMXR0GlobalInit(void)
1083	{
1084	#ifdef HMVMX_USE_FUNCTION_TABLE
1085	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1086	# ifdef VBOX_STRICT
1087	for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1088	Assert(g_apfnVMExitHandlers[i]);
1089	# endif
1090	#endif
1091	return VINF_SUCCESS;
1092	}
1093
1094
1095	/**
1096	* Does global VT-x termination (called during module termination).
1097	*/
1098	VMMR0DECL(void) VMXR0GlobalTerm()
1099	{
1100	/* Nothing to do currently. */
1101	}
1102
1103
1104	/**
1105	* Sets up and activates VT-x on the current CPU.
1106	*
1107	* @returns VBox status code.
1108	* @param pCpu Pointer to the global CPU info struct.
1109	* @param pVM The cross context VM structure. Can be
1110	* NULL after a host resume operation.
1111	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1112	* fEnabledByHost is @c true).
1113	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1114	* @a fEnabledByHost is @c true).
1115	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1116	* enable VT-x on the host.
1117	* @param pvMsrs Opaque pointer to VMXMSRS struct.
1118	*/
1119	VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1120	void *pvMsrs)
1121	{
1122	Assert(pCpu);
1123	Assert(pvMsrs);
1124	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1125
1126	/* Enable VT-x if it's not already enabled by the host. */
1127	if (!fEnabledByHost)
1128	{
1129	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1130	if (RT_FAILURE(rc))
1131	return rc;
1132	}
1133
1134	/*
1135	* Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been using EPTPs) so
1136	* we don't retain any stale guest-physical mappings which won't get invalidated when flushing by VPID.
1137	*/
1138	PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1139	if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1140	{
1141	hmR0VmxFlushEpt(NULL /* pVCpu */, VMXFLUSHEPT_ALL_CONTEXTS);
1142	pCpu->fFlushAsidBeforeUse = false;
1143	}
1144	else
1145	pCpu->fFlushAsidBeforeUse = true;
1146
1147	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1148	++pCpu->cTlbFlushes;
1149
1150	return VINF_SUCCESS;
1151	}
1152
1153
1154	/**
1155	* Deactivates VT-x on the current CPU.
1156	*
1157	* @returns VBox status code.
1158	* @param pCpu Pointer to the global CPU info struct.
1159	* @param pvCpuPage Pointer to the VMXON region.
1160	* @param HCPhysCpuPage Physical address of the VMXON region.
1161	*
1162	* @remarks This function should never be called when SUPR0EnableVTx() or
1163	* similar was used to enable VT-x on the host.
1164	*/
1165	VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1166	{
1167	NOREF(pCpu);
1168	NOREF(pvCpuPage);
1169	NOREF(HCPhysCpuPage);
1170
1171	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1172	return hmR0VmxLeaveRootMode();
1173	}
1174
1175
1176	/**
1177	* Sets the permission bits for the specified MSR in the MSR bitmap.
1178	*
1179	* @param pVCpu The cross context virtual CPU structure.
1180	* @param uMsr The MSR value.
1181	* @param enmRead Whether reading this MSR causes a VM-exit.
1182	* @param enmWrite Whether writing this MSR causes a VM-exit.
1183	*/
1184	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1185	{
1186	int32_t iBit;
1187	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
1188
1189	/*
1190	* Layout:
1191	* 0x000 - 0x3ff - Low MSR read bits
1192	* 0x400 - 0x7ff - High MSR read bits
1193	* 0x800 - 0xbff - Low MSR write bits
1194	* 0xc00 - 0xfff - High MSR write bits
1195	*/
1196	if (uMsr <= 0x00001FFF)
1197	iBit = uMsr;
1198	else if (uMsr - UINT32_C(0xC0000000) <= UINT32_C(0x00001FFF))
1199	{
1200	iBit = uMsr - UINT32_C(0xC0000000);
1201	pbMsrBitmap += 0x400;
1202	}
1203	else
1204	AssertMsgFailedReturnVoid(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1205
1206	Assert(iBit <= 0x1fff);
1207	if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1208	ASMBitSet(pbMsrBitmap, iBit);
1209	else
1210	ASMBitClear(pbMsrBitmap, iBit);
1211
1212	if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1213	ASMBitSet(pbMsrBitmap + 0x800, iBit);
1214	else
1215	ASMBitClear(pbMsrBitmap + 0x800, iBit);
1216	}
1217
1218
1219	#ifdef VBOX_STRICT
1220	/**
1221	* Gets the permission bits for the specified MSR in the MSR bitmap.
1222	*
1223	* @returns VBox status code.
1224	* @retval VINF_SUCCESS if the specified MSR is found.
1225	* @retval VERR_NOT_FOUND if the specified MSR is not found.
1226	* @retval VERR_NOT_SUPPORTED if VT-x doesn't allow the MSR.
1227	*
1228	* @param pVCpu The cross context virtual CPU structure.
1229	* @param uMsr The MSR.
1230	* @param penmRead Where to store the read permissions.
1231	* @param penmWrite Where to store the write permissions.
1232	*/
1233	static int hmR0VmxGetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, PVMXMSREXITREAD penmRead, PVMXMSREXITWRITE penmWrite)
1234	{
1235	AssertPtrReturn(penmRead, VERR_INVALID_PARAMETER);
1236	AssertPtrReturn(penmWrite, VERR_INVALID_PARAMETER);
1237	int32_t iBit;
1238	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
1239
1240	/* See hmR0VmxSetMsrPermission() for the layout. */
1241	if (uMsr <= 0x00001FFF)
1242	iBit = uMsr;
1243	else if ( uMsr >= 0xC0000000
1244	&& uMsr <= 0xC0001FFF)
1245	{
1246	iBit = (uMsr - 0xC0000000);
1247	pbMsrBitmap += 0x400;
1248	}
1249	else
1250	AssertMsgFailedReturn(("hmR0VmxGetMsrPermission: Invalid MSR %#RX32\n", uMsr), VERR_NOT_SUPPORTED);
1251
1252	Assert(iBit <= 0x1fff);
1253	if (ASMBitTest(pbMsrBitmap, iBit))
1254	*penmRead = VMXMSREXIT_INTERCEPT_READ;
1255	else
1256	*penmRead = VMXMSREXIT_PASSTHRU_READ;
1257
1258	if (ASMBitTest(pbMsrBitmap + 0x800, iBit))
1259	*penmWrite = VMXMSREXIT_INTERCEPT_WRITE;
1260	else
1261	*penmWrite = VMXMSREXIT_PASSTHRU_WRITE;
1262	return VINF_SUCCESS;
1263	}
1264	#endif /* VBOX_STRICT */
1265
1266
1267	/**
1268	* Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1269	* area.
1270	*
1271	* @returns VBox status code.
1272	* @param pVCpu The cross context virtual CPU structure.
1273	* @param cMsrs The number of MSRs.
1274	*/
1275	DECLINLINE(int) hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1276	{
1277	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1278	uint32_t const cMaxSupportedMsrs = MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1279	if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1280	{
1281	LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1282	pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1283	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1284	}
1285
1286	/* Update number of guest MSRs to load/store across the world-switch. */
1287	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs);
1288	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs);
1289
1290	/* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1291	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs);
1292	AssertRCReturn(rc, rc);
1293
1294	/* Update the VCPU's copy of the MSR count. */
1295	pVCpu->hm.s.vmx.cMsrs = cMsrs;
1296
1297	return VINF_SUCCESS;
1298	}
1299
1300
1301	/**
1302	* Adds a new (or updates the value of an existing) guest/host MSR
1303	* pair to be swapped during the world-switch as part of the
1304	* auto-load/store MSR area in the VMCS.
1305	*
1306	* @returns VBox status code.
1307	* @param pVCpu The cross context virtual CPU structure.
1308	* @param uMsr The MSR.
1309	* @param uGuestMsrValue Value of the guest MSR.
1310	* @param fUpdateHostMsr Whether to update the value of the host MSR if
1311	* necessary.
1312	* @param pfAddedAndUpdated Where to store whether the MSR was added -and-
1313	* its value was updated. Optional, can be NULL.
1314	*/
1315	static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr,
1316	bool *pfAddedAndUpdated)
1317	{
1318	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1319	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1320	uint32_t i;
1321	for (i = 0; i < cMsrs; i++)
1322	{
1323	if (pGuestMsr->u32Msr == uMsr)
1324	break;
1325	pGuestMsr++;
1326	}
1327
1328	bool fAdded = false;
1329	if (i == cMsrs)
1330	{
1331	++cMsrs;
1332	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1333	AssertMsgRCReturn(rc, ("hmR0VmxAddAutoLoadStoreMsr: Insufficient space to add MSR %u\n", uMsr), rc);
1334
1335	/* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1336	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1337	hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1338
1339	fAdded = true;
1340	}
1341
1342	/* Update the MSR values in the auto-load/store MSR area. */
1343	pGuestMsr->u32Msr = uMsr;
1344	pGuestMsr->u64Value = uGuestMsrValue;
1345
1346	/* Create/update the MSR slot in the host MSR area. */
1347	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1348	pHostMsr += i;
1349	pHostMsr->u32Msr = uMsr;
1350
1351	/*
1352	* Update the host MSR only when requested by the caller AND when we're
1353	* adding it to the auto-load/store area. Otherwise, it would have been
1354	* updated by hmR0VmxSaveHostMsrs(). We do this for performance reasons.
1355	*/
1356	bool fUpdatedMsrValue = false;
1357	if ( fAdded
1358	&& fUpdateHostMsr)
1359	{
1360	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1361	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1362	pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1363	fUpdatedMsrValue = true;
1364	}
1365
1366	if (pfAddedAndUpdated)
1367	*pfAddedAndUpdated = fUpdatedMsrValue;
1368	return VINF_SUCCESS;
1369	}
1370
1371
1372	/**
1373	* Removes a guest/host MSR pair to be swapped during the world-switch from the
1374	* auto-load/store MSR area in the VMCS.
1375	*
1376	* @returns VBox status code.
1377	* @param pVCpu The cross context virtual CPU structure.
1378	* @param uMsr The MSR.
1379	*/
1380	static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1381	{
1382	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1383	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1384	for (uint32_t i = 0; i < cMsrs; i++)
1385	{
1386	/* Find the MSR. */
1387	if (pGuestMsr->u32Msr == uMsr)
1388	{
1389	/* If it's the last MSR, simply reduce the count. */
1390	if (i == cMsrs - 1)
1391	{
1392	--cMsrs;
1393	break;
1394	}
1395
1396	/* Remove it by swapping the last MSR in place of it, and reducing the count. */
1397	PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1398	pLastGuestMsr += cMsrs - 1;
1399	pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1400	pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1401
1402	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1403	PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1404	pLastHostMsr += cMsrs - 1;
1405	pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1406	pHostMsr->u64Value = pLastHostMsr->u64Value;
1407	--cMsrs;
1408	break;
1409	}
1410	pGuestMsr++;
1411	}
1412
1413	/* Update the VMCS if the count changed (meaning the MSR was found). */
1414	if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1415	{
1416	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1417	AssertRCReturn(rc, rc);
1418
1419	/* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1420	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1421	hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1422
1423	Log4(("Removed MSR %#RX32 new cMsrs=%u\n", uMsr, pVCpu->hm.s.vmx.cMsrs));
1424	return VINF_SUCCESS;
1425	}
1426
1427	return VERR_NOT_FOUND;
1428	}
1429
1430
1431	/**
1432	* Checks if the specified guest MSR is part of the auto-load/store area in
1433	* the VMCS.
1434	*
1435	* @returns true if found, false otherwise.
1436	* @param pVCpu The cross context virtual CPU structure.
1437	* @param uMsr The MSR to find.
1438	*/
1439	static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1440	{
1441	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1442	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1443
1444	for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1445	{
1446	if (pGuestMsr->u32Msr == uMsr)
1447	return true;
1448	}
1449	return false;
1450	}
1451
1452
1453	/**
1454	* Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1455	*
1456	* @param pVCpu The cross context virtual CPU structure.
1457	*
1458	* @remarks No-long-jump zone!!!
1459	*/
1460	static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1461	{
1462	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1463	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1464	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1465	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1466
1467	for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1468	{
1469	AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1470
1471	/*
1472	* Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1473	* Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1474	*/
1475	if (pHostMsr->u32Msr == MSR_K6_EFER)
1476	pHostMsr->u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostEfer;
1477	else
1478	pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1479	}
1480
1481	pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1482	}
1483
1484
1485	/**
1486	* Saves a set of host MSRs to allow read/write passthru access to the guest and
1487	* perform lazy restoration of the host MSRs while leaving VT-x.
1488	*
1489	* @param pVCpu The cross context virtual CPU structure.
1490	*
1491	* @remarks No-long-jump zone!!!
1492	*/
1493	static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1494	{
1495	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1496
1497	/*
1498	* Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1499	*/
1500	if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST))
1501	{
1502	Assert(!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)); /* Guest MSRs better not be loaded now. */
1503	#if HC_ARCH_BITS == 64
1504	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1505	{
1506	pVCpu->hm.s.vmx.u64HostLStarMsr = ASMRdMsr(MSR_K8_LSTAR);
1507	pVCpu->hm.s.vmx.u64HostStarMsr = ASMRdMsr(MSR_K6_STAR);
1508	pVCpu->hm.s.vmx.u64HostSFMaskMsr = ASMRdMsr(MSR_K8_SF_MASK);
1509	pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1510	}
1511	#endif
1512	pVCpu->hm.s.vmx.fLazyMsrs \|= VMX_LAZY_MSRS_SAVED_HOST;
1513	}
1514	}
1515
1516
1517	/**
1518	* Checks whether the MSR belongs to the set of guest MSRs that we restore
1519	* lazily while leaving VT-x.
1520	*
1521	* @returns true if it does, false otherwise.
1522	* @param pVCpu The cross context virtual CPU structure.
1523	* @param uMsr The MSR to check.
1524	*/
1525	static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1526	{
1527	NOREF(pVCpu);
1528	#if HC_ARCH_BITS == 64
1529	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1530	{
1531	switch (uMsr)
1532	{
1533	case MSR_K8_LSTAR:
1534	case MSR_K6_STAR:
1535	case MSR_K8_SF_MASK:
1536	case MSR_K8_KERNEL_GS_BASE:
1537	return true;
1538	}
1539	}
1540	#else
1541	RT_NOREF(pVCpu, uMsr);
1542	#endif
1543	return false;
1544	}
1545
1546
1547	/**
1548	* Saves a set of guest MSRs back into the guest-CPU context.
1549	*
1550	* @param pVCpu The cross context virtual CPU structure.
1551	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
1552	* out-of-sync. Make sure to update the required fields
1553	* before using them.
1554	*
1555	* @remarks No-long-jump zone!!!
1556	*/
1557	static void hmR0VmxLazySaveGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1558	{
1559	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1560	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1561
1562	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1563	{
1564	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1565	#if HC_ARCH_BITS == 64
1566	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1567	{
1568	pMixedCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
1569	pMixedCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
1570	pMixedCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
1571	pMixedCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1572	}
1573	#else
1574	NOREF(pMixedCtx);
1575	#endif
1576	}
1577	}
1578
1579
1580	/**
1581	* Loads a set of guests MSRs to allow read/passthru to the guest.
1582	*
1583	* The name of this function is slightly confusing. This function does NOT
1584	* postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1585	* common prefix for functions dealing with "lazy restoration" of the shared
1586	* MSRs.
1587	*
1588	* @param pVCpu The cross context virtual CPU structure.
1589	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
1590	* out-of-sync. Make sure to update the required fields
1591	* before using them.
1592	*
1593	* @remarks No-long-jump zone!!!
1594	*/
1595	static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1596	{
1597	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1598	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1599
1600	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1601	#if HC_ARCH_BITS == 64
1602	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1603	{
1604	/*
1605	* If the guest MSRs are not loaded -and- if all the guest MSRs are identical
1606	* to the MSRs on the CPU (which are the saved host MSRs, see assertion above) then
1607	* we can skip a few MSR writes.
1608	*
1609	* Otherwise, it implies either 1. they're not loaded, or 2. they're loaded but the
1610	* guest MSR values in the guest-CPU context might be different to what's currently
1611	* loaded in the CPU. In either case, we need to write the new guest MSR values to the
1612	* CPU, see @bugref{8728}.
1613	*/
1614	if ( !(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1615	&& pMixedCtx->msrKERNELGSBASE == pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr
1616	&& pMixedCtx->msrLSTAR == pVCpu->hm.s.vmx.u64HostLStarMsr
1617	&& pMixedCtx->msrSTAR == pVCpu->hm.s.vmx.u64HostStarMsr
1618	&& pMixedCtx->msrSFMASK == pVCpu->hm.s.vmx.u64HostSFMaskMsr)
1619	{
1620	#ifdef VBOX_STRICT
1621	Assert(ASMRdMsr(MSR_K8_KERNEL_GS_BASE) == pMixedCtx->msrKERNELGSBASE);
1622	Assert(ASMRdMsr(MSR_K8_LSTAR) == pMixedCtx->msrLSTAR);
1623	Assert(ASMRdMsr(MSR_K6_STAR) == pMixedCtx->msrSTAR);
1624	Assert(ASMRdMsr(MSR_K8_SF_MASK) == pMixedCtx->msrSFMASK);
1625	#endif
1626	}
1627	else
1628	{
1629	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE);
1630	ASMWrMsr(MSR_K8_LSTAR, pMixedCtx->msrLSTAR);
1631	ASMWrMsr(MSR_K6_STAR, pMixedCtx->msrSTAR);
1632	ASMWrMsr(MSR_K8_SF_MASK, pMixedCtx->msrSFMASK);
1633	}
1634	}
1635	#else
1636	RT_NOREF(pMixedCtx);
1637	#endif
1638	pVCpu->hm.s.vmx.fLazyMsrs \|= VMX_LAZY_MSRS_LOADED_GUEST;
1639	}
1640
1641
1642	/**
1643	* Performs lazy restoration of the set of host MSRs if they were previously
1644	* loaded with guest MSR values.
1645	*
1646	* @param pVCpu The cross context virtual CPU structure.
1647	*
1648	* @remarks No-long-jump zone!!!
1649	* @remarks The guest MSRs should have been saved back into the guest-CPU
1650	* context by hmR0VmxSaveGuestLazyMsrs()!!!
1651	*/
1652	static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1653	{
1654	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1655	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1656
1657	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1658	{
1659	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1660	#if HC_ARCH_BITS == 64
1661	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1662	{
1663	ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostLStarMsr);
1664	ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostStarMsr);
1665	ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostSFMaskMsr);
1666	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr);
1667	}
1668	#endif
1669	}
1670	pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST \| VMX_LAZY_MSRS_SAVED_HOST);
1671	}
1672
1673
1674	/**
1675	* Verifies that our cached values of the VMCS controls are all
1676	* consistent with what's actually present in the VMCS.
1677	*
1678	* @returns VBox status code.
1679	* @param pVCpu The cross context virtual CPU structure.
1680	*/
1681	static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu)
1682	{
1683	uint32_t u32Val;
1684	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
1685	AssertRCReturn(rc, rc);
1686	AssertMsgReturn(pVCpu->hm.s.vmx.u32EntryCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32EntryCtls, u32Val),
1687	VERR_VMX_ENTRY_CTLS_CACHE_INVALID);
1688
1689	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
1690	AssertRCReturn(rc, rc);
1691	AssertMsgReturn(pVCpu->hm.s.vmx.u32ExitCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ExitCtls, u32Val),
1692	VERR_VMX_EXIT_CTLS_CACHE_INVALID);
1693
1694	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
1695	AssertRCReturn(rc, rc);
1696	AssertMsgReturn(pVCpu->hm.s.vmx.u32PinCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32PinCtls, u32Val),
1697	VERR_VMX_PIN_EXEC_CTLS_CACHE_INVALID);
1698
1699	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
1700	AssertRCReturn(rc, rc);
1701	AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls, u32Val),
1702	VERR_VMX_PROC_EXEC_CTLS_CACHE_INVALID);
1703
1704	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1705	{
1706	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
1707	AssertRCReturn(rc, rc);
1708	AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls2 == u32Val,
1709	("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls2, u32Val),
1710	VERR_VMX_PROC_EXEC2_CTLS_CACHE_INVALID);
1711	}
1712
1713	return VINF_SUCCESS;
1714	}
1715
1716
1717	#ifdef VBOX_STRICT
1718	/**
1719	* Verifies that our cached host EFER value has not changed
1720	* since we cached it.
1721	*
1722	* @param pVCpu The cross context virtual CPU structure.
1723	*/
1724	static void hmR0VmxCheckHostEferMsr(PVMCPU pVCpu)
1725	{
1726	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1727
1728	if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
1729	{
1730	uint64_t u64Val;
1731	int rc = VMXReadVmcs64(VMX_VMCS64_HOST_EFER_FULL, &u64Val);
1732	AssertRC(rc);
1733
1734	uint64_t u64HostEferMsr = ASMRdMsr(MSR_K6_EFER);
1735	AssertMsgReturnVoid(u64HostEferMsr == u64Val, ("u64HostEferMsr=%#RX64 u64Val=%#RX64\n", u64HostEferMsr, u64Val));
1736	}
1737	}
1738
1739
1740	/**
1741	* Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1742	* VMCS are correct.
1743	*
1744	* @param pVCpu The cross context virtual CPU structure.
1745	*/
1746	static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1747	{
1748	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1749
1750	/* Verify MSR counts in the VMCS are what we think it should be. */
1751	uint32_t cMsrs;
1752	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1753	Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1754
1755	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1756	Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1757
1758	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1759	Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1760
1761	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1762	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1763	for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1764	{
1765	/* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1766	AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32 cMsrs=%u\n", pHostMsr->u32Msr,
1767	pGuestMsr->u32Msr, cMsrs));
1768
1769	uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1770	AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
1771	pHostMsr->u32Msr, pHostMsr->u64Value, u64Msr, cMsrs));
1772
1773	/* Verify that the permissions are as expected in the MSR bitmap. */
1774	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1775	{
1776	VMXMSREXITREAD enmRead;
1777	VMXMSREXITWRITE enmWrite;
1778	rc = hmR0VmxGetMsrPermission(pVCpu, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1779	AssertMsgReturnVoid(rc == VINF_SUCCESS, ("hmR0VmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1780	if (pGuestMsr->u32Msr == MSR_K6_EFER)
1781	{
1782	AssertMsgReturnVoid(enmRead == VMXMSREXIT_INTERCEPT_READ, ("Passthru read for EFER!?\n"));
1783	AssertMsgReturnVoid(enmWrite == VMXMSREXIT_INTERCEPT_WRITE, ("Passthru write for EFER!?\n"));
1784	}
1785	else
1786	{
1787	AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 cMsrs=%u No passthru read!\n",
1788	pGuestMsr->u32Msr, cMsrs));
1789	AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 cMsrs=%u No passthru write!\n",
1790	pGuestMsr->u32Msr, cMsrs));
1791	}
1792	}
1793	}
1794	}
1795	#endif /* VBOX_STRICT */
1796
1797
1798	/**
1799	* Flushes the TLB using EPT.
1800	*
1801	* @returns VBox status code.
1802	* @param pVCpu The cross context virtual CPU structure of the calling
1803	* EMT. Can be NULL depending on @a enmFlush.
1804	* @param enmFlush Type of flush.
1805	*
1806	* @remarks Caller is responsible for making sure this function is called only
1807	* when NestedPaging is supported and providing @a enmFlush that is
1808	* supported by the CPU.
1809	* @remarks Can be called with interrupts disabled.
1810	*/
1811	static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush)
1812	{
1813	uint64_t au64Descriptor[2];
1814	if (enmFlush == VMXFLUSHEPT_ALL_CONTEXTS)
1815	au64Descriptor[0] = 0;
1816	else
1817	{
1818	Assert(pVCpu);
1819	au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1820	}
1821	au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1822
1823	int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1824	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1825	rc));
1826	if ( RT_SUCCESS(rc)
1827	&& pVCpu)
1828	{
1829	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1830	}
1831	}
1832
1833
1834	/**
1835	* Flushes the TLB using VPID.
1836	*
1837	* @returns VBox status code.
1838	* @param pVM The cross context VM structure.
1839	* @param pVCpu The cross context virtual CPU structure of the calling
1840	* EMT. Can be NULL depending on @a enmFlush.
1841	* @param enmFlush Type of flush.
1842	* @param GCPtr Virtual address of the page to flush (can be 0 depending
1843	* on @a enmFlush).
1844	*
1845	* @remarks Can be called with interrupts disabled.
1846	*/
1847	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr)
1848	{
1849	NOREF(pVM);
1850	AssertPtr(pVM);
1851	Assert(pVM->hm.s.vmx.fVpid);
1852
1853	uint64_t au64Descriptor[2];
1854	if (enmFlush == VMXFLUSHVPID_ALL_CONTEXTS)
1855	{
1856	au64Descriptor[0] = 0;
1857	au64Descriptor[1] = 0;
1858	}
1859	else
1860	{
1861	AssertPtr(pVCpu);
1862	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1863	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1864	au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1865	au64Descriptor[1] = GCPtr;
1866	}
1867
1868	int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]); NOREF(rc);
1869	AssertMsg(rc == VINF_SUCCESS,
1870	("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1871	if ( RT_SUCCESS(rc)
1872	&& pVCpu)
1873	{
1874	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1875	}
1876	}
1877
1878
1879	/**
1880	* Invalidates a guest page by guest virtual address. Only relevant for
1881	* EPT/VPID, otherwise there is nothing really to invalidate.
1882	*
1883	* @returns VBox status code.
1884	* @param pVM The cross context VM structure.
1885	* @param pVCpu The cross context virtual CPU structure.
1886	* @param GCVirt Guest virtual address of the page to invalidate.
1887	*/
1888	VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1889	{
1890	AssertPtr(pVM);
1891	AssertPtr(pVCpu);
1892	LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1893
1894	bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1895	if (!fFlushPending)
1896	{
1897	/*
1898	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1899	* See @bugref{6043} and @bugref{6177}.
1900	*
1901	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1902	* function maybe called in a loop with individual addresses.
1903	*/
1904	if (pVM->hm.s.vmx.fVpid)
1905	{
1906	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1907	{
1908	hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_INDIV_ADDR, GCVirt);
1909	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1910	}
1911	else
1912	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1913	}
1914	else if (pVM->hm.s.fNestedPaging)
1915	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1916	}
1917
1918	return VINF_SUCCESS;
1919	}
1920
1921
1922	/**
1923	* Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1924	* otherwise there is nothing really to invalidate.
1925	*
1926	* @returns VBox status code.
1927	* @param pVM The cross context VM structure.
1928	* @param pVCpu The cross context virtual CPU structure.
1929	* @param GCPhys Guest physical address of the page to invalidate.
1930	*/
1931	VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1932	{
1933	NOREF(pVM); NOREF(GCPhys);
1934	LogFlowFunc(("%RGp\n", GCPhys));
1935
1936	/*
1937	* We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1938	* by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1939	* This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1940	*/
1941	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1942	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1943	return VINF_SUCCESS;
1944	}
1945
1946
1947	/**
1948	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1949	* case where neither EPT nor VPID is supported by the CPU.
1950	*
1951	* @param pVM The cross context VM structure.
1952	* @param pVCpu The cross context virtual CPU structure.
1953	* @param pCpu Pointer to the global HM struct.
1954	*
1955	* @remarks Called with interrupts disabled.
1956	*/
1957	static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1958	{
1959	AssertPtr(pVCpu);
1960	AssertPtr(pCpu);
1961	NOREF(pVM);
1962
1963	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1964
1965	Assert(pCpu->idCpu != NIL_RTCPUID);
1966	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1967	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1968	pVCpu->hm.s.fForceTLBFlush = false;
1969	return;
1970	}
1971
1972
1973	/**
1974	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1975	*
1976	* @param pVM The cross context VM structure.
1977	* @param pVCpu The cross context virtual CPU structure.
1978	* @param pCpu Pointer to the global HM CPU struct.
1979	* @remarks All references to "ASID" in this function pertains to "VPID" in
1980	* Intel's nomenclature. The reason is, to avoid confusion in compare
1981	* statements since the host-CPU copies are named "ASID".
1982	*
1983	* @remarks Called with interrupts disabled.
1984	*/
1985	static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1986	{
1987	#ifdef VBOX_WITH_STATISTICS
1988	bool fTlbFlushed = false;
1989	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1990	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1991	if (!fTlbFlushed) \
1992	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1993	} while (0)
1994	#else
1995	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1996	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1997	#endif
1998
1999	AssertPtr(pVM);
2000	AssertPtr(pCpu);
2001	AssertPtr(pVCpu);
2002	Assert(pCpu->idCpu != NIL_RTCPUID);
2003
2004	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
2005	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
2006	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
2007
2008	/*
2009	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2010	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
2011	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
2012	*/
2013	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2014	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2015	{
2016	++pCpu->uCurrentAsid;
2017	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2018	{
2019	pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
2020	pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2021	pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2022	}
2023
2024	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2025	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2026	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2027
2028	/*
2029	* Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
2030	* invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
2031	*/
2032	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2033	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2034	HMVMX_SET_TAGGED_TLB_FLUSHED();
2035	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
2036	}
2037
2038	/* Check for explicit TLB flushes. */
2039	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2040	{
2041	/*
2042	* Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
2043	* guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
2044	* Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
2045	* but not guest-physical mappings.
2046	* See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
2047	*/
2048	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2049	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2050	HMVMX_SET_TAGGED_TLB_FLUSHED();
2051	}
2052
2053	pVCpu->hm.s.fForceTLBFlush = false;
2054	HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
2055
2056	Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
2057	Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
2058	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2059	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2060	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2061	("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2062	pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2063	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2064	("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2065
2066	/* Update VMCS with the VPID. */
2067	int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2068	AssertRC(rc);
2069
2070	#undef HMVMX_SET_TAGGED_TLB_FLUSHED
2071	}
2072
2073
2074	/**
2075	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
2076	*
2077	* @returns VBox status code.
2078	* @param pVM The cross context VM structure.
2079	* @param pVCpu The cross context virtual CPU structure.
2080	* @param pCpu Pointer to the global HM CPU struct.
2081	*
2082	* @remarks Called with interrupts disabled.
2083	*/
2084	static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2085	{
2086	AssertPtr(pVM);
2087	AssertPtr(pVCpu);
2088	AssertPtr(pCpu);
2089	Assert(pCpu->idCpu != NIL_RTCPUID);
2090	AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
2091	AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
2092
2093	/*
2094	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2095	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2096	*/
2097	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2098	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2099	{
2100	pVCpu->hm.s.fForceTLBFlush = true;
2101	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2102	}
2103
2104	/* Check for explicit TLB flushes. */
2105	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2106	{
2107	pVCpu->hm.s.fForceTLBFlush = true;
2108	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2109	}
2110
2111	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2112	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2113
2114	if (pVCpu->hm.s.fForceTLBFlush)
2115	{
2116	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2117	pVCpu->hm.s.fForceTLBFlush = false;
2118	}
2119	}
2120
2121
2122	/**
2123	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
2124	*
2125	* @returns VBox status code.
2126	* @param pVM The cross context VM structure.
2127	* @param pVCpu The cross context virtual CPU structure.
2128	* @param pCpu Pointer to the global HM CPU struct.
2129	*
2130	* @remarks Called with interrupts disabled.
2131	*/
2132	static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2133	{
2134	AssertPtr(pVM);
2135	AssertPtr(pVCpu);
2136	AssertPtr(pCpu);
2137	Assert(pCpu->idCpu != NIL_RTCPUID);
2138	AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
2139	AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
2140
2141	/*
2142	* Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
2143	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
2144	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
2145	*/
2146	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2147	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2148	{
2149	pVCpu->hm.s.fForceTLBFlush = true;
2150	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2151	}
2152
2153	/* Check for explicit TLB flushes. */
2154	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2155	{
2156	/*
2157	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
2158	* we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
2159	* pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
2160	*/
2161	pVCpu->hm.s.fForceTLBFlush = true;
2162	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2163	}
2164
2165	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2166	if (pVCpu->hm.s.fForceTLBFlush)
2167	{
2168	++pCpu->uCurrentAsid;
2169	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2170	{
2171	pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2172	pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2173	pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2174	}
2175
2176	pVCpu->hm.s.fForceTLBFlush = false;
2177	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2178	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2179	if (pCpu->fFlushAsidBeforeUse)
2180	{
2181	if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_SINGLE_CONTEXT)
2182	hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2183	else if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_ALL_CONTEXTS)
2184	{
2185	hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2186	pCpu->fFlushAsidBeforeUse = false;
2187	}
2188	else
2189	{
2190	/* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2191	AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2192	}
2193	}
2194	}
2195
2196	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2197	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2198	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2199	("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2200	pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2201	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2202	("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2203
2204	int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2205	AssertRC(rc);
2206	}
2207
2208
2209	/**
2210	* Flushes the guest TLB entry based on CPU capabilities.
2211	*
2212	* @param pVCpu The cross context virtual CPU structure.
2213	* @param pCpu Pointer to the global HM CPU struct.
2214	*/
2215	DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2216	{
2217	#ifdef HMVMX_ALWAYS_FLUSH_TLB
2218	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2219	#endif
2220	PVM pVM = pVCpu->CTX_SUFF(pVM);
2221	switch (pVM->hm.s.vmx.uFlushTaggedTlb)
2222	{
2223	case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu, pCpu); break;
2224	case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu, pCpu); break;
2225	case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu, pCpu); break;
2226	case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu, pCpu); break;
2227	default:
2228	AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2229	break;
2230	}
2231
2232	/* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2233	}
2234
2235
2236	/**
2237	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2238	* TLB entries from the host TLB before VM-entry.
2239	*
2240	* @returns VBox status code.
2241	* @param pVM The cross context VM structure.
2242	*/
2243	static int hmR0VmxSetupTaggedTlb(PVM pVM)
2244	{
2245	/*
2246	* Determine optimal flush type for Nested Paging.
2247	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2248	* guest execution (see hmR3InitFinalizeR0()).
2249	*/
2250	if (pVM->hm.s.fNestedPaging)
2251	{
2252	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2253	{
2254	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2255	pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_SINGLE_CONTEXT;
2256	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2257	pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_ALL_CONTEXTS;
2258	else
2259	{
2260	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2261	pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2262	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
2263	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2264	}
2265
2266	/* Make sure the write-back cacheable memory type for EPT is supported. */
2267	if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
2268	{
2269	pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2270	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
2271	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2272	}
2273
2274	/* EPT requires a page-walk length of 4. */
2275	if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
2276	{
2277	pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2278	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
2279	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2280	}
2281	}
2282	else
2283	{
2284	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2285	pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2286	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
2287	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2288	}
2289	}
2290
2291	/*
2292	* Determine optimal flush type for VPID.
2293	*/
2294	if (pVM->hm.s.vmx.fVpid)
2295	{
2296	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2297	{
2298	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2299	pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_SINGLE_CONTEXT;
2300	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2301	pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_ALL_CONTEXTS;
2302	else
2303	{
2304	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2305	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2306	LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
2307	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2308	LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2309	pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2310	pVM->hm.s.vmx.fVpid = false;
2311	}
2312	}
2313	else
2314	{
2315	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2316	Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
2317	pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2318	pVM->hm.s.vmx.fVpid = false;
2319	}
2320	}
2321
2322	/*
2323	* Setup the handler for flushing tagged-TLBs.
2324	*/
2325	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2326	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
2327	else if (pVM->hm.s.fNestedPaging)
2328	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
2329	else if (pVM->hm.s.vmx.fVpid)
2330	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
2331	else
2332	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
2333	return VINF_SUCCESS;
2334	}
2335
2336
2337	/**
2338	* Sets up pin-based VM-execution controls in the VMCS.
2339	*
2340	* @returns VBox status code.
2341	* @param pVM The cross context VM structure.
2342	* @param pVCpu The cross context virtual CPU structure.
2343	*/
2344	static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
2345	{
2346	AssertPtr(pVM);
2347	AssertPtr(pVCpu);
2348
2349	uint32_t val = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
2350	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2351
2352	val \|= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2353	\| VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2354
2355	if (pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
2356	val \|= VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2357
2358	/* Enable the VMX preemption timer. */
2359	if (pVM->hm.s.vmx.fUsePreemptTimer)
2360	{
2361	Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
2362	val \|= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
2363	}
2364
2365	#if 0
2366	/* Enable posted-interrupt processing. */
2367	if (pVM->hm.s.fPostedIntrs)
2368	{
2369	Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR);
2370	Assert(pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT);
2371	val \|= VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR;
2372	}
2373	#endif
2374
2375	if ((val & zap) != val)
2376	{
2377	LogRel(("hmR0VmxSetupPinCtls: Invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2378	pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, val, zap));
2379	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2380	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2381	}
2382
2383	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
2384	AssertRCReturn(rc, rc);
2385
2386	pVCpu->hm.s.vmx.u32PinCtls = val;
2387	return rc;
2388	}
2389
2390
2391	/**
2392	* Sets up processor-based VM-execution controls in the VMCS.
2393	*
2394	* @returns VBox status code.
2395	* @param pVM The cross context VM structure.
2396	* @param pVCpu The cross context virtual CPU structure.
2397	*/
2398	static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
2399	{
2400	AssertPtr(pVM);
2401	AssertPtr(pVCpu);
2402
2403	int rc = VERR_INTERNAL_ERROR_5;
2404	uint32_t val = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2405	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2406
2407	val \|= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
2408	\| VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2409	\| VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2410	\| VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2411	\| VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2412	\| VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2413	\| VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2414
2415	/* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2416	if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
2417	\|\| (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
2418	{
2419	LogRel(("hmR0VmxSetupProcCtls: Unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
2420	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2421	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2422	}
2423
2424	/* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2425	if (!pVM->hm.s.fNestedPaging)
2426	{
2427	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2428	val \|= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
2429	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2430	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2431	}
2432
2433	/* Use TPR shadowing if supported by the CPU. */
2434	if ( PDMHasApic(pVM)
2435	&& pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2436	{
2437	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2438	Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2439	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2440	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2441	AssertRCReturn(rc, rc);
2442
2443	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2444	/* CR8 writes cause a VM-exit based on TPR threshold. */
2445	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
2446	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
2447	}
2448	else
2449	{
2450	/*
2451	* Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2452	* Set this control only for 64-bit guests.
2453	*/
2454	if (pVM->hm.s.fAllow64BitGuests)
2455	{
2456	val \|= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
2457	\| VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
2458	}
2459	}
2460
2461	/* Use MSR-bitmaps if supported by the CPU. */
2462	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2463	{
2464	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
2465
2466	Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2467	Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2468	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2469	AssertRCReturn(rc, rc);
2470
2471	/*
2472	* The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2473	* automatically using dedicated fields in the VMCS.
2474	*/
2475	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2476	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2477	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2478	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2479	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2480
2481	#if HC_ARCH_BITS == 64
2482	/*
2483	* Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2484	*/
2485	if (pVM->hm.s.fAllow64BitGuests)
2486	{
2487	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2488	hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2489	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2490	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2491	}
2492	#endif
2493	/* Though MSR_IA32_PERF_GLOBAL_CTRL is saved/restored lazily, we want intercept reads/write to it for now. */
2494	}
2495
2496	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2497	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2498	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
2499
2500	if ((val & zap) != val)
2501	{
2502	LogRel(("hmR0VmxSetupProcCtls: Invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2503	pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, val, zap));
2504	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2505	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2506	}
2507
2508	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
2509	AssertRCReturn(rc, rc);
2510
2511	pVCpu->hm.s.vmx.u32ProcCtls = val;
2512
2513	/*
2514	* Secondary processor-based VM-execution controls.
2515	*/
2516	if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
2517	{
2518	val = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2519	zap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2520
2521	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
2522	val \|= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
2523
2524	if (pVM->hm.s.fNestedPaging)
2525	val \|= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
2526
2527	/*
2528	* Enable the INVPCID instruction if supported by the hardware and we expose
2529	* it to the guest. Without this, guest executing INVPCID would cause a #UD.
2530	*/
2531	#if 0
2532	if ( (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
2533	&& pVM->cpum.ro.GuestFeatures.fInvpcid)
2534	val \|= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
2535	#endif
2536
2537	if (pVM->hm.s.vmx.fVpid)
2538	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
2539
2540	if (pVM->hm.s.vmx.fUnrestrictedGuest)
2541	val \|= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
2542
2543	#if 0
2544	if (pVM->hm.s.fVirtApicRegs)
2545	{
2546	Assert(pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT);
2547	val \|= VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT; /* Enable APIC-register virtualization. */
2548
2549	Assert(pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY);
2550	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY; /* Enable virtual-interrupt delivery. */
2551	}
2552	#endif
2553
2554	/* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
2555	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2556	* done dynamically. */
2557	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2558	{
2559	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2560	Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2561	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
2562	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2563	AssertRCReturn(rc, rc);
2564	}
2565
2566	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2567	val \|= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
2568
2569	if ( pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT
2570	&& pVM->hm.s.vmx.cPleGapTicks
2571	&& pVM->hm.s.vmx.cPleWindowTicks)
2572	{
2573	val \|= VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT; /* Enable pause-loop exiting. */
2574
2575	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks);
2576	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks);
2577	AssertRCReturn(rc, rc);
2578	}
2579
2580	if ((val & zap) != val)
2581	{
2582	LogRel(("hmR0VmxSetupProcCtls: Invalid secondary processor-based VM-execution controls combo! "
2583	"cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, val, zap));
2584	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2585	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2586	}
2587
2588	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
2589	AssertRCReturn(rc, rc);
2590
2591	pVCpu->hm.s.vmx.u32ProcCtls2 = val;
2592	}
2593	else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2594	{
2595	LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
2596	"available\n"));
2597	pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2598	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2599	}
2600
2601	return VINF_SUCCESS;
2602	}
2603
2604
2605	/**
2606	* Sets up miscellaneous (everything other than Pin & Processor-based
2607	* VM-execution) control fields in the VMCS.
2608	*
2609	* @returns VBox status code.
2610	* @param pVM The cross context VM structure.
2611	* @param pVCpu The cross context virtual CPU structure.
2612	*/
2613	static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
2614	{
2615	NOREF(pVM);
2616	AssertPtr(pVM);
2617	AssertPtr(pVCpu);
2618
2619	int rc = VERR_GENERAL_FAILURE;
2620
2621	/* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2622	#if 0
2623	/* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestCR3AndCR4())*/
2624	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0);
2625	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0);
2626
2627	/*
2628	* Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2629	* and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2630	* We thus use the exception bitmap to control it rather than use both.
2631	*/
2632	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0);
2633	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0);
2634
2635	/** @todo Explore possibility of using IO-bitmaps. */
2636	/* All IO & IOIO instructions cause VM-exits. */
2637	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0);
2638	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0);
2639
2640	/* Initialize the MSR-bitmap area. */
2641	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
2642	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0);
2643	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0);
2644	AssertRCReturn(rc, rc);
2645	#endif
2646
2647	/* Setup MSR auto-load/store area. */
2648	Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2649	Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2650	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2651	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2652	AssertRCReturn(rc, rc);
2653
2654	Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2655	Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2656	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2657	AssertRCReturn(rc, rc);
2658
2659	/* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2660	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2661	AssertRCReturn(rc, rc);
2662
2663	/* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2664	#if 0
2665	/* Setup debug controls */
2666	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
2667	rc \|= VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2668	AssertRCReturn(rc, rc);
2669	#endif
2670
2671	return rc;
2672	}
2673
2674
2675	/**
2676	* Sets up the initial exception bitmap in the VMCS based on static conditions.
2677	*
2678	* We shall setup those exception intercepts that don't change during the
2679	* lifetime of the VM here. The rest are done dynamically while loading the
2680	* guest state.
2681	*
2682	* @returns VBox status code.
2683	* @param pVM The cross context VM structure.
2684	* @param pVCpu The cross context virtual CPU structure.
2685	*/
2686	static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
2687	{
2688	AssertPtr(pVM);
2689	AssertPtr(pVCpu);
2690
2691	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
2692
2693	uint32_t u32XcptBitmap = 0;
2694
2695	/* Must always intercept #AC to prevent the guest from hanging the CPU. */
2696	u32XcptBitmap \|= RT_BIT_32(X86_XCPT_AC);
2697
2698	/* Because we need to maintain the DR6 state even when intercepting DRx reads
2699	and writes, and because recursive #DBs can cause the CPU hang, we must always
2700	intercept #DB. */
2701	u32XcptBitmap \|= RT_BIT_32(X86_XCPT_DB);
2702
2703	/* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2704	if (!pVM->hm.s.fNestedPaging)
2705	u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
2706
2707	pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2708	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2709	AssertRCReturn(rc, rc);
2710	return rc;
2711	}
2712
2713
2714	/**
2715	* Sets up the initial guest-state mask. The guest-state mask is consulted
2716	* before reading guest-state fields from the VMCS as VMREADs can be expensive
2717	* for the nested virtualization case (as it would cause a VM-exit).
2718	*
2719	* @param pVCpu The cross context virtual CPU structure.
2720	*/
2721	static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
2722	{
2723	/* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
2724	HMVMXCPU_GST_RESET_TO(pVCpu, HMVMX_UPDATED_GUEST_ALL);
2725	return VINF_SUCCESS;
2726	}
2727
2728
2729	/**
2730	* Does per-VM VT-x initialization.
2731	*
2732	* @returns VBox status code.
2733	* @param pVM The cross context VM structure.
2734	*/
2735	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2736	{
2737	LogFlowFunc(("pVM=%p\n", pVM));
2738
2739	int rc = hmR0VmxStructsAlloc(pVM);
2740	if (RT_FAILURE(rc))
2741	{
2742	LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2743	return rc;
2744	}
2745
2746	return VINF_SUCCESS;
2747	}
2748
2749
2750	/**
2751	* Does per-VM VT-x termination.
2752	*
2753	* @returns VBox status code.
2754	* @param pVM The cross context VM structure.
2755	*/
2756	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2757	{
2758	LogFlowFunc(("pVM=%p\n", pVM));
2759
2760	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2761	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2762	ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2763	#endif
2764	hmR0VmxStructsFree(pVM);
2765	return VINF_SUCCESS;
2766	}
2767
2768
2769	/**
2770	* Sets up the VM for execution under VT-x.
2771	* This function is only called once per-VM during initialization.
2772	*
2773	* @returns VBox status code.
2774	* @param pVM The cross context VM structure.
2775	*/
2776	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2777	{
2778	AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2779	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2780
2781	LogFlowFunc(("pVM=%p\n", pVM));
2782
2783	/*
2784	* Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable must always be allocated.
2785	* We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0Intel().
2786	*/
2787	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2788	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2789	\|\| !pVM->hm.s.vmx.pRealModeTSS))
2790	{
2791	LogRel(("VMXR0SetupVM: Invalid real-on-v86 state.\n"));
2792	return VERR_INTERNAL_ERROR;
2793	}
2794
2795	/* Initialize these always, see hmR3InitFinalizeR0().*/
2796	pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NONE;
2797	pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NONE;
2798
2799	/* Setup the tagged-TLB flush handlers. */
2800	int rc = hmR0VmxSetupTaggedTlb(pVM);
2801	if (RT_FAILURE(rc))
2802	{
2803	LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2804	return rc;
2805	}
2806
2807	/* Check if we can use the VMCS controls for swapping the EFER MSR. */
2808	Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
2809	#if HC_ARCH_BITS == 64
2810	if ( (pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1 & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
2811	&& (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
2812	&& (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR))
2813	{
2814	pVM->hm.s.vmx.fSupportsVmcsEfer = true;
2815	}
2816	#endif
2817
2818	/* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
2819	RTCCUINTREG uHostCR4 = ASMGetCR4();
2820	if (RT_UNLIKELY(!(uHostCR4 & X86_CR4_VMXE)))
2821	return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
2822
2823	for (VMCPUID i = 0; i < pVM->cCpus; i++)
2824	{
2825	PVMCPU pVCpu = &pVM->aCpus[i];
2826	AssertPtr(pVCpu);
2827	AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2828
2829	/* Log the VCPU pointers, useful for debugging SMP VMs. */
2830	Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2831
2832	/* Initialize the VM-exit history array with end-of-array markers (UINT16_MAX). */
2833	Assert(!pVCpu->hm.s.idxExitHistoryFree);
2834	HMCPU_EXIT_HISTORY_RESET(pVCpu);
2835
2836	/* Set revision dword at the beginning of the VMCS structure. */
2837	(uint32_t )pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2838
2839	/* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2840	rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2841	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2842	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2843
2844	/* Load this VMCS as the current VMCS. */
2845	rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2846	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2847	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2848
2849	rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2850	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2851	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2852
2853	rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2854	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2855	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2856
2857	rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2858	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2859	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2860
2861	rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2862	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2863	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2864
2865	rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2866	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2867	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2868
2869	#if HC_ARCH_BITS == 32
2870	rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2871	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2872	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2873	#endif
2874
2875	/* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2876	rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2877	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2878	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2879
2880	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2881
2882	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2883	}
2884
2885	return VINF_SUCCESS;
2886	}
2887
2888
2889	/**
2890	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2891	* the VMCS.
2892	*
2893	* @returns VBox status code.
2894	* @param pVM The cross context VM structure.
2895	* @param pVCpu The cross context virtual CPU structure.
2896	*/
2897	DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2898	{
2899	NOREF(pVM); NOREF(pVCpu);
2900
2901	RTCCUINTREG uReg = ASMGetCR0();
2902	int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2903	AssertRCReturn(rc, rc);
2904
2905	uReg = ASMGetCR3();
2906	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2907	AssertRCReturn(rc, rc);
2908
2909	uReg = ASMGetCR4();
2910	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2911	AssertRCReturn(rc, rc);
2912	return rc;
2913	}
2914
2915
2916	#if HC_ARCH_BITS == 64
2917	/**
2918	* Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2919	* requirements. See hmR0VmxSaveHostSegmentRegs().
2920	*/
2921	# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2922	if ((selValue) & (X86_SEL_RPL \| X86_SEL_LDT)) \
2923	{ \
2924	bool fValidSelector = true; \
2925	if ((selValue) & X86_SEL_LDT) \
2926	{ \
2927	uint32_t uAttr = ASMGetSegAttr((selValue)); \
2928	fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
2929	} \
2930	if (fValidSelector) \
2931	{ \
2932	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_##seg; \
2933	pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2934	} \
2935	(selValue) = 0; \
2936	}
2937	#endif
2938
2939
2940	/**
2941	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2942	* the host-state area in the VMCS.
2943	*
2944	* @returns VBox status code.
2945	* @param pVM The cross context VM structure.
2946	* @param pVCpu The cross context virtual CPU structure.
2947	*/
2948	DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2949	{
2950	int rc = VERR_INTERNAL_ERROR_5;
2951
2952	#if HC_ARCH_BITS == 64
2953	/*
2954	* If we've executed guest code using VT-x, the host-state bits will be messed up. We
2955	* should -not- save the messed up state without restoring the original host-state. See @bugref{7240}.
2956	*
2957	* This apparently can happen (most likely the FPU changes), deal with it rather than asserting.
2958	* Was observed booting Solaris10u10 32-bit guest.
2959	*/
2960	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
2961	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
2962	{
2963	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags,
2964	pVCpu->idCpu));
2965	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
2966	}
2967	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2968	#else
2969	RT_NOREF(pVCpu);
2970	#endif
2971
2972	/*
2973	* Host DS, ES, FS and GS segment registers.
2974	*/
2975	#if HC_ARCH_BITS == 64
2976	RTSEL uSelDS = ASMGetDS();
2977	RTSEL uSelES = ASMGetES();
2978	RTSEL uSelFS = ASMGetFS();
2979	RTSEL uSelGS = ASMGetGS();
2980	#else
2981	RTSEL uSelDS = 0;
2982	RTSEL uSelES = 0;
2983	RTSEL uSelFS = 0;
2984	RTSEL uSelGS = 0;
2985	#endif
2986
2987	/*
2988	* Host CS and SS segment registers.
2989	*/
2990	RTSEL uSelCS = ASMGetCS();
2991	RTSEL uSelSS = ASMGetSS();
2992
2993	/*
2994	* Host TR segment register.
2995	*/
2996	RTSEL uSelTR = ASMGetTR();
2997
2998	#if HC_ARCH_BITS == 64
2999	/*
3000	* Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
3001	* before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
3002	*/
3003	VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
3004	VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
3005	VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
3006	VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
3007	# undef VMXLOCAL_ADJUST_HOST_SEG
3008	#endif
3009
3010	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
3011	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
3012	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
3013	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
3014	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
3015	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
3016	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
3017	Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
3018	Assert(uSelCS);
3019	Assert(uSelTR);
3020
3021	/* Assertion is right but we would not have updated u32ExitCtls yet. */
3022	#if 0
3023	if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
3024	Assert(uSelSS != 0);
3025	#endif
3026
3027	/* Write these host selector fields into the host-state area in the VMCS. */
3028	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_CS_SEL, uSelCS);
3029	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_SS_SEL, uSelSS);
3030	#if HC_ARCH_BITS == 64
3031	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_DS_SEL, uSelDS);
3032	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_ES_SEL, uSelES);
3033	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FS_SEL, uSelFS);
3034	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_GS_SEL, uSelGS);
3035	#else
3036	NOREF(uSelDS);
3037	NOREF(uSelES);
3038	NOREF(uSelFS);
3039	NOREF(uSelGS);
3040	#endif
3041	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_TR_SEL, uSelTR);
3042	AssertRCReturn(rc, rc);
3043
3044	/*
3045	* Host GDTR and IDTR.
3046	*/
3047	RTGDTR Gdtr;
3048	RTIDTR Idtr;
3049	RT_ZERO(Gdtr);
3050	RT_ZERO(Idtr);
3051	ASMGetGDTR(&Gdtr);
3052	ASMGetIDTR(&Idtr);
3053	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
3054	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
3055	AssertRCReturn(rc, rc);
3056
3057	#if HC_ARCH_BITS == 64
3058	/*
3059	* Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
3060	* maximum limit (0xffff) on every VM-exit.
3061	*/
3062	if (Gdtr.cbGdt != 0xffff)
3063	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDTR;
3064
3065	/*
3066	* IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT"
3067	* and Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit as 0xfff, VT-x
3068	* bloating the limit to 0xffff shouldn't cause any different CPU behavior. However, several hosts either insists
3069	* on 0xfff being the limit (Windows Patch Guard) or uses the limit for other purposes (darwin puts the CPU ID in there
3070	* but botches sidt alignment in at least one consumer). So, we're only allowing IDTR.LIMIT to be left at 0xffff on
3071	* hosts where we are pretty sure it won't cause trouble.
3072	*/
3073	# if defined(RT_OS_LINUX) \|\| defined(RT_OS_SOLARIS)
3074	if (Idtr.cbIdt < 0x0fff)
3075	# else
3076	if (Idtr.cbIdt != 0xffff)
3077	# endif
3078	{
3079	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_IDTR;
3080	AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
3081	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
3082	}
3083	#endif
3084
3085	/*
3086	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
3087	* is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
3088	*/
3089	AssertMsgReturn((uSelTR \| X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
3090	("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt),
3091	VERR_VMX_INVALID_HOST_STATE);
3092
3093	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
3094	#if HC_ARCH_BITS == 64
3095	uintptr_t uTRBase = X86DESC64_BASE(pDesc);
3096
3097	/*
3098	* VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on all VM-exits.
3099	* The type is the same for 64-bit busy TSS[1]. The limit needs manual restoration if the host has something else.
3100	* Task switching is not supported in 64-bit mode[2], but the limit still matters as IOPM is supported in 64-bit mode.
3101	* Restoring the limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
3102	*
3103	* [1] See Intel spec. 3.5 "System Descriptor Types".
3104	* [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
3105	*/
3106	Assert(pDesc->System.u4Type == 11);
3107	if ( pDesc->System.u16LimitLow != 0x67
3108	\|\| pDesc->System.u4LimitHigh)
3109	{
3110	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_TR;
3111	/* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
3112	if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
3113	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDT_READ_ONLY;
3114	pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
3115	}
3116
3117	/*
3118	* Store the GDTR as we need it when restoring the GDT and while restoring the TR.
3119	*/
3120	if (pVCpu->hm.s.vmx.fRestoreHostFlags & (VMX_RESTORE_HOST_GDTR \| VMX_RESTORE_HOST_SEL_TR))
3121	{
3122	AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
3123	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3124	if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_NEED_WRITABLE)
3125	{
3126	/* The GDT is read-only but the writable GDT is available. */
3127	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDT_NEED_WRITABLE;
3128	pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.cb = Gdtr.cbGdt;
3129	rc = SUPR0GetCurrentGdtRw(&pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.uAddr);
3130	AssertRCReturn(rc, rc);
3131	}
3132	}
3133	#else
3134	NOREF(pVM);
3135	uintptr_t uTRBase = X86DESC_BASE(pDesc);
3136	#endif
3137	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
3138	AssertRCReturn(rc, rc);
3139
3140	/*
3141	* Host FS base and GS base.
3142	*/
3143	#if HC_ARCH_BITS == 64
3144	uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
3145	uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
3146	rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase);
3147	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase);
3148	AssertRCReturn(rc, rc);
3149
3150	/* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
3151	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
3152	pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
3153	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
3154	pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
3155	#endif
3156	return rc;
3157	}
3158
3159
3160	/**
3161	* Saves certain host MSRs in the VM-exit MSR-load area and some in the
3162	* host-state area of the VMCS. Theses MSRs will be automatically restored on
3163	* the host after every successful VM-exit.
3164	*
3165	* @returns VBox status code.
3166	* @param pVM The cross context VM structure.
3167	* @param pVCpu The cross context virtual CPU structure.
3168	*
3169	* @remarks No-long-jump zone!!!
3170	*/
3171	DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
3172	{
3173	NOREF(pVM);
3174
3175	AssertPtr(pVCpu);
3176	AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3177
3178	/*
3179	* Save MSRs that we restore lazily (due to preemption or transition to ring-3)
3180	* rather than swapping them on every VM-entry.
3181	*/
3182	hmR0VmxLazySaveHostMsrs(pVCpu);
3183
3184	/*
3185	* Host Sysenter MSRs.
3186	*/
3187	int rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3188	#if HC_ARCH_BITS == 32
3189	rc \|= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3190	rc \|= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3191	#else
3192	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3193	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3194	#endif
3195	AssertRCReturn(rc, rc);
3196
3197	/*
3198	* Host EFER MSR.
3199	* If the CPU supports the newer VMCS controls for managing EFER, use it.
3200	* Otherwise it's done as part of auto-load/store MSR area in the VMCS, see hmR0VmxLoadGuestMsrs().
3201	*/
3202	if (pVM->hm.s.vmx.fSupportsVmcsEfer)
3203	{
3204	rc = VMXWriteVmcs64(VMX_VMCS64_HOST_EFER_FULL, pVM->hm.s.vmx.u64HostEfer);
3205	AssertRCReturn(rc, rc);
3206	}
3207
3208	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see
3209	* hmR0VmxLoadGuestExitCtls() !! */
3210
3211	return rc;
3212	}
3213
3214
3215	/**
3216	* Figures out if we need to swap the EFER MSR which is particularly expensive.
3217	*
3218	* We check all relevant bits. For now, that's everything besides LMA/LME, as
3219	* these two bits are handled by VM-entry, see hmR0VmxLoadGuestExitCtls() and
3220	* hmR0VMxLoadGuestEntryCtls().
3221	*
3222	* @returns true if we need to load guest EFER, false otherwise.
3223	* @param pVCpu The cross context virtual CPU structure.
3224	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3225	* out-of-sync. Make sure to update the required fields
3226	* before using them.
3227	*
3228	* @remarks Requires EFER, CR4.
3229	* @remarks No-long-jump zone!!!
3230	*/
3231	static bool hmR0VmxShouldSwapEferMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3232	{
3233	#ifdef HMVMX_ALWAYS_SWAP_EFER
3234	return true;
3235	#endif
3236
3237	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
3238	/* For 32-bit hosts running 64-bit guests, we always swap EFER in the world-switcher. Nothing to do here. */
3239	if (CPUMIsGuestInLongMode(pVCpu))
3240	return false;
3241	#endif
3242
3243	PVM pVM = pVCpu->CTX_SUFF(pVM);
3244	uint64_t u64HostEfer = pVM->hm.s.vmx.u64HostEfer;
3245	uint64_t u64GuestEfer = pMixedCtx->msrEFER;
3246
3247	/*
3248	* For 64-bit guests, if EFER.SCE bit differs, we need to swap to ensure that the
3249	* guest's SYSCALL behaviour isn't screwed. See @bugref{7386}.
3250	*/
3251	if ( CPUMIsGuestInLongMode(pVCpu)
3252	&& (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
3253	{
3254	return true;
3255	}
3256
3257	/*
3258	* If the guest uses PAE and EFER.NXE bit differs, we need to swap EFER as it
3259	* affects guest paging. 64-bit paging implies CR4.PAE as well.
3260	* See Intel spec. 4.5 "IA-32e Paging" and Intel spec. 4.1.1 "Three Paging Modes".
3261	*/
3262	if ( (pMixedCtx->cr4 & X86_CR4_PAE)
3263	&& (pMixedCtx->cr0 & X86_CR0_PG)
3264	&& (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
3265	{
3266	/* Assert that host is PAE capable. */
3267	Assert(pVM->hm.s.cpuid.u32AMDFeatureEDX & X86_CPUID_EXT_FEATURE_EDX_NX);
3268	return true;
3269	}
3270
3271	/** @todo Check the latest Intel spec. for any other bits,
3272	* like SMEP/SMAP? */
3273	return false;
3274	}
3275
3276
3277	/**
3278	* Sets up VM-entry controls in the VMCS. These controls can affect things done
3279	* on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
3280	* controls".
3281	*
3282	* @returns VBox status code.
3283	* @param pVCpu The cross context virtual CPU structure.
3284	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3285	* out-of-sync. Make sure to update the required fields
3286	* before using them.
3287	*
3288	* @remarks Requires EFER.
3289	* @remarks No-long-jump zone!!!
3290	*/
3291	DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3292	{
3293	int rc = VINF_SUCCESS;
3294	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS))
3295	{
3296	PVM pVM = pVCpu->CTX_SUFF(pVM);
3297	uint32_t val = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
3298	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3299
3300	/* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3301	val \|= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
3302
3303	/* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3304	if (CPUMIsGuestInLongModeEx(pMixedCtx))
3305	{
3306	val \|= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
3307	Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST\n", pVCpu->idCpu));
3308	}
3309	else
3310	Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
3311
3312	/* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it. */
3313	if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3314	&& hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3315	{
3316	val \|= VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR;
3317	Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR\n", pVCpu->idCpu));
3318	}
3319
3320	/*
3321	* The following should -not- be set (since we're not in SMM mode):
3322	* - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
3323	* - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
3324	*/
3325
3326	/** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
3327	* VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR. */
3328
3329	if ((val & zap) != val)
3330	{
3331	LogRel(("hmR0VmxLoadGuestEntryCtls: Invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3332	pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, val, zap));
3333	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3334	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3335	}
3336
3337	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
3338	AssertRCReturn(rc, rc);
3339
3340	pVCpu->hm.s.vmx.u32EntryCtls = val;
3341	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS);
3342	}
3343	return rc;
3344	}
3345
3346
3347	/**
3348	* Sets up the VM-exit controls in the VMCS.
3349	*
3350	* @returns VBox status code.
3351	* @param pVCpu The cross context virtual CPU structure.
3352	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3353	* out-of-sync. Make sure to update the required fields
3354	* before using them.
3355	*
3356	* @remarks Requires EFER.
3357	*/
3358	DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3359	{
3360	NOREF(pMixedCtx);
3361
3362	int rc = VINF_SUCCESS;
3363	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_EXIT_CTLS))
3364	{
3365	PVM pVM = pVCpu->CTX_SUFF(pVM);
3366	uint32_t val = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
3367	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3368
3369	/* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3370	val \|= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
3371
3372	/*
3373	* Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3374	* On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
3375	*/
3376	#if HC_ARCH_BITS == 64
3377	val \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3378	Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3379	#else
3380	Assert( pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64
3381	\|\| pVCpu->hm.s.vmx.pfnStartVM == VMXR0StartVM32);
3382	/* Set the host address-space size based on the switcher, not guest state. See @bugref{8432}. */
3383	if (pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64)
3384	{
3385	/* The switcher returns to long mode, EFER is managed by the switcher. */
3386	val \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3387	Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3388	}
3389	else
3390	Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3391	#endif
3392
3393	/* If the newer VMCS fields for managing EFER exists, use it. */
3394	if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3395	&& hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3396	{
3397	val \|= VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR
3398	\| VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR;
3399	Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR, VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR\n", pVCpu->idCpu));
3400	}
3401
3402	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3403	Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
3404
3405	/** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
3406	* VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
3407	* VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR. */
3408
3409	if ( pVM->hm.s.vmx.fUsePreemptTimer
3410	&& (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER))
3411	val \|= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
3412
3413	if ((val & zap) != val)
3414	{
3415	LogRel(("hmR0VmxSetupProcCtls: Invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3416	pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, val, zap));
3417	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3418	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3419	}
3420
3421	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
3422	AssertRCReturn(rc, rc);
3423
3424	pVCpu->hm.s.vmx.u32ExitCtls = val;
3425	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_EXIT_CTLS);
3426	}
3427	return rc;
3428	}
3429
3430
3431	/**
3432	* Sets the TPR threshold in the VMCS.
3433	*
3434	* @returns VBox status code.
3435	* @param pVCpu The cross context virtual CPU structure.
3436	* @param u32TprThreshold The TPR threshold (task-priority class only).
3437	*/
3438	DECLINLINE(int) hmR0VmxApicSetTprThreshold(PVMCPU pVCpu, uint32_t u32TprThreshold)
3439	{
3440	Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3441	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW); RT_NOREF_PV(pVCpu);
3442	return VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3443	}
3444
3445
3446	/**
3447	* Loads the guest APIC and related state.
3448	*
3449	* @returns VBox status code.
3450	* @param pVCpu The cross context virtual CPU structure.
3451	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3452	* out-of-sync. Make sure to update the required fields
3453	* before using them.
3454	*
3455	* @remarks No-long-jump zone!!!
3456	*/
3457	DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3458	{
3459	NOREF(pMixedCtx);
3460
3461	int rc = VINF_SUCCESS;
3462	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE))
3463	{
3464	if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
3465	&& APICIsEnabled(pVCpu))
3466	{
3467	/*
3468	* Setup TPR shadowing.
3469	*/
3470	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
3471	{
3472	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3473
3474	bool fPendingIntr = false;
3475	uint8_t u8Tpr = 0;
3476	uint8_t u8PendingIntr = 0;
3477	rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3478	AssertRCReturn(rc, rc);
3479
3480	/*
3481	* If there are interrupts pending but masked by the TPR, instruct VT-x to cause a TPR-below-threshold VM-exit
3482	* when the guest lowers its TPR below the priority of the pending interrupt so we can deliver the interrupt.
3483	* If there are no interrupts pending, set threshold to 0 to not cause any TPR-below-threshold VM-exits.
3484	*/
3485	pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR] = u8Tpr;
3486	uint32_t u32TprThreshold = 0;
3487	if (fPendingIntr)
3488	{
3489	/* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3490	const uint8_t u8PendingPriority = u8PendingIntr >> 4;
3491	const uint8_t u8TprPriority = u8Tpr >> 4;
3492	if (u8PendingPriority <= u8TprPriority)
3493	u32TprThreshold = u8PendingPriority;
3494	}
3495
3496	rc = hmR0VmxApicSetTprThreshold(pVCpu, u32TprThreshold);
3497	AssertRCReturn(rc, rc);
3498	}
3499	}
3500	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
3501	}
3502
3503	return rc;
3504	}
3505
3506
3507	/**
3508	* Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3509	*
3510	* @returns Guest's interruptibility-state.
3511	* @param pVCpu The cross context virtual CPU structure.
3512	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3513	* out-of-sync. Make sure to update the required fields
3514	* before using them.
3515	*
3516	* @remarks No-long-jump zone!!!
3517	*/
3518	DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3519	{
3520	/*
3521	* Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
3522	*/
3523	uint32_t uIntrState = 0;
3524	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3525	{
3526	/* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
3527	AssertMsg(HMVMXCPU_GST_IS_SET(pVCpu, HMVMX_UPDATED_GUEST_RIP \| HMVMX_UPDATED_GUEST_RFLAGS),
3528	("%#x\n", HMVMXCPU_GST_VALUE(pVCpu)));
3529	if (pMixedCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
3530	{
3531	if (pMixedCtx->eflags.Bits.u1IF)
3532	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
3533	else
3534	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
3535	}
3536	else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3537	{
3538	/*
3539	* We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
3540	* VT-x, the flag's condition to be cleared is met and thus the cleared state is correct.
3541	*/
3542	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3543	}
3544	}
3545
3546	/*
3547	* NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
3548	* bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
3549	* setting this would block host-NMIs and IRET will not clear the blocking.
3550	*
3551	* See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
3552	*/
3553	if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
3554	&& (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
3555	{
3556	uIntrState \|= VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI;
3557	}
3558
3559	return uIntrState;
3560	}
3561
3562
3563	/**
3564	* Loads the guest's interruptibility-state into the guest-state area in the
3565	* VMCS.
3566	*
3567	* @returns VBox status code.
3568	* @param pVCpu The cross context virtual CPU structure.
3569	* @param uIntrState The interruptibility-state to set.
3570	*/
3571	static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
3572	{
3573	NOREF(pVCpu);
3574	AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
3575	Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3576	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
3577	AssertRC(rc);
3578	return rc;
3579	}
3580
3581
3582	/**
3583	* Loads the exception intercepts required for guest execution in the VMCS.
3584	*
3585	* @returns VBox status code.
3586	* @param pVCpu The cross context virtual CPU structure.
3587	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3588	* out-of-sync. Make sure to update the required fields
3589	* before using them.
3590	*/
3591	static int hmR0VmxLoadGuestXcptIntercepts(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3592	{
3593	NOREF(pMixedCtx);
3594	int rc = VINF_SUCCESS;
3595	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS))
3596	{
3597	/* The remaining exception intercepts are handled elsewhere, e.g. in hmR0VmxLoadSharedCR0(). */
3598	if (pVCpu->hm.s.fGIMTrapXcptUD)
3599	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_UD);
3600	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3601	else
3602	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_UD);
3603	#endif
3604
3605	Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_AC));
3606	Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
3607
3608	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3609	AssertRCReturn(rc, rc);
3610
3611	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
3612	Log4(("Load[%RU32]: VMX_VMCS32_CTRL_EXCEPTION_BITMAP=%#RX64 fContextUseFlags=%#RX32\n", pVCpu->idCpu,
3613	pVCpu->hm.s.vmx.u32XcptBitmap, HMCPU_CF_VALUE(pVCpu)));
3614	}
3615	return rc;
3616	}
3617
3618
3619	/**
3620	* Loads the guest's RIP into the guest-state area in the VMCS.
3621	*
3622	* @returns VBox status code.
3623	* @param pVCpu The cross context virtual CPU structure.
3624	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3625	* out-of-sync. Make sure to update the required fields
3626	* before using them.
3627	*
3628	* @remarks No-long-jump zone!!!
3629	*/
3630	static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3631	{
3632	int rc = VINF_SUCCESS;
3633	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP))
3634	{
3635	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
3636	AssertRCReturn(rc, rc);
3637
3638	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RIP);
3639	Log4(("Load[%RU32]: VMX_VMCS_GUEST_RIP=%#RX64 fContextUseFlags=%#RX32\n", pVCpu->idCpu, pMixedCtx->rip,
3640	HMCPU_CF_VALUE(pVCpu)));
3641	}
3642	return rc;
3643	}
3644
3645
3646	/**
3647	* Loads the guest's RSP into the guest-state area in the VMCS.
3648	*
3649	* @returns VBox status code.
3650	* @param pVCpu The cross context virtual CPU structure.
3651	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3652	* out-of-sync. Make sure to update the required fields
3653	* before using them.
3654	*
3655	* @remarks No-long-jump zone!!!
3656	*/
3657	static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3658	{
3659	int rc = VINF_SUCCESS;
3660	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RSP))
3661	{
3662	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
3663	AssertRCReturn(rc, rc);
3664
3665	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RSP);
3666	Log4(("Load[%RU32]: VMX_VMCS_GUEST_RSP=%#RX64\n", pVCpu->idCpu, pMixedCtx->rsp));
3667	}
3668	return rc;
3669	}
3670
3671
3672	/**
3673	* Loads the guest's RFLAGS into the guest-state area in the VMCS.
3674	*
3675	* @returns VBox status code.
3676	* @param pVCpu The cross context virtual CPU structure.
3677	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3678	* out-of-sync. Make sure to update the required fields
3679	* before using them.
3680	*
3681	* @remarks No-long-jump zone!!!
3682	*/
3683	static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3684	{
3685	int rc = VINF_SUCCESS;
3686	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
3687	{
3688	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3689	Let us assert it as such and use 32-bit VMWRITE. */
3690	Assert(!(pMixedCtx->rflags.u64 >> 32));
3691	X86EFLAGS Eflags = pMixedCtx->eflags;
3692	/** @todo r=bird: There shall be no need to OR in X86_EFL_1 here, nor
3693	* shall there be any reason for clearing bits 63:22, 15, 5 and 3.
3694	* These will never be cleared/set, unless some other part of the VMM
3695	* code is buggy - in which case we're better of finding and fixing
3696	* those bugs than hiding them. */
3697	Assert(Eflags.u32 & X86_EFL_RA1_MASK);
3698	Assert(!(Eflags.u32 & ~(X86_EFL_1 \| X86_EFL_LIVE_MASK)));
3699	Eflags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
3700	Eflags.u32 \|= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
3701
3702	/*
3703	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM-exit.
3704	* Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
3705	*/
3706	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3707	{
3708	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3709	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3710	pVCpu->hm.s.vmx.RealMode.Eflags.u32 = Eflags.u32; /* Save the original eflags of the real-mode guest. */
3711	Eflags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3712	Eflags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3713	}
3714
3715	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, Eflags.u32);
3716	AssertRCReturn(rc, rc);
3717
3718	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3719	Log4(("Load[%RU32]: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", pVCpu->idCpu, Eflags.u32));
3720	}
3721	return rc;
3722	}
3723
3724
3725	/**
3726	* Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
3727	*
3728	* @returns VBox status code.
3729	* @param pVCpu The cross context virtual CPU structure.
3730	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3731	* out-of-sync. Make sure to update the required fields
3732	* before using them.
3733	*
3734	* @remarks No-long-jump zone!!!
3735	*/
3736	DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3737	{
3738	int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
3739	rc \|= hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
3740	rc \|= hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
3741	AssertRCReturn(rc, rc);
3742	return rc;
3743	}
3744
3745
3746	/**
3747	* Loads the guest CR0 control register into the guest-state area in the VMCS.
3748	* CR0 is partially shared with the host and we have to consider the FPU bits.
3749	*
3750	* @returns VBox status code.
3751	* @param pVCpu The cross context virtual CPU structure.
3752	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3753	* out-of-sync. Make sure to update the required fields
3754	* before using them.
3755	*
3756	* @remarks No-long-jump zone!!!
3757	*/
3758	static int hmR0VmxLoadSharedCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3759	{
3760	/*
3761	* Guest CR0.
3762	* Guest FPU.
3763	*/
3764	int rc = VINF_SUCCESS;
3765	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
3766	{
3767	Assert(!(pMixedCtx->cr0 >> 32));
3768	uint32_t u32GuestCR0 = pMixedCtx->cr0;
3769	PVM pVM = pVCpu->CTX_SUFF(pVM);
3770
3771	/* The guest's view (read access) of its CR0 is unblemished. */
3772	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
3773	AssertRCReturn(rc, rc);
3774	Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR0));
3775
3776	/* Setup VT-x's view of the guest CR0. */
3777	/* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
3778	if (pVM->hm.s.fNestedPaging)
3779	{
3780	if (CPUMIsGuestPagingEnabledEx(pMixedCtx))
3781	{
3782	/* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3783	pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3784	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3785	}
3786	else
3787	{
3788	/* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3789	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3790	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3791	}
3792
3793	/* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3794	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3795	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3796
3797	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3798	AssertRCReturn(rc, rc);
3799	}
3800	else
3801	u32GuestCR0 \|= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3802
3803	/*
3804	* Guest FPU bits.
3805	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
3806	* CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3807	*/
3808	u32GuestCR0 \|= X86_CR0_NE;
3809	bool fInterceptNM = false;
3810	if (CPUMIsGuestFPUStateActive(pVCpu))
3811	{
3812	fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
3813	/* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
3814	We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
3815	}
3816	else
3817	{
3818	fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
3819	u32GuestCR0 \|= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
3820	\| X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
3821	}
3822
3823	/* Catch floating point exceptions if we need to report them to the guest in a different way. */
3824	bool fInterceptMF = false;
3825	if (!(pMixedCtx->cr0 & X86_CR0_NE))
3826	fInterceptMF = true;
3827
3828	/* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
3829	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3830	{
3831	Assert(PDMVmmDevHeapIsEnabled(pVM));
3832	Assert(pVM->hm.s.vmx.pRealModeTSS);
3833	pVCpu->hm.s.vmx.u32XcptBitmap \|= HMVMX_REAL_MODE_XCPT_MASK;
3834	fInterceptNM = true;
3835	fInterceptMF = true;
3836	}
3837	else
3838	{
3839	/* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
3840	pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3841	}
3842	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
3843
3844	if (fInterceptNM)
3845	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_NM);
3846	else
3847	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
3848
3849	if (fInterceptMF)
3850	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_MF);
3851	else
3852	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
3853
3854	/* Additional intercepts for debugging, define these yourself explicitly. */
3855	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3856	pVCpu->hm.s.vmx.u32XcptBitmap \|= 0
3857	\| RT_BIT(X86_XCPT_BP)
3858	\| RT_BIT(X86_XCPT_DE)
3859	\| RT_BIT(X86_XCPT_NM)
3860	\| RT_BIT(X86_XCPT_TS)
3861	\| RT_BIT(X86_XCPT_UD)
3862	\| RT_BIT(X86_XCPT_NP)
3863	\| RT_BIT(X86_XCPT_SS)
3864	\| RT_BIT(X86_XCPT_GP)
3865	\| RT_BIT(X86_XCPT_PF)
3866	\| RT_BIT(X86_XCPT_MF)
3867	;
3868	#elif defined(HMVMX_ALWAYS_TRAP_PF)
3869	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
3870	#endif
3871
3872	Assert(pVM->hm.s.fNestedPaging \|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3873
3874	/* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
3875	uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3876	uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3877	if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3878	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
3879	else
3880	Assert((uSetCR0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
3881
3882	u32GuestCR0 \|= uSetCR0;
3883	u32GuestCR0 &= uZapCR0;
3884	u32GuestCR0 &= ~(X86_CR0_CD \| X86_CR0_NW); /* Always enable caching. */
3885
3886	/* Write VT-x's view of the guest CR0 into the VMCS. */
3887	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
3888	AssertRCReturn(rc, rc);
3889	Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", pVCpu->idCpu, u32GuestCR0, uSetCR0,
3890	uZapCR0));
3891
3892	/*
3893	* CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3894	* by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3895	* we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3896	*/
3897	uint32_t u32CR0Mask = 0;
3898	u32CR0Mask = X86_CR0_PE
3899	\| X86_CR0_NE
3900	\| X86_CR0_WP
3901	\| X86_CR0_PG
3902	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3903	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3904	\| X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3905
3906	/** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3907	* enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3908	* and @bugref{6944}. */
3909	#if 0
3910	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3911	u32CR0Mask &= ~X86_CR0_PE;
3912	#endif
3913	if (pVM->hm.s.fNestedPaging)
3914	u32CR0Mask &= ~X86_CR0_WP;
3915
3916	/* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
3917	if (fInterceptNM)
3918	{
3919	u32CR0Mask \|= X86_CR0_TS
3920	\| X86_CR0_MP;
3921	}
3922
3923	/* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3924	pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3925	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3926	AssertRCReturn(rc, rc);
3927	Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_MASK=%#RX32\n", pVCpu->idCpu, u32CR0Mask));
3928
3929	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR0);
3930	}
3931	return rc;
3932	}
3933
3934
3935	/**
3936	* Loads the guest control registers (CR3, CR4) into the guest-state area
3937	* in the VMCS.
3938	*
3939	* @returns VBox strict status code.
3940	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
3941	* without unrestricted guest access and the VMMDev is not presently
3942	* mapped (e.g. EFI32).
3943	*
3944	* @param pVCpu The cross context virtual CPU structure.
3945	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3946	* out-of-sync. Make sure to update the required fields
3947	* before using them.
3948	*
3949	* @remarks No-long-jump zone!!!
3950	*/
3951	static VBOXSTRICTRC hmR0VmxLoadGuestCR3AndCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3952	{
3953	int rc = VINF_SUCCESS;
3954	PVM pVM = pVCpu->CTX_SUFF(pVM);
3955
3956	/*
3957	* Guest CR2.
3958	* It's always loaded in the assembler code. Nothing to do here.
3959	*/
3960
3961	/*
3962	* Guest CR3.
3963	*/
3964	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3))
3965	{
3966	RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3967	if (pVM->hm.s.fNestedPaging)
3968	{
3969	pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3970
3971	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3972	Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3973	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3974	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3975
3976	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3977	pVCpu->hm.s.vmx.HCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
3978	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3979
3980	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3981	AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3982	&& ((pVCpu->hm.s.vmx.HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
3983	("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3984	AssertMsg( !((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
3985	\|\| (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
3986	("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3987
3988	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3989	AssertRCReturn(rc, rc);
3990	Log4(("Load[%RU32]: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->idCpu, pVCpu->hm.s.vmx.HCPhysEPTP));
3991
3992	if ( pVM->hm.s.vmx.fUnrestrictedGuest
3993	\|\| CPUMIsGuestPagingEnabledEx(pMixedCtx))
3994	{
3995	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3996	if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3997	{
3998	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
3999	AssertRCReturn(rc, rc);
4000	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
4001	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
4002	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
4003	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
4004	AssertRCReturn(rc, rc);
4005	}
4006
4007	/* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
4008	have Unrestricted Execution to handle the guest when it's not using paging. */
4009	GCPhysGuestCR3 = pMixedCtx->cr3;
4010	}
4011	else
4012	{
4013	/*
4014	* The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
4015	* directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
4016	* EPT takes care of translating it to host-physical addresses.
4017	*/
4018	RTGCPHYS GCPhys;
4019	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
4020
4021	/* We obtain it here every time as the guest could have relocated this PCI region. */
4022	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
4023	if (RT_SUCCESS(rc))
4024	{ /* likely */ }
4025	else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
4026	{
4027	Log4(("Load[%RU32]: VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n", pVCpu->idCpu));
4028	return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
4029	}
4030	else
4031	AssertMsgFailedReturn(("%Rrc\n", rc), rc);
4032
4033	GCPhysGuestCR3 = GCPhys;
4034	}
4035
4036	Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RGp (GstN)\n", pVCpu->idCpu, GCPhysGuestCR3));
4037	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
4038	}
4039	else
4040	{
4041	/* Non-nested paging case, just use the hypervisor's CR3. */
4042	RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
4043
4044	Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", pVCpu->idCpu, HCPhysGuestCR3));
4045	rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
4046	}
4047	AssertRCReturn(rc, rc);
4048
4049	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR3);
4050	}
4051
4052	/*
4053	* Guest CR4.
4054	* ASSUMES this is done everytime we get in from ring-3! (XCR0)
4055	*/
4056	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4))
4057	{
4058	Assert(!(pMixedCtx->cr4 >> 32));
4059	uint32_t u32GuestCR4 = pMixedCtx->cr4;
4060
4061	/* The guest's view of its CR4 is unblemished. */
4062	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
4063	AssertRCReturn(rc, rc);
4064	Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR4));
4065
4066	/* Setup VT-x's view of the guest CR4. */
4067	/*
4068	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
4069	* interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
4070	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
4071	*/
4072	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4073	{
4074	Assert(pVM->hm.s.vmx.pRealModeTSS);
4075	Assert(PDMVmmDevHeapIsEnabled(pVM));
4076	u32GuestCR4 &= ~X86_CR4_VME;
4077	}
4078
4079	if (pVM->hm.s.fNestedPaging)
4080	{
4081	if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
4082	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
4083	{
4084	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
4085	u32GuestCR4 \|= X86_CR4_PSE;
4086	/* Our identity mapping is a 32-bit page directory. */
4087	u32GuestCR4 &= ~X86_CR4_PAE;
4088	}
4089	/* else use guest CR4.*/
4090	}
4091	else
4092	{
4093	/*
4094	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
4095	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
4096	*/
4097	switch (pVCpu->hm.s.enmShadowMode)
4098	{
4099	case PGMMODE_REAL: /* Real-mode. */
4100	case PGMMODE_PROTECTED: /* Protected mode without paging. */
4101	case PGMMODE_32_BIT: /* 32-bit paging. */
4102	{
4103	u32GuestCR4 &= ~X86_CR4_PAE;
4104	break;
4105	}
4106
4107	case PGMMODE_PAE: /* PAE paging. */
4108	case PGMMODE_PAE_NX: /* PAE paging with NX. */
4109	{
4110	u32GuestCR4 \|= X86_CR4_PAE;
4111	break;
4112	}
4113
4114	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
4115	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
4116	#ifdef VBOX_ENABLE_64_BITS_GUESTS
4117	break;
4118	#endif
4119	default:
4120	AssertFailed();
4121	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4122	}
4123	}
4124
4125	/* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
4126	uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4127	uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4128	u32GuestCR4 \|= uSetCR4;
4129	u32GuestCR4 &= uZapCR4;
4130
4131	/* Write VT-x's view of the guest CR4 into the VMCS. */
4132	Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", pVCpu->idCpu, u32GuestCR4, uSetCR4, uZapCR4));
4133	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
4134	AssertRCReturn(rc, rc);
4135
4136	/* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM-exit. */
4137	uint32_t u32CR4Mask = X86_CR4_VME
4138	\| X86_CR4_PAE
4139	\| X86_CR4_PGE
4140	\| X86_CR4_PSE
4141	\| X86_CR4_VMXE;
4142	if (pVM->cpum.ro.HostFeatures.fXSaveRstor)
4143	u32CR4Mask \|= X86_CR4_OSXSAVE;
4144	#if 0
4145	if (pVM->cpum.ro.GuestFeatures.fPcid)
4146	u32CR4Mask \|= X86_CR4_PCIDE;
4147	#endif
4148	pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
4149	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
4150	AssertRCReturn(rc, rc);
4151
4152	/* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
4153	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
4154
4155	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR4);
4156	}
4157	return rc;
4158	}
4159
4160
4161	/**
4162	* Loads the guest debug registers into the guest-state area in the VMCS.
4163	*
4164	* This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
4165	*
4166	* The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
4167	*
4168	* @returns VBox status code.
4169	* @param pVCpu The cross context virtual CPU structure.
4170	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4171	* out-of-sync. Make sure to update the required fields
4172	* before using them.
4173	*
4174	* @remarks No-long-jump zone!!!
4175	*/
4176	static int hmR0VmxLoadSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4177	{
4178	if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
4179	return VINF_SUCCESS;
4180
4181	#ifdef VBOX_STRICT
4182	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
4183	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
4184	{
4185	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
4186	Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK \| X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
4187	Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
4188	}
4189	#endif
4190
4191	int rc;
4192	PVM pVM = pVCpu->CTX_SUFF(pVM);
4193	bool fSteppingDB = false;
4194	bool fInterceptMovDRx = false;
4195	if (pVCpu->hm.s.fSingleInstruction)
4196	{
4197	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
4198	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
4199	{
4200	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
4201	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4202	AssertRCReturn(rc, rc);
4203	Assert(fSteppingDB == false);
4204	}
4205	else
4206	{
4207	pMixedCtx->eflags.u32 \|= X86_EFL_TF;
4208	pVCpu->hm.s.fClearTrapFlag = true;
4209	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
4210	fSteppingDB = true;
4211	}
4212	}
4213
4214	if ( fSteppingDB
4215	\|\| (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
4216	{
4217	/*
4218	* Use the combined guest and host DRx values found in the hypervisor
4219	* register set because the debugger has breakpoints active or someone
4220	* is single stepping on the host side without a monitor trap flag.
4221	*
4222	* Note! DBGF expects a clean DR6 state before executing guest code.
4223	*/
4224	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4225	if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4226	&& !CPUMIsHyperDebugStateActivePending(pVCpu))
4227	{
4228	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4229	Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
4230	Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
4231	}
4232	else
4233	#endif
4234	if (!CPUMIsHyperDebugStateActive(pVCpu))
4235	{
4236	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4237	Assert(CPUMIsHyperDebugStateActive(pVCpu));
4238	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
4239	}
4240
4241	/* Update DR7. (The other DRx values are handled by CPUM one way or the other.) */
4242	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)CPUMGetHyperDR7(pVCpu));
4243	AssertRCReturn(rc, rc);
4244
4245	pVCpu->hm.s.fUsingHyperDR7 = true;
4246	fInterceptMovDRx = true;
4247	}
4248	else
4249	{
4250	/*
4251	* If the guest has enabled debug registers, we need to load them prior to
4252	* executing guest code so they'll trigger at the right time.
4253	*/
4254	if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD)) /** @todo Why GD? */
4255	{
4256	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4257	if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4258	&& !CPUMIsGuestDebugStateActivePending(pVCpu))
4259	{
4260	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4261	Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
4262	Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
4263	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4264	}
4265	else
4266	#endif
4267	if (!CPUMIsGuestDebugStateActive(pVCpu))
4268	{
4269	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4270	Assert(CPUMIsGuestDebugStateActive(pVCpu));
4271	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
4272	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4273	}
4274	Assert(!fInterceptMovDRx);
4275	}
4276	/*
4277	* If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
4278	* must intercept #DB in order to maintain a correct DR6 guest value, and
4279	* because we need to intercept it to prevent nested #DBs from hanging the
4280	* CPU, we end up always having to intercept it. See hmR0VmxInitXcptBitmap.
4281	*/
4282	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4283	else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
4284	&& !CPUMIsGuestDebugStateActive(pVCpu))
4285	#else
4286	else if (!CPUMIsGuestDebugStateActive(pVCpu))
4287	#endif
4288	{
4289	fInterceptMovDRx = true;
4290	}
4291
4292	/* Update guest DR7. */
4293	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
4294	AssertRCReturn(rc, rc);
4295
4296	pVCpu->hm.s.fUsingHyperDR7 = false;
4297	}
4298
4299	/*
4300	* Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions.
4301	*/
4302	if (fInterceptMovDRx)
4303	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4304	else
4305	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4306	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4307	AssertRCReturn(rc, rc);
4308
4309	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_DEBUG);
4310	return VINF_SUCCESS;
4311	}
4312
4313
4314	#ifdef VBOX_STRICT
4315	/**
4316	* Strict function to validate segment registers.
4317	*
4318	* @remarks ASSUMES CR0 is up to date.
4319	*/
4320	static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4321	{
4322	/* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
4323	/* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
4324	* only updates the VMCS' copy of the value with the unusable bit and doesn't change the guest-context value. */
4325	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4326	&& ( !CPUMIsGuestInRealModeEx(pCtx)
4327	&& !CPUMIsGuestInV86ModeEx(pCtx)))
4328	{
4329	/* Protected mode checks */
4330	/* CS */
4331	Assert(pCtx->cs.Attr.n.u1Present);
4332	Assert(!(pCtx->cs.Attr.u & 0xf00));
4333	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4334	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4335	\|\| !(pCtx->cs.Attr.n.u1Granularity));
4336	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4337	\|\| (pCtx->cs.Attr.n.u1Granularity));
4338	/* CS cannot be loaded with NULL in protected mode. */
4339	Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
4340	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
4341	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4342	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
4343	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4344	else
4345	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4346	/* SS */
4347	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4348	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4349	if ( !(pCtx->cr0 & X86_CR0_PE)
4350	\|\| pCtx->cs.Attr.n.u4Type == 3)
4351	{
4352	Assert(!pCtx->ss.Attr.n.u2Dpl);
4353	}
4354	if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4355	{
4356	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4357	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
4358	Assert(pCtx->ss.Attr.n.u1Present);
4359	Assert(!(pCtx->ss.Attr.u & 0xf00));
4360	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4361	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4362	\|\| !(pCtx->ss.Attr.n.u1Granularity));
4363	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4364	\|\| (pCtx->ss.Attr.n.u1Granularity));
4365	}
4366	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
4367	if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4368	{
4369	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4370	Assert(pCtx->ds.Attr.n.u1Present);
4371	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4372	Assert(!(pCtx->ds.Attr.u & 0xf00));
4373	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4374	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4375	\|\| !(pCtx->ds.Attr.n.u1Granularity));
4376	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4377	\|\| (pCtx->ds.Attr.n.u1Granularity));
4378	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4379	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4380	}
4381	if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4382	{
4383	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4384	Assert(pCtx->es.Attr.n.u1Present);
4385	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4386	Assert(!(pCtx->es.Attr.u & 0xf00));
4387	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4388	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4389	\|\| !(pCtx->es.Attr.n.u1Granularity));
4390	Assert( !(pCtx->es.u32Limit & 0xfff00000)
4391	\|\| (pCtx->es.Attr.n.u1Granularity));
4392	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4393	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4394	}
4395	if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4396	{
4397	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4398	Assert(pCtx->fs.Attr.n.u1Present);
4399	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4400	Assert(!(pCtx->fs.Attr.u & 0xf00));
4401	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4402	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4403	\|\| !(pCtx->fs.Attr.n.u1Granularity));
4404	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4405	\|\| (pCtx->fs.Attr.n.u1Granularity));
4406	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4407	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4408	}
4409	if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4410	{
4411	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4412	Assert(pCtx->gs.Attr.n.u1Present);
4413	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4414	Assert(!(pCtx->gs.Attr.u & 0xf00));
4415	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4416	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4417	\|\| !(pCtx->gs.Attr.n.u1Granularity));
4418	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4419	\|\| (pCtx->gs.Attr.n.u1Granularity));
4420	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4421	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4422	}
4423	/* 64-bit capable CPUs. */
4424	# if HC_ARCH_BITS == 64
4425	Assert(!(pCtx->cs.u64Base >> 32));
4426	Assert(!pCtx->ss.Attr.u \|\| !(pCtx->ss.u64Base >> 32));
4427	Assert(!pCtx->ds.Attr.u \|\| !(pCtx->ds.u64Base >> 32));
4428	Assert(!pCtx->es.Attr.u \|\| !(pCtx->es.u64Base >> 32));
4429	# endif
4430	}
4431	else if ( CPUMIsGuestInV86ModeEx(pCtx)
4432	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
4433	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
4434	{
4435	/* Real and v86 mode checks. */
4436	/* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4437	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4438	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4439	{
4440	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4441	}
4442	else
4443	{
4444	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4445	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4446	}
4447
4448	/* CS */
4449	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4450	Assert(pCtx->cs.u32Limit == 0xffff);
4451	Assert(u32CSAttr == 0xf3);
4452	/* SS */
4453	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4454	Assert(pCtx->ss.u32Limit == 0xffff);
4455	Assert(u32SSAttr == 0xf3);
4456	/* DS */
4457	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4458	Assert(pCtx->ds.u32Limit == 0xffff);
4459	Assert(u32DSAttr == 0xf3);
4460	/* ES */
4461	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4462	Assert(pCtx->es.u32Limit == 0xffff);
4463	Assert(u32ESAttr == 0xf3);
4464	/* FS */
4465	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4466	Assert(pCtx->fs.u32Limit == 0xffff);
4467	Assert(u32FSAttr == 0xf3);
4468	/* GS */
4469	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4470	Assert(pCtx->gs.u32Limit == 0xffff);
4471	Assert(u32GSAttr == 0xf3);
4472	/* 64-bit capable CPUs. */
4473	# if HC_ARCH_BITS == 64
4474	Assert(!(pCtx->cs.u64Base >> 32));
4475	Assert(!u32SSAttr \|\| !(pCtx->ss.u64Base >> 32));
4476	Assert(!u32DSAttr \|\| !(pCtx->ds.u64Base >> 32));
4477	Assert(!u32ESAttr \|\| !(pCtx->es.u64Base >> 32));
4478	# endif
4479	}
4480	}
4481	#endif /* VBOX_STRICT */
4482
4483
4484	/**
4485	* Writes a guest segment register into the guest-state area in the VMCS.
4486	*
4487	* @returns VBox status code.
4488	* @param pVCpu The cross context virtual CPU structure.
4489	* @param idxSel Index of the selector in the VMCS.
4490	* @param idxLimit Index of the segment limit in the VMCS.
4491	* @param idxBase Index of the segment base in the VMCS.
4492	* @param idxAccess Index of the access rights of the segment in the VMCS.
4493	* @param pSelReg Pointer to the segment selector.
4494	*
4495	* @remarks No-long-jump zone!!!
4496	*/
4497	static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
4498	uint32_t idxAccess, PCPUMSELREG pSelReg)
4499	{
4500	int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4501	rc \|= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4502	rc \|= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4503	AssertRCReturn(rc, rc);
4504
4505	uint32_t u32Access = pSelReg->Attr.u;
4506	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4507	{
4508	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4509	u32Access = 0xf3;
4510	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4511	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4512	}
4513	else
4514	{
4515	/*
4516	* The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
4517	* real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
4518	* protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
4519	* loaded in protected-mode have their attribute as 0.
4520	*/
4521	if (!u32Access)
4522	u32Access = X86DESCATTR_UNUSABLE;
4523	}
4524
4525	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4526	AssertMsg((u32Access & X86DESCATTR_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
4527	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4528
4529	rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4530	AssertRCReturn(rc, rc);
4531	return rc;
4532	}
4533
4534
4535	/**
4536	* Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4537	* into the guest-state area in the VMCS.
4538	*
4539	* @returns VBox status code.
4540	* @param pVCpu The cross context virtual CPU structure.
4541	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4542	* out-of-sync. Make sure to update the required fields
4543	* before using them.
4544	*
4545	* @remarks ASSUMES pMixedCtx->cr0 is up to date (strict builds validation).
4546	* @remarks No-long-jump zone!!!
4547	*/
4548	static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4549	{
4550	int rc = VERR_INTERNAL_ERROR_5;
4551	PVM pVM = pVCpu->CTX_SUFF(pVM);
4552
4553	/*
4554	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
4555	*/
4556	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS))
4557	{
4558	/* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
4559	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4560	{
4561	pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
4562	pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
4563	pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
4564	pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
4565	pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
4566	pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
4567	}
4568
4569	#ifdef VBOX_WITH_REM
4570	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4571	{
4572	Assert(pVM->hm.s.vmx.pRealModeTSS);
4573	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4574	if ( pVCpu->hm.s.vmx.fWasInRealMode
4575	&& PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4576	{
4577	/* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4578	in real-mode (e.g. OpenBSD 4.0) */
4579	REMFlushTBs(pVM);
4580	Log4(("Load[%RU32]: Switch to protected mode detected!\n", pVCpu->idCpu));
4581	pVCpu->hm.s.vmx.fWasInRealMode = false;
4582	}
4583	}
4584	#endif
4585	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_CS_SEL, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
4586	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs);
4587	AssertRCReturn(rc, rc);
4588	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_SS_SEL, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
4589	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss);
4590	AssertRCReturn(rc, rc);
4591	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_DS_SEL, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
4592	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds);
4593	AssertRCReturn(rc, rc);
4594	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_ES_SEL, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
4595	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es);
4596	AssertRCReturn(rc, rc);
4597	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FS_SEL, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
4598	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs);
4599	AssertRCReturn(rc, rc);
4600	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_GS_SEL, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
4601	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs);
4602	AssertRCReturn(rc, rc);
4603
4604	#ifdef VBOX_STRICT
4605	/* Validate. */
4606	hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
4607	#endif
4608
4609	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS);
4610	Log4(("Load[%RU32]: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pVCpu->idCpu, pMixedCtx->cs.Sel,
4611	pMixedCtx->cs.u64Base, pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
4612	}
4613
4614	/*
4615	* Guest TR.
4616	*/
4617	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR))
4618	{
4619	/*
4620	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
4621	* using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
4622	* See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4623	*/
4624	uint16_t u16Sel = 0;
4625	uint32_t u32Limit = 0;
4626	uint64_t u64Base = 0;
4627	uint32_t u32AccessRights = 0;
4628
4629	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4630	{
4631	u16Sel = pMixedCtx->tr.Sel;
4632	u32Limit = pMixedCtx->tr.u32Limit;
4633	u64Base = pMixedCtx->tr.u64Base;
4634	u32AccessRights = pMixedCtx->tr.Attr.u;
4635	}
4636	else
4637	{
4638	Assert(pVM->hm.s.vmx.pRealModeTSS);
4639	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
4640
4641	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4642	RTGCPHYS GCPhys;
4643	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4644	AssertRCReturn(rc, rc);
4645
4646	X86DESCATTR DescAttr;
4647	DescAttr.u = 0;
4648	DescAttr.n.u1Present = 1;
4649	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4650
4651	u16Sel = 0;
4652	u32Limit = HM_VTX_TSS_SIZE;
4653	u64Base = GCPhys; /* in real-mode phys = virt. */
4654	u32AccessRights = DescAttr.u;
4655	}
4656
4657	/* Validate. */
4658	Assert(!(u16Sel & RT_BIT(2)));
4659	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4660	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4661	AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4662	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4663	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4664	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4665	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4666	Assert( (u32Limit & 0xfff) == 0xfff
4667	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4668	Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
4669	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4670
4671	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_TR_SEL, u16Sel);
4672	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
4673	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
4674	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
4675	AssertRCReturn(rc, rc);
4676
4677	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_TR);
4678	Log4(("Load[%RU32]: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", pVCpu->idCpu, u64Base));
4679	}
4680
4681	/*
4682	* Guest GDTR.
4683	*/
4684	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR))
4685	{
4686	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt);
4687	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt);
4688	AssertRCReturn(rc, rc);
4689
4690	/* Validate. */
4691	Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4692
4693	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_GDTR);
4694	Log4(("Load[%RU32]: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->gdtr.pGdt));
4695	}
4696
4697	/*
4698	* Guest LDTR.
4699	*/
4700	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR))
4701	{
4702	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4703	uint32_t u32Access = 0;
4704	if (!pMixedCtx->ldtr.Attr.u)
4705	u32Access = X86DESCATTR_UNUSABLE;
4706	else
4707	u32Access = pMixedCtx->ldtr.Attr.u;
4708
4709	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, pMixedCtx->ldtr.Sel);
4710	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit);
4711	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base);
4712	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
4713	AssertRCReturn(rc, rc);
4714
4715	/* Validate. */
4716	if (!(u32Access & X86DESCATTR_UNUSABLE))
4717	{
4718	Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4719	Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4720	Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4721	Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4722	Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4723	Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4724	Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
4725	\|\| !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4726	Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
4727	\|\| pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4728	}
4729
4730	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LDTR);
4731	Log4(("Load[%RU32]: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->ldtr.u64Base));
4732	}
4733
4734	/*
4735	* Guest IDTR.
4736	*/
4737	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR))
4738	{
4739	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt);
4740	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt);
4741	AssertRCReturn(rc, rc);
4742
4743	/* Validate. */
4744	Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4745
4746	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_IDTR);
4747	Log4(("Load[%RU32]: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->idtr.pIdt));
4748	}
4749
4750	return VINF_SUCCESS;
4751	}
4752
4753
4754	/**
4755	* Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4756	* areas.
4757	*
4758	* These MSRs will automatically be loaded to the host CPU on every successful
4759	* VM-entry and stored from the host CPU on every successful VM-exit. This also
4760	* creates/updates MSR slots for the host MSRs. The actual host MSR values are
4761	* -not- updated here for performance reasons. See hmR0VmxSaveHostMsrs().
4762	*
4763	* Also loads the sysenter MSRs into the guest-state area in the VMCS.
4764	*
4765	* @returns VBox status code.
4766	* @param pVCpu The cross context virtual CPU structure.
4767	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4768	* out-of-sync. Make sure to update the required fields
4769	* before using them.
4770	*
4771	* @remarks No-long-jump zone!!!
4772	*/
4773	static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4774	{
4775	AssertPtr(pVCpu);
4776	AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4777
4778	/*
4779	* MSRs that we use the auto-load/store MSR area in the VMCS.
4780	*/
4781	PVM pVM = pVCpu->CTX_SUFF(pVM);
4782	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS))
4783	{
4784	/* For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs(). */
4785	#if HC_ARCH_BITS == 32
4786	if (pVM->hm.s.fAllow64BitGuests)
4787	{
4788	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pMixedCtx->msrLSTAR, false, NULL);
4789	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pMixedCtx->msrSTAR, false, NULL);
4790	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pMixedCtx->msrSFMASK, false, NULL);
4791	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE, false, NULL);
4792	AssertRCReturn(rc, rc);
4793	# ifdef LOG_ENABLED
4794	PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4795	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4796	{
4797	Log4(("Load[%RU32]: MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", pVCpu->idCpu, i, pMsr->u32Msr,
4798	pMsr->u64Value));
4799	}
4800	# endif
4801	}
4802	#endif
4803	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4804	}
4805
4806	/*
4807	* Guest Sysenter MSRs.
4808	* These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4809	* VM-exits on WRMSRs for these MSRs.
4810	*/
4811	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR))
4812	{
4813	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
4814	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4815	}
4816
4817	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR))
4818	{
4819	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
4820	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4821	}
4822
4823	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR))
4824	{
4825	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
4826	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4827	}
4828
4829	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_EFER_MSR))
4830	{
4831	if (hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
4832	{
4833	/*
4834	* If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
4835	* but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
4836	*/
4837	if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4838	{
4839	int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pMixedCtx->msrEFER);
4840	AssertRCReturn(rc,rc);
4841	Log4(("Load[%RU32]: VMX_VMCS64_GUEST_EFER_FULL=%#RX64\n", pVCpu->idCpu, pMixedCtx->msrEFER));
4842	}
4843	else
4844	{
4845	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_EFER, pMixedCtx->msrEFER, false /* fUpdateHostMsr */,
4846	NULL /* pfAddedAndUpdated */);
4847	AssertRCReturn(rc, rc);
4848
4849	/* We need to intercept reads too, see @bugref{7386#c16}. */
4850	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
4851	hmR0VmxSetMsrPermission(pVCpu, MSR_K6_EFER, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
4852	Log4(("Load[%RU32]: MSR[--]: u32Msr=%#RX32 u64Value=%#RX64 cMsrs=%u\n", pVCpu->idCpu, MSR_K6_EFER,
4853	pMixedCtx->msrEFER, pVCpu->hm.s.vmx.cMsrs));
4854	}
4855	}
4856	else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
4857	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K6_EFER);
4858	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_EFER_MSR);
4859	}
4860
4861	return VINF_SUCCESS;
4862	}
4863
4864
4865	/**
4866	* Loads the guest activity state into the guest-state area in the VMCS.
4867	*
4868	* @returns VBox status code.
4869	* @param pVCpu The cross context virtual CPU structure.
4870	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4871	* out-of-sync. Make sure to update the required fields
4872	* before using them.
4873	*
4874	* @remarks No-long-jump zone!!!
4875	*/
4876	static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4877	{
4878	NOREF(pMixedCtx);
4879	/** @todo See if we can make use of other states, e.g.
4880	* VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
4881	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE))
4882	{
4883	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
4884	AssertRCReturn(rc, rc);
4885
4886	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE);
4887	}
4888	return VINF_SUCCESS;
4889	}
4890
4891
4892	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
4893	/**
4894	* Check if guest state allows safe use of 32-bit switcher again.
4895	*
4896	* Segment bases and protected mode structures must be 32-bit addressable
4897	* because the 32-bit switcher will ignore high dword when writing these VMCS
4898	* fields. See @bugref{8432} for details.
4899	*
4900	* @returns true if safe, false if must continue to use the 64-bit switcher.
4901	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4902	* out-of-sync. Make sure to update the required fields
4903	* before using them.
4904	*
4905	* @remarks No-long-jump zone!!!
4906	*/
4907	static bool hmR0VmxIs32BitSwitcherSafe(PCPUMCTX pMixedCtx)
4908	{
4909	if (pMixedCtx->gdtr.pGdt & UINT64_C(0xffffffff00000000))
4910	return false;
4911	if (pMixedCtx->idtr.pIdt & UINT64_C(0xffffffff00000000))
4912	return false;
4913	if (pMixedCtx->ldtr.u64Base & UINT64_C(0xffffffff00000000))
4914	return false;
4915	if (pMixedCtx->tr.u64Base & UINT64_C(0xffffffff00000000))
4916	return false;
4917	if (pMixedCtx->es.u64Base & UINT64_C(0xffffffff00000000))
4918	return false;
4919	if (pMixedCtx->cs.u64Base & UINT64_C(0xffffffff00000000))
4920	return false;
4921	if (pMixedCtx->ss.u64Base & UINT64_C(0xffffffff00000000))
4922	return false;
4923	if (pMixedCtx->ds.u64Base & UINT64_C(0xffffffff00000000))
4924	return false;
4925	if (pMixedCtx->fs.u64Base & UINT64_C(0xffffffff00000000))
4926	return false;
4927	if (pMixedCtx->gs.u64Base & UINT64_C(0xffffffff00000000))
4928	return false;
4929	/* All good, bases are 32-bit. */
4930	return true;
4931	}
4932	#endif
4933
4934
4935	/**
4936	* Sets up the appropriate function to run guest code.
4937	*
4938	* @returns VBox status code.
4939	* @param pVCpu The cross context virtual CPU structure.
4940	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4941	* out-of-sync. Make sure to update the required fields
4942	* before using them.
4943	*
4944	* @remarks No-long-jump zone!!!
4945	*/
4946	static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4947	{
4948	if (CPUMIsGuestInLongModeEx(pMixedCtx))
4949	{
4950	#ifndef VBOX_ENABLE_64_BITS_GUESTS
4951	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4952	#endif
4953	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4954	#if HC_ARCH_BITS == 32
4955	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4956	if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4957	{
4958	if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4959	{
4960	/* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4961	AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_VMX_EXIT_CTLS
4962	\| HM_CHANGED_VMX_ENTRY_CTLS
4963	\| HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4964	}
4965	pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4966
4967	/* Mark that we've switched to 64-bit handler, we can't safely switch back to 32-bit for
4968	the rest of the VM run (until VM reset). See @bugref{8432#c7}. */
4969	pVCpu->hm.s.vmx.fSwitchedTo64on32 = true;
4970	Log4(("Load[%RU32]: hmR0VmxSetupVMRunHandler: selected 64-bit switcher\n", pVCpu->idCpu));
4971	}
4972	#else
4973	/* 64-bit host. */
4974	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4975	#endif
4976	}
4977	else
4978	{
4979	/* Guest is not in long mode, use the 32-bit handler. */
4980	#if HC_ARCH_BITS == 32
4981	if ( pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32
4982	&& !pVCpu->hm.s.vmx.fSwitchedTo64on32 /* If set, guest mode change does not imply switcher change. */
4983	&& pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4984	{
4985	/* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4986	AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_VMX_EXIT_CTLS
4987	\| HM_CHANGED_VMX_ENTRY_CTLS
4988	\| HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4989	}
4990	# ifdef VBOX_ENABLE_64_BITS_GUESTS
4991	/*
4992	* Keep using the 64-bit switcher even though we're in 32-bit because of bad Intel design, see @bugref{8432#c7}.
4993	* If real-on-v86 mode is active, clear the 64-bit switcher flag because now we know the guest is in a sane
4994	* state where it's safe to use the 32-bit switcher. Otherwise check the guest state if it's safe to use
4995	* the much faster 32-bit switcher again.
4996	*/
4997	if (!pVCpu->hm.s.vmx.fSwitchedTo64on32)
4998	{
4999	if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
5000	Log4(("Load[%RU32]: hmR0VmxSetupVMRunHandler: selected 32-bit switcher\n", pVCpu->idCpu));
5001	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
5002	}
5003	else
5004	{
5005	Assert(pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64);
5006	if ( pVCpu->hm.s.vmx.RealMode.fRealOnV86Active
5007	\|\| hmR0VmxIs32BitSwitcherSafe(pMixedCtx))
5008	{
5009	pVCpu->hm.s.vmx.fSwitchedTo64on32 = false;
5010	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
5011	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_EFER_MSR
5012	\| HM_CHANGED_VMX_ENTRY_CTLS
5013	\| HM_CHANGED_VMX_EXIT_CTLS
5014	\| HM_CHANGED_HOST_CONTEXT);
5015	Log4(("Load[%RU32]: hmR0VmxSetupVMRunHandler: selected 32-bit switcher (safe)\n", pVCpu->idCpu));
5016	}
5017	}
5018	# else
5019	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
5020	# endif
5021	#else
5022	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
5023	#endif
5024	}
5025	Assert(pVCpu->hm.s.vmx.pfnStartVM);
5026	return VINF_SUCCESS;
5027	}
5028
5029
5030	/**
5031	* Wrapper for running the guest code in VT-x.
5032	*
5033	* @returns VBox status code, no informational status codes.
5034	* @param pVM The cross context VM structure.
5035	* @param pVCpu The cross context virtual CPU structure.
5036	* @param pCtx Pointer to the guest-CPU context.
5037	*
5038	* @remarks No-long-jump zone!!!
5039	*/
5040	DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
5041	{
5042	/*
5043	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
5044	* using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
5045	* Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
5046	*/
5047	bool const fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
5048	/** @todo Add stats for resume vs launch. */
5049	#ifdef VBOX_WITH_KERNEL_USING_XMM
5050	int rc = hmR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
5051	#else
5052	int rc = pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
5053	#endif
5054	AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
5055	return rc;
5056	}
5057
5058
5059	/**
5060	* Reports world-switch error and dumps some useful debug info.
5061	*
5062	* @param pVM The cross context VM structure.
5063	* @param pVCpu The cross context virtual CPU structure.
5064	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
5065	* @param pCtx Pointer to the guest-CPU context.
5066	* @param pVmxTransient Pointer to the VMX transient structure (only
5067	* exitReason updated).
5068	*/
5069	static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
5070	{
5071	Assert(pVM);
5072	Assert(pVCpu);
5073	Assert(pCtx);
5074	Assert(pVmxTransient);
5075	HMVMX_ASSERT_PREEMPT_SAFE();
5076
5077	Log4(("VM-entry failure: %Rrc\n", rcVMRun));
5078	switch (rcVMRun)
5079	{
5080	case VERR_VMX_INVALID_VMXON_PTR:
5081	AssertFailed();
5082	break;
5083	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
5084	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
5085	{
5086	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
5087	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
5088	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
5089	AssertRC(rc);
5090
5091	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
5092	/* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
5093	Cannot do it here as we may have been long preempted. */
5094
5095	#ifdef VBOX_STRICT
5096	Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
5097	pVmxTransient->uExitReason));
5098	Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
5099	Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
5100	if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
5101	Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
5102	else
5103	Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
5104	Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
5105	Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
5106
5107	/* VMX control bits. */
5108	uint32_t u32Val;
5109	uint64_t u64Val;
5110	RTHCUINTREG uHCReg;
5111	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
5112	Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
5113	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
5114	Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
5115	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
5116	{
5117	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
5118	Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
5119	}
5120	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
5121	Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
5122	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
5123	Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
5124	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
5125	Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
5126	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
5127	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
5128	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
5129	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
5130	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
5131	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
5132	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
5133	Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
5134	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
5135	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
5136	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5137	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
5138	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5139	Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
5140	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
5141	Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
5142	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
5143	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
5144	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
5145	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
5146	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
5147	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
5148	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
5149	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5150	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
5151	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
5152	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
5153	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5154	if (pVM->hm.s.fNestedPaging)
5155	{
5156	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
5157	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
5158	}
5159
5160	/* Guest bits. */
5161	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
5162	Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
5163	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
5164	Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
5165	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
5166	Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
5167	if (pVM->hm.s.vmx.fVpid)
5168	{
5169	rc = VMXReadVmcs32(VMX_VMCS16_VPID, &u32Val); AssertRC(rc);
5170	Log4(("VMX_VMCS16_VPID %u\n", u32Val));
5171	}
5172
5173	/* Host bits. */
5174	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
5175	Log4(("Host CR0 %#RHr\n", uHCReg));
5176	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
5177	Log4(("Host CR3 %#RHr\n", uHCReg));
5178	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
5179	Log4(("Host CR4 %#RHr\n", uHCReg));
5180
5181	RTGDTR HostGdtr;
5182	PCX86DESCHC pDesc;
5183	ASMGetGDTR(&HostGdtr);
5184	rc = VMXReadVmcs32(VMX_VMCS16_HOST_CS_SEL, &u32Val); AssertRC(rc);
5185	Log4(("Host CS %#08x\n", u32Val));
5186	if (u32Val < HostGdtr.cbGdt)
5187	{
5188	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5189	hmR0DumpDescriptor(pDesc, u32Val, "CS: ");
5190	}
5191
5192	rc = VMXReadVmcs32(VMX_VMCS16_HOST_DS_SEL, &u32Val); AssertRC(rc);
5193	Log4(("Host DS %#08x\n", u32Val));
5194	if (u32Val < HostGdtr.cbGdt)
5195	{
5196	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5197	hmR0DumpDescriptor(pDesc, u32Val, "DS: ");
5198	}
5199
5200	rc = VMXReadVmcs32(VMX_VMCS16_HOST_ES_SEL, &u32Val); AssertRC(rc);
5201	Log4(("Host ES %#08x\n", u32Val));
5202	if (u32Val < HostGdtr.cbGdt)
5203	{
5204	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5205	hmR0DumpDescriptor(pDesc, u32Val, "ES: ");
5206	}
5207
5208	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FS_SEL, &u32Val); AssertRC(rc);
5209	Log4(("Host FS %#08x\n", u32Val));
5210	if (u32Val < HostGdtr.cbGdt)
5211	{
5212	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5213	hmR0DumpDescriptor(pDesc, u32Val, "FS: ");
5214	}
5215
5216	rc = VMXReadVmcs32(VMX_VMCS16_HOST_GS_SEL, &u32Val); AssertRC(rc);
5217	Log4(("Host GS %#08x\n", u32Val));
5218	if (u32Val < HostGdtr.cbGdt)
5219	{
5220	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5221	hmR0DumpDescriptor(pDesc, u32Val, "GS: ");
5222	}
5223
5224	rc = VMXReadVmcs32(VMX_VMCS16_HOST_SS_SEL, &u32Val); AssertRC(rc);
5225	Log4(("Host SS %#08x\n", u32Val));
5226	if (u32Val < HostGdtr.cbGdt)
5227	{
5228	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5229	hmR0DumpDescriptor(pDesc, u32Val, "SS: ");
5230	}
5231
5232	rc = VMXReadVmcs32(VMX_VMCS16_HOST_TR_SEL, &u32Val); AssertRC(rc);
5233	Log4(("Host TR %#08x\n", u32Val));
5234	if (u32Val < HostGdtr.cbGdt)
5235	{
5236	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5237	hmR0DumpDescriptor(pDesc, u32Val, "TR: ");
5238	}
5239
5240	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
5241	Log4(("Host TR Base %#RHv\n", uHCReg));
5242	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
5243	Log4(("Host GDTR Base %#RHv\n", uHCReg));
5244	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
5245	Log4(("Host IDTR Base %#RHv\n", uHCReg));
5246	rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
5247	Log4(("Host SYSENTER CS %#08x\n", u32Val));
5248	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
5249	Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
5250	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
5251	Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
5252	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
5253	Log4(("Host RSP %#RHv\n", uHCReg));
5254	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
5255	Log4(("Host RIP %#RHv\n", uHCReg));
5256	# if HC_ARCH_BITS == 64
5257	Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
5258	Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
5259	Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
5260	Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
5261	Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
5262	Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
5263	# endif
5264	#endif /* VBOX_STRICT */
5265	break;
5266	}
5267
5268	default:
5269	/* Impossible */
5270	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
5271	break;
5272	}
5273	NOREF(pVM); NOREF(pCtx);
5274	}
5275
5276
5277	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
5278	#ifndef VMX_USE_CACHED_VMCS_ACCESSES
5279	# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
5280	#endif
5281	#ifdef VBOX_STRICT
5282	static bool hmR0VmxIsValidWriteField(uint32_t idxField)
5283	{
5284	switch (idxField)
5285	{
5286	case VMX_VMCS_GUEST_RIP:
5287	case VMX_VMCS_GUEST_RSP:
5288	case VMX_VMCS_GUEST_SYSENTER_EIP:
5289	case VMX_VMCS_GUEST_SYSENTER_ESP:
5290	case VMX_VMCS_GUEST_GDTR_BASE:
5291	case VMX_VMCS_GUEST_IDTR_BASE:
5292	case VMX_VMCS_GUEST_CS_BASE:
5293	case VMX_VMCS_GUEST_DS_BASE:
5294	case VMX_VMCS_GUEST_ES_BASE:
5295	case VMX_VMCS_GUEST_FS_BASE:
5296	case VMX_VMCS_GUEST_GS_BASE:
5297	case VMX_VMCS_GUEST_SS_BASE:
5298	case VMX_VMCS_GUEST_LDTR_BASE:
5299	case VMX_VMCS_GUEST_TR_BASE:
5300	case VMX_VMCS_GUEST_CR3:
5301	return true;
5302	}
5303	return false;
5304	}
5305
5306	static bool hmR0VmxIsValidReadField(uint32_t idxField)
5307	{
5308	switch (idxField)
5309	{
5310	/* Read-only fields. */
5311	case VMX_VMCS_RO_EXIT_QUALIFICATION:
5312	return true;
5313	}
5314	/* Remaining readable fields should also be writable. */
5315	return hmR0VmxIsValidWriteField(idxField);
5316	}
5317	#endif /* VBOX_STRICT */
5318
5319
5320	/**
5321	* Executes the specified handler in 64-bit mode.
5322	*
5323	* @returns VBox status code (no informational status codes).
5324	* @param pVM The cross context VM structure.
5325	* @param pVCpu The cross context virtual CPU structure.
5326	* @param pCtx Pointer to the guest CPU context.
5327	* @param enmOp The operation to perform.
5328	* @param cParams Number of parameters.
5329	* @param paParam Array of 32-bit parameters.
5330	*/
5331	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp,
5332	uint32_t cParams, uint32_t *paParam)
5333	{
5334	NOREF(pCtx);
5335
5336	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
5337	Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
5338	Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
5339	Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
5340
5341	#ifdef VBOX_STRICT
5342	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
5343	Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
5344
5345	for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
5346	Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
5347	#endif
5348
5349	/* Disable interrupts. */
5350	RTCCUINTREG fOldEFlags = ASMIntDisableFlags();
5351
5352	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
5353	RTCPUID idHostCpu = RTMpCpuId();
5354	CPUMR0SetLApic(pVCpu, idHostCpu);
5355	#endif
5356
5357	PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
5358	RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5359
5360	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
5361	VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5362	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
5363
5364	/* Leave VMX Root Mode. */
5365	VMXDisable();
5366
5367	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5368
5369	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
5370	CPUMSetHyperEIP(pVCpu, enmOp);
5371	for (int i = (int)cParams - 1; i >= 0; i--)
5372	CPUMPushHyper(pVCpu, paParam[i]);
5373
5374	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
5375
5376	/* Call the switcher. */
5377	int rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
5378	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
5379
5380	/** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
5381	/* Make sure the VMX instructions don't cause #UD faults. */
5382	SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
5383
5384	/* Re-enter VMX Root Mode */
5385	int rc2 = VMXEnable(HCPhysCpuPage);
5386	if (RT_FAILURE(rc2))
5387	{
5388	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5389	ASMSetFlags(fOldEFlags);
5390	pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
5391	return rc2;
5392	}
5393
5394	rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5395	AssertRC(rc2);
5396	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
5397	Assert(!(ASMGetFlags() & X86_EFL_IF));
5398	ASMSetFlags(fOldEFlags);
5399	return rc;
5400	}
5401
5402
5403	/**
5404	* Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
5405	* supporting 64-bit guests.
5406	*
5407	* @returns VBox status code.
5408	* @param fResume Whether to VMLAUNCH or VMRESUME.
5409	* @param pCtx Pointer to the guest-CPU context.
5410	* @param pCache Pointer to the VMCS cache.
5411	* @param pVM The cross context VM structure.
5412	* @param pVCpu The cross context virtual CPU structure.
5413	*/
5414	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
5415	{
5416	NOREF(fResume);
5417
5418	PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
5419	RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5420
5421	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5422	pCache->uPos = 1;
5423	pCache->interPD = PGMGetInterPaeCR3(pVM);
5424	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
5425	#endif
5426
5427	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5428	pCache->TestIn.HCPhysCpuPage = 0;
5429	pCache->TestIn.HCPhysVmcs = 0;
5430	pCache->TestIn.pCache = 0;
5431	pCache->TestOut.HCPhysVmcs = 0;
5432	pCache->TestOut.pCache = 0;
5433	pCache->TestOut.pCtx = 0;
5434	pCache->TestOut.eflags = 0;
5435	#else
5436	NOREF(pCache);
5437	#endif
5438
5439	uint32_t aParam[10];
5440	aParam[0] = RT_LO_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5441	aParam[1] = RT_HI_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Hi. */
5442	aParam[2] = RT_LO_U32(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
5443	aParam[3] = RT_HI_U32(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Hi. */
5444	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
5445	aParam[5] = 0;
5446	aParam[6] = VM_RC_ADDR(pVM, pVM);
5447	aParam[7] = 0;
5448	aParam[8] = VM_RC_ADDR(pVM, pVCpu);
5449	aParam[9] = 0;
5450
5451	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5452	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
5453	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
5454	#endif
5455	int rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, RT_ELEMENTS(aParam), &aParam[0]);
5456
5457	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5458	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
5459	Assert(pCtx->dr[4] == 10);
5460	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
5461	#endif
5462
5463	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5464	AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5465	AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5466	pVCpu->hm.s.vmx.HCPhysVmcs));
5467	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5468	pCache->TestOut.HCPhysVmcs));
5469	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5470	pCache->TestOut.pCache));
5471	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5472	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5473	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5474	pCache->TestOut.pCtx));
5475	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5476	#endif
5477	return rc;
5478	}
5479
5480
5481	/**
5482	* Initialize the VMCS-Read cache.
5483	*
5484	* The VMCS cache is used for 32-bit hosts running 64-bit guests (except 32-bit
5485	* Darwin which runs with 64-bit paging in 32-bit mode) for 64-bit fields that
5486	* cannot be accessed in 32-bit mode. Some 64-bit fields -can- be accessed
5487	* (those that have a 32-bit FULL & HIGH part).
5488	*
5489	* @returns VBox status code.
5490	* @param pVM The cross context VM structure.
5491	* @param pVCpu The cross context virtual CPU structure.
5492	*/
5493	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
5494	{
5495	#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5496	{ \
5497	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5498	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5499	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5500	++cReadFields; \
5501	}
5502
5503	AssertPtr(pVM);
5504	AssertPtr(pVCpu);
5505	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5506	uint32_t cReadFields = 0;
5507
5508	/*
5509	* Don't remove the #if 0'd fields in this code. They're listed here for consistency
5510	* and serve to indicate exceptions to the rules.
5511	*/
5512
5513	/* Guest-natural selector base fields. */
5514	#if 0
5515	/* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5516	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5517	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5518	#endif
5519	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5520	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5521	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5522	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5523	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5524	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5525	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5526	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5527	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5528	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5529	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5530	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5531	#if 0
5532	/* Unused natural width guest-state fields. */
5533	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5534	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5535	#endif
5536	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5537	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5538
5539	/* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
5540	#if 0
5541	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5542	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5543	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5544	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5545	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5546	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5547	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5548	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5549	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5550	#endif
5551
5552	/* Natural width guest-state fields. */
5553	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5554	#if 0
5555	/* Currently unused field. */
5556	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
5557	#endif
5558
5559	if (pVM->hm.s.fNestedPaging)
5560	{
5561	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5562	AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5563	VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5564	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5565	}
5566	else
5567	{
5568	AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5569	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5570	}
5571
5572	#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5573	return VINF_SUCCESS;
5574	}
5575
5576
5577	/**
5578	* Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5579	* queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5580	* darwin, running 64-bit guests).
5581	*
5582	* @returns VBox status code.
5583	* @param pVCpu The cross context virtual CPU structure.
5584	* @param idxField The VMCS field encoding.
5585	* @param u64Val 16, 32 or 64-bit value.
5586	*/
5587	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5588	{
5589	int rc;
5590	switch (idxField)
5591	{
5592	/*
5593	* These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5594	*/
5595	/* 64-bit Control fields. */
5596	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5597	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5598	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5599	case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5600	case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5601	case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5602	case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5603	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5604	case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
5605	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5606	case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5607	case VMX_VMCS64_CTRL_EPTP_FULL:
5608	case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5609	/* 64-bit Guest-state fields. */
5610	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5611	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5612	case VMX_VMCS64_GUEST_PAT_FULL:
5613	case VMX_VMCS64_GUEST_EFER_FULL:
5614	case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5615	case VMX_VMCS64_GUEST_PDPTE0_FULL:
5616	case VMX_VMCS64_GUEST_PDPTE1_FULL:
5617	case VMX_VMCS64_GUEST_PDPTE2_FULL:
5618	case VMX_VMCS64_GUEST_PDPTE3_FULL:
5619	/* 64-bit Host-state fields. */
5620	case VMX_VMCS64_HOST_PAT_FULL:
5621	case VMX_VMCS64_HOST_EFER_FULL:
5622	case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5623	{
5624	rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
5625	rc \|= VMXWriteVmcs32(idxField + 1, RT_HI_U32(u64Val));
5626	break;
5627	}
5628
5629	/*
5630	* These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5631	* values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5632	*/
5633	/* Natural-width Guest-state fields. */
5634	case VMX_VMCS_GUEST_CR3:
5635	case VMX_VMCS_GUEST_ES_BASE:
5636	case VMX_VMCS_GUEST_CS_BASE:
5637	case VMX_VMCS_GUEST_SS_BASE:
5638	case VMX_VMCS_GUEST_DS_BASE:
5639	case VMX_VMCS_GUEST_FS_BASE:
5640	case VMX_VMCS_GUEST_GS_BASE:
5641	case VMX_VMCS_GUEST_LDTR_BASE:
5642	case VMX_VMCS_GUEST_TR_BASE:
5643	case VMX_VMCS_GUEST_GDTR_BASE:
5644	case VMX_VMCS_GUEST_IDTR_BASE:
5645	case VMX_VMCS_GUEST_RSP:
5646	case VMX_VMCS_GUEST_RIP:
5647	case VMX_VMCS_GUEST_SYSENTER_ESP:
5648	case VMX_VMCS_GUEST_SYSENTER_EIP:
5649	{
5650	if (!(RT_HI_U32(u64Val)))
5651	{
5652	/* If this field is 64-bit, VT-x will zero out the top bits. */
5653	rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
5654	}
5655	else
5656	{
5657	/* Assert that only the 32->64 switcher case should ever come here. */
5658	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5659	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5660	}
5661	break;
5662	}
5663
5664	default:
5665	{
5666	AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5667	rc = VERR_INVALID_PARAMETER;
5668	break;
5669	}
5670	}
5671	AssertRCReturn(rc, rc);
5672	return rc;
5673	}
5674
5675
5676	/**
5677	* Queue up a VMWRITE by using the VMCS write cache.
5678	* This is only used on 32-bit hosts (except darwin) for 64-bit guests.
5679	*
5680	* @param pVCpu The cross context virtual CPU structure.
5681	* @param idxField The VMCS field encoding.
5682	* @param u64Val 16, 32 or 64-bit value.
5683	*/
5684	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5685	{
5686	AssertPtr(pVCpu);
5687	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5688
5689	AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5690	("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5691
5692	/* Make sure there are no duplicates. */
5693	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5694	{
5695	if (pCache->Write.aField[i] == idxField)
5696	{
5697	pCache->Write.aFieldVal[i] = u64Val;
5698	return VINF_SUCCESS;
5699	}
5700	}
5701
5702	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5703	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5704	pCache->Write.cValidEntries++;
5705	return VINF_SUCCESS;
5706	}
5707	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */
5708
5709
5710	/**
5711	* Sets up the usage of TSC-offsetting and updates the VMCS.
5712	*
5713	* If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
5714	* VMX preemption timer.
5715	*
5716	* @returns VBox status code.
5717	* @param pVM The cross context VM structure.
5718	* @param pVCpu The cross context virtual CPU structure.
5719	*
5720	* @remarks No-long-jump zone!!!
5721	*/
5722	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVM pVM, PVMCPU pVCpu)
5723	{
5724	int rc;
5725	bool fOffsettedTsc;
5726	bool fParavirtTsc;
5727	if (pVM->hm.s.vmx.fUsePreemptTimer)
5728	{
5729	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset,
5730	&fOffsettedTsc, &fParavirtTsc);
5731
5732	/* Make sure the returned values have sane upper and lower boundaries. */
5733	uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
5734	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5735	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5736	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5737
5738	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5739	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
5740	}
5741	else
5742	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset, &fParavirtTsc);
5743
5744	/** @todo later optimize this to be done elsewhere and not before every
5745	* VM-entry. */
5746	if (fParavirtTsc)
5747	{
5748	/* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
5749	information before every VM-entry, hence disable it for performance sake. */
5750	#if 0
5751	rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
5752	AssertRC(rc);
5753	#endif
5754	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
5755	}
5756
5757	if (fOffsettedTsc && RT_LIKELY(!pVCpu->hm.s.fDebugWantRdTscExit))
5758	{
5759	/* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
5760	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
5761
5762	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5763	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5764	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5765	}
5766	else
5767	{
5768	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5769	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5770	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5771	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5772	}
5773	}
5774
5775
5776	#ifdef HMVMX_USE_IEM_EVENT_REFLECTION
5777	/**
5778	* Gets the IEM exception flags for the specified vector and IDT vectoring /
5779	* VM-exit interruption info type.
5780	*
5781	* @returns The IEM exception flags.
5782	* @param uVector The event vector.
5783	* @param uVmxVectorType The VMX event type.
5784	*
5785	* @remarks This function currently only constructs flags required for
5786	* IEMEvaluateRecursiveXcpt and not the complete flags (e.g, error-code
5787	* and CR2 aspects of an exception are not included).
5788	*/
5789	static uint32_t hmR0VmxGetIemXcptFlags(uint8_t uVector, uint32_t uVmxVectorType)
5790	{
5791	uint32_t fIemXcptFlags;
5792	switch (uVmxVectorType)
5793	{
5794	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
5795	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
5796	fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
5797	break;
5798
5799	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
5800	fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
5801	break;
5802
5803	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
5804	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT \| IEM_XCPT_FLAGS_ICEBP_INSTR;
5805	break;
5806
5807	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT:
5808	{
5809	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
5810	if (uVector == X86_XCPT_BP)
5811	fIemXcptFlags \|= IEM_XCPT_FLAGS_BP_INSTR;
5812	else if (uVector == X86_XCPT_OF)
5813	fIemXcptFlags \|= IEM_XCPT_FLAGS_OF_INSTR;
5814	else
5815	{
5816	fIemXcptFlags = 0;
5817	AssertMsgFailed(("Unexpected vector for software int. uVector=%#x", uVector));
5818	}
5819	break;
5820	}
5821
5822	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
5823	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
5824	break;
5825
5826	default:
5827	fIemXcptFlags = 0;
5828	AssertMsgFailed(("Unexpected vector type! uVmxVectorType=%#x uVector=%#x", uVmxVectorType, uVector));
5829	break;
5830	}
5831	return fIemXcptFlags;
5832	}
5833
5834	#else
5835	/**
5836	* Determines if an exception is a contributory exception.
5837	*
5838	* Contributory exceptions are ones which can cause double-faults unless the
5839	* original exception was a benign exception. Page-fault is intentionally not
5840	* included here as it's a conditional contributory exception.
5841	*
5842	* @returns true if the exception is contributory, false otherwise.
5843	* @param uVector The exception vector.
5844	*/
5845	DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
5846	{
5847	switch (uVector)
5848	{
5849	case X86_XCPT_GP:
5850	case X86_XCPT_SS:
5851	case X86_XCPT_NP:
5852	case X86_XCPT_TS:
5853	case X86_XCPT_DE:
5854	return true;
5855	default:
5856	break;
5857	}
5858	return false;
5859	}
5860	#endif /* HMVMX_USE_IEM_EVENT_REFLECTION */
5861
5862
5863	/**
5864	* Sets an event as a pending event to be injected into the guest.
5865	*
5866	* @param pVCpu The cross context virtual CPU structure.
5867	* @param u32IntInfo The VM-entry interruption-information field.
5868	* @param cbInstr The VM-entry instruction length in bytes (for software
5869	* interrupts, exceptions and privileged software
5870	* exceptions).
5871	* @param u32ErrCode The VM-entry exception error code.
5872	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5873	* page-fault.
5874	*
5875	* @remarks Statistics counter assumes this is a guest event being injected or
5876	* re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5877	* always incremented.
5878	*/
5879	DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5880	RTGCUINTPTR GCPtrFaultAddress)
5881	{
5882	Assert(!pVCpu->hm.s.Event.fPending);
5883	pVCpu->hm.s.Event.fPending = true;
5884	pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5885	pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5886	pVCpu->hm.s.Event.cbInstr = cbInstr;
5887	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5888	}
5889
5890
5891	/**
5892	* Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
5893	*
5894	* @param pVCpu The cross context virtual CPU structure.
5895	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5896	* out-of-sync. Make sure to update the required fields
5897	* before using them.
5898	*/
5899	DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5900	{
5901	NOREF(pMixedCtx);
5902	uint32_t u32IntInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
5903	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5904	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5905	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
5906	}
5907
5908
5909	/**
5910	* Handle a condition that occurred while delivering an event through the guest
5911	* IDT.
5912	*
5913	* @returns Strict VBox status code (i.e. informational status codes too).
5914	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
5915	* @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
5916	* to continue execution of the guest which will delivery the \#DF.
5917	* @retval VINF_EM_RESET if we detected a triple-fault condition.
5918	* @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
5919	*
5920	* @param pVCpu The cross context virtual CPU structure.
5921	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5922	* out-of-sync. Make sure to update the required fields
5923	* before using them.
5924	* @param pVmxTransient Pointer to the VMX transient structure.
5925	*
5926	* @remarks No-long-jump zone!!!
5927	*/
5928	static VBOXSTRICTRC hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
5929	{
5930	uint32_t const uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5931
5932	int rc2 = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient); AssertRCReturn(rc2, rc2);
5933	rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient); AssertRCReturn(rc2, rc2);
5934
5935	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
5936	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5937	{
5938	uint32_t const uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5939	uint32_t const uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5940	#ifdef HMVMX_USE_IEM_EVENT_REFLECTION
5941	/*
5942	* If the event was a software interrupt (generated with INT n) or a software exception (generated
5943	* by INT3/INTO) or a privileged software exception (generated by INT1), we can handle the VM-exit
5944	* and continue guest execution which will re-execute the instruction rather than re-injecting the
5945	* exception, as that can cause premature trips to ring-3 before injection and involve TRPM which
5946	* currently has no way of storing that these exceptions were caused by these instructions
5947	* (ICEBP's #DB poses the problem).
5948	*/
5949	IEMXCPTRAISE enmRaise;
5950	IEMXCPTRAISEINFO fRaiseInfo;
5951	if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5952	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5953	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
5954	{
5955	enmRaise = IEMXCPTRAISE_REEXEC_INSTR;
5956	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
5957	}
5958	else if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5959	{
5960	uint32_t const uExitVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uExitIntInfo);
5961	uint32_t const fIdtVectorFlags = hmR0VmxGetIemXcptFlags(uIdtVector, uIdtVectorType);
5962	uint32_t const fExitVectorFlags = hmR0VmxGetIemXcptFlags(uExitVector, uExitVectorType);
5963	/** @todo Make AssertMsgReturn as just AssertMsg later. */
5964	AssertMsgReturn(uExitVectorType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT,
5965	("hmR0VmxCheckExitDueToEventDelivery: Unexpected VM-exit interruption info. %#x!\n",
5966	uExitVectorType), VERR_VMX_IPE_5);
5967	enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector, &fRaiseInfo);
5968
5969	/* Determine a vectoring #PF condition, see comment in hmR0VmxExitXcptPF(). */
5970	if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF \| IEMXCPTRAISEINFO_NMI_PF))
5971	{
5972	pVmxTransient->fVectoringPF = true;
5973	enmRaise = IEMXCPTRAISE_PREV_EVENT;
5974	}
5975	}
5976	else
5977	{
5978	/*
5979	* If an exception or hardware interrupt delivery caused an EPT violation/misconfig or APIC access
5980	* VM-exit, then the VM-exit interruption-information will not be valid and we end up here.
5981	* It is sufficient to reflect the original event to the guest after handling the VM-exit.
5982	*/
5983	Assert( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5984	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5985	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT);
5986	enmRaise = IEMXCPTRAISE_PREV_EVENT;
5987	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
5988	}
5989
5990	/*
5991	* On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig
5992	* etc.) occurred while delivering the NMI, we need to clear the block-by-NMI field in the guest
5993	* interruptibility-state before re-delivering the NMI after handling the VM-exit. Otherwise the
5994	* subsequent VM-entry would fail.
5995	*
5996	* See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
5997	*/
5998	if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
5999	&& uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
6000	&& ( enmRaise == IEMXCPTRAISE_PREV_EVENT
6001	\|\| (fRaiseInfo & IEMXCPTRAISEINFO_NMI_PF))
6002	&& (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
6003	{
6004	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6005	}
6006
6007	switch (enmRaise)
6008	{
6009	case IEMXCPTRAISE_CURRENT_XCPT:
6010	{
6011	Log4(("IDT: vcpu[%RU32] Pending secondary xcpt: uIdtVectoringInfo=%#RX64 uExitIntInfo=%#RX64\n", pVCpu->idCpu,
6012	pVmxTransient->uIdtVectoringInfo, pVmxTransient->uExitIntInfo));
6013	Assert(rcStrict == VINF_SUCCESS);
6014	break;
6015	}
6016
6017	case IEMXCPTRAISE_PREV_EVENT:
6018	{
6019	uint32_t u32ErrCode;
6020	if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
6021	{
6022	rc2 = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
6023	AssertRCReturn(rc2, rc2);
6024	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
6025	}
6026	else
6027	u32ErrCode = 0;
6028
6029	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF, see hmR0VmxExitXcptPF(). */
6030	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
6031	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
6032	0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
6033
6034	Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
6035	pVCpu->hm.s.Event.u32ErrCode));
6036	Assert(rcStrict == VINF_SUCCESS);
6037	break;
6038	}
6039
6040	case IEMXCPTRAISE_REEXEC_INSTR:
6041	Assert(rcStrict == VINF_SUCCESS);
6042	break;
6043
6044	case IEMXCPTRAISE_DOUBLE_FAULT:
6045	{
6046	/*
6047	* Determing a vectoring double #PF condition. Used later, when PGM evaluates the
6048	* second #PF as a guest #PF (and not a shadow #PF) and needs to be converted into a #DF.
6049	*/
6050	if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
6051	{
6052	pVmxTransient->fVectoringDoublePF = true;
6053	Log4(("IDT: vcpu[%RU32] Vectoring double #PF %#RX64 cr2=%#RX64\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
6054	pMixedCtx->cr2));
6055	rcStrict = VINF_SUCCESS;
6056	}
6057	else
6058	{
6059	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
6060	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
6061	Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
6062	pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
6063	rcStrict = VINF_HM_DOUBLE_FAULT;
6064	}
6065	break;
6066	}
6067
6068	case IEMXCPTRAISE_TRIPLE_FAULT:
6069	{
6070	Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
6071	uExitVector));
6072	rcStrict = VINF_EM_RESET;
6073	break;
6074	}
6075
6076	case IEMXCPTRAISE_CPU_HANG:
6077	{
6078	Log4(("IDT: vcpu[%RU32] Bad guest! Entering CPU hang. fRaiseInfo=%#x\n", pVCpu->idCpu, fRaiseInfo));
6079	rcStrict = VERR_EM_GUEST_CPU_HANG;
6080	break;
6081	}
6082
6083	default:
6084	{
6085	AssertMsgFailed(("IDT: vcpu[%RU32] Unexpected/invalid value! enmRaise=%#x\n", pVCpu->idCpu, enmRaise));
6086	rcStrict = VERR_VMX_IPE_2;
6087	break;
6088	}
6089	}
6090	#else
6091	typedef enum
6092	{
6093	VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
6094	VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
6095	VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
6096	VMXREFLECTXCPT_HANG, /* Indicate bad VM trying to deadlock the CPU. */
6097	VMXREFLECTXCPT_NONE /* Nothing to reflect. */
6098	} VMXREFLECTXCPT;
6099
6100	/* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
6101	VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
6102	if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
6103	{
6104	if (uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
6105	{
6106	enmReflect = VMXREFLECTXCPT_XCPT;
6107	#ifdef VBOX_STRICT
6108	if ( hmR0VmxIsContributoryXcpt(uIdtVector)
6109	&& uExitVector == X86_XCPT_PF)
6110	{
6111	Log4(("IDT: vcpu[%RU32] Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
6112	}
6113	#endif
6114	if ( uExitVector == X86_XCPT_PF
6115	&& uIdtVector == X86_XCPT_PF)
6116	{
6117	pVmxTransient->fVectoringDoublePF = true;
6118	Log4(("IDT: vcpu[%RU32] Vectoring Double #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
6119	}
6120	else if ( uExitVector == X86_XCPT_AC
6121	&& uIdtVector == X86_XCPT_AC)
6122	{
6123	enmReflect = VMXREFLECTXCPT_HANG;
6124	Log4(("IDT: Nested #AC - Bad guest\n"));
6125	}
6126	else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
6127	&& hmR0VmxIsContributoryXcpt(uExitVector)
6128	&& ( hmR0VmxIsContributoryXcpt(uIdtVector)
6129	\|\| uIdtVector == X86_XCPT_PF))
6130	{
6131	enmReflect = VMXREFLECTXCPT_DF;
6132	}
6133	else if (uIdtVector == X86_XCPT_DF)
6134	enmReflect = VMXREFLECTXCPT_TF;
6135	}
6136	else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
6137	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
6138	{
6139	/*
6140	* Ignore software interrupts (INT n), software exceptions (#BP, #OF) and
6141	* privileged software exception (#DB from ICEBP) as they reoccur when restarting the instruction.
6142	*/
6143	enmReflect = VMXREFLECTXCPT_XCPT;
6144
6145	if (uExitVector == X86_XCPT_PF)
6146	{
6147	pVmxTransient->fVectoringPF = true;
6148	Log4(("IDT: vcpu[%RU32] Vectoring #PF due to Ext-Int/NMI. uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
6149	}
6150	}
6151	}
6152	else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6153	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
6154	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
6155	{
6156	/*
6157	* If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
6158	* interruption-information will not be valid as it's not an exception and we end up here. In such cases,
6159	* it is sufficient to reflect the original exception to the guest after handling the VM-exit.
6160	*/
6161	enmReflect = VMXREFLECTXCPT_XCPT;
6162	}
6163
6164	/*
6165	* On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig etc.) occurred
6166	* while delivering the NMI, we need to clear the block-by-NMI field in the guest interruptibility-state before
6167	* re-delivering the NMI after handling the VM-exit. Otherwise the subsequent VM-entry would fail.
6168	*
6169	* See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
6170	*/
6171	if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
6172	&& enmReflect == VMXREFLECTXCPT_XCPT
6173	&& (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
6174	&& VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6175	{
6176	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6177	}
6178
6179	switch (enmReflect)
6180	{
6181	case VMXREFLECTXCPT_XCPT:
6182	{
6183	Assert( uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6184	&& uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6185	&& uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
6186
6187	uint32_t u32ErrCode = 0;
6188	if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
6189	{
6190	rc2 = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
6191	AssertRCReturn(rc2, rc2);
6192	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
6193	}
6194
6195	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
6196	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
6197	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
6198	0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
6199	rcStrict = VINF_SUCCESS;
6200	Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu,
6201	pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.u32ErrCode));
6202
6203	break;
6204	}
6205
6206	case VMXREFLECTXCPT_DF:
6207	{
6208	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
6209	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
6210	rcStrict = VINF_HM_DOUBLE_FAULT;
6211	Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
6212	pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
6213
6214	break;
6215	}
6216
6217	case VMXREFLECTXCPT_TF:
6218	{
6219	rcStrict = VINF_EM_RESET;
6220	Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
6221	uExitVector));
6222	break;
6223	}
6224
6225	case VMXREFLECTXCPT_HANG:
6226	{
6227	rcStrict = VERR_EM_GUEST_CPU_HANG;
6228	break;
6229	}
6230
6231	default:
6232	Assert(rcStrict == VINF_SUCCESS);
6233	break;
6234	}
6235	#endif /* HMVMX_USE_IEM_EVENT_REFLECTION */
6236	}
6237	else if ( VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo)
6238	&& VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
6239	&& uExitVector != X86_XCPT_DF
6240	&& (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
6241	{
6242	/*
6243	* Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
6244	* We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
6245	* See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
6246	*/
6247	if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6248	{
6249	Log4(("hmR0VmxCheckExitDueToEventDelivery: vcpu[%RU32] Setting VMCPU_FF_BLOCK_NMIS. Valid=%RTbool uExitReason=%u\n",
6250	pVCpu->idCpu, VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo), pVmxTransient->uExitReason));
6251	VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6252	}
6253	}
6254
6255	Assert( rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_HM_DOUBLE_FAULT
6256	\|\| rcStrict == VINF_EM_RESET \|\| rcStrict == VERR_EM_GUEST_CPU_HANG);
6257	return rcStrict;
6258	}
6259
6260
6261	/**
6262	* Saves the guest's CR0 register from the VMCS into the guest-CPU context.
6263	*
6264	* @returns VBox status code.
6265	* @param pVCpu The cross context virtual CPU structure.
6266	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6267	* out-of-sync. Make sure to update the required fields
6268	* before using them.
6269	*
6270	* @remarks No-long-jump zone!!!
6271	*/
6272	static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6273	{
6274	NOREF(pMixedCtx);
6275
6276	/*
6277	* While in the middle of saving guest-CR0, we could get preempted and re-invoked from the preemption hook,
6278	* see hmR0VmxLeave(). Safer to just make this code non-preemptible.
6279	*/
6280	VMMRZCallRing3Disable(pVCpu);
6281	HM_DISABLE_PREEMPT();
6282
6283	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0))
6284	{
6285	#ifndef DEBUG_bird /** @todo this triggers running bs3-cpu-generated-1.img with --debug-command-line
6286	* and 'dbgc-init' containing:
6287	* sxe "xcpt_de"
6288	* sxe "xcpt_bp"
6289	* sxi "xcpt_gp"
6290	* sxi "xcpt_ss"
6291	* sxi "xcpt_np"
6292	*/
6293	/** @todo r=ramshankar: Should be fixed after r119291. */
6294	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0));
6295	#endif
6296	uint32_t uVal = 0;
6297	uint32_t uShadow = 0;
6298	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
6299	rc \|= VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
6300	AssertRCReturn(rc, rc);
6301
6302	uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) \| (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
6303	CPUMSetGuestCR0(pVCpu, uVal);
6304	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0);
6305	}
6306
6307	HM_RESTORE_PREEMPT();
6308	VMMRZCallRing3Enable(pVCpu);
6309	return VINF_SUCCESS;
6310	}
6311
6312
6313	/**
6314	* Saves the guest's CR4 register from the VMCS into the guest-CPU context.
6315	*
6316	* @returns VBox status code.
6317	* @param pVCpu The cross context virtual CPU structure.
6318	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6319	* out-of-sync. Make sure to update the required fields
6320	* before using them.
6321	*
6322	* @remarks No-long-jump zone!!!
6323	*/
6324	static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6325	{
6326	NOREF(pMixedCtx);
6327
6328	int rc = VINF_SUCCESS;
6329	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4))
6330	{
6331	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4));
6332	uint32_t uVal = 0;
6333	uint32_t uShadow = 0;
6334	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
6335	rc \|= VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
6336	AssertRCReturn(rc, rc);
6337
6338	uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) \| (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
6339	CPUMSetGuestCR4(pVCpu, uVal);
6340	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4);
6341	}
6342	return rc;
6343	}
6344
6345
6346	/**
6347	* Saves the guest's RIP register from the VMCS into the guest-CPU context.
6348	*
6349	* @returns VBox status code.
6350	* @param pVCpu The cross context virtual CPU structure.
6351	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6352	* out-of-sync. Make sure to update the required fields
6353	* before using them.
6354	*
6355	* @remarks No-long-jump zone!!!
6356	*/
6357	static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6358	{
6359	int rc = VINF_SUCCESS;
6360	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP))
6361	{
6362	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP));
6363	uint64_t u64Val = 0;
6364	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
6365	AssertRCReturn(rc, rc);
6366
6367	pMixedCtx->rip = u64Val;
6368	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP);
6369	}
6370	return rc;
6371	}
6372
6373
6374	/**
6375	* Saves the guest's RSP register from the VMCS into the guest-CPU context.
6376	*
6377	* @returns VBox status code.
6378	* @param pVCpu The cross context virtual CPU structure.
6379	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6380	* out-of-sync. Make sure to update the required fields
6381	* before using them.
6382	*
6383	* @remarks No-long-jump zone!!!
6384	*/
6385	static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6386	{
6387	int rc = VINF_SUCCESS;
6388	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP))
6389	{
6390	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RSP));
6391	uint64_t u64Val = 0;
6392	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
6393	AssertRCReturn(rc, rc);
6394
6395	pMixedCtx->rsp = u64Val;
6396	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP);
6397	}
6398	return rc;
6399	}
6400
6401
6402	/**
6403	* Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
6404	*
6405	* @returns VBox status code.
6406	* @param pVCpu The cross context virtual CPU structure.
6407	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6408	* out-of-sync. Make sure to update the required fields
6409	* before using them.
6410	*
6411	* @remarks No-long-jump zone!!!
6412	*/
6413	static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6414	{
6415	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS))
6416	{
6417	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS));
6418	uint32_t uVal = 0;
6419	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
6420	AssertRCReturn(rc, rc);
6421
6422	pMixedCtx->eflags.u32 = uVal;
6423	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
6424	{
6425	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
6426	Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
6427
6428	pMixedCtx->eflags.Bits.u1VM = 0;
6429	pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
6430	}
6431
6432	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS);
6433	}
6434	return VINF_SUCCESS;
6435	}
6436
6437
6438	/**
6439	* Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
6440	* guest-CPU context.
6441	*/
6442	DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6443	{
6444	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6445	rc \|= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
6446	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6447	return rc;
6448	}
6449
6450
6451	/**
6452	* Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
6453	* from the guest-state area in the VMCS.
6454	*
6455	* @param pVCpu The cross context virtual CPU structure.
6456	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6457	* out-of-sync. Make sure to update the required fields
6458	* before using them.
6459	*
6460	* @remarks No-long-jump zone!!!
6461	*/
6462	static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6463	{
6464	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE))
6465	{
6466	uint32_t uIntrState = 0;
6467	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
6468	AssertRC(rc);
6469
6470	if (!uIntrState)
6471	{
6472	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6473	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6474
6475	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6476	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6477	}
6478	else
6479	{
6480	if (uIntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
6481	\| VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI))
6482	{
6483	rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6484	AssertRC(rc);
6485	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
6486	AssertRC(rc);
6487
6488	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
6489	Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
6490	}
6491	else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6492	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6493
6494	if (uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI)
6495	{
6496	if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6497	VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6498	}
6499	else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6500	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6501	}
6502
6503	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE);
6504	}
6505	}
6506
6507
6508	/**
6509	* Saves the guest's activity state.
6510	*
6511	* @returns VBox status code.
6512	* @param pVCpu The cross context virtual CPU structure.
6513	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6514	* out-of-sync. Make sure to update the required fields
6515	* before using them.
6516	*
6517	* @remarks No-long-jump zone!!!
6518	*/
6519	static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6520	{
6521	NOREF(pMixedCtx);
6522	/* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
6523	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_ACTIVITY_STATE);
6524	return VINF_SUCCESS;
6525	}
6526
6527
6528	/**
6529	* Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
6530	* the current VMCS into the guest-CPU context.
6531	*
6532	* @returns VBox status code.
6533	* @param pVCpu The cross context virtual CPU structure.
6534	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6535	* out-of-sync. Make sure to update the required fields
6536	* before using them.
6537	*
6538	* @remarks No-long-jump zone!!!
6539	*/
6540	static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6541	{
6542	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
6543	{
6544	Assert(!HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR));
6545	uint32_t u32Val = 0;
6546	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
6547	pMixedCtx->SysEnter.cs = u32Val;
6548	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR);
6549	}
6550
6551	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
6552	{
6553	Assert(!HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR));
6554	uint64_t u64Val = 0;
6555	int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
6556	pMixedCtx->SysEnter.eip = u64Val;
6557	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR);
6558	}
6559	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
6560	{
6561	Assert(!HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR));
6562	uint64_t u64Val = 0;
6563	int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
6564	pMixedCtx->SysEnter.esp = u64Val;
6565	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR);
6566	}
6567	return VINF_SUCCESS;
6568	}
6569
6570
6571	/**
6572	* Saves the set of guest MSRs (that we restore lazily while leaving VT-x) from
6573	* the CPU back into the guest-CPU context.
6574	*
6575	* @returns VBox status code.
6576	* @param pVCpu The cross context virtual CPU structure.
6577	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6578	* out-of-sync. Make sure to update the required fields
6579	* before using them.
6580	*
6581	* @remarks No-long-jump zone!!!
6582	*/
6583	static int hmR0VmxSaveGuestLazyMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6584	{
6585	/* Since this can be called from our preemption hook it's safer to make the guest-MSRs update non-preemptible. */
6586	VMMRZCallRing3Disable(pVCpu);
6587	HM_DISABLE_PREEMPT();
6588
6589	/* Doing the check here ensures we don't overwrite already-saved guest MSRs from a preemption hook. */
6590	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS))
6591	{
6592	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS));
6593	hmR0VmxLazySaveGuestMsrs(pVCpu, pMixedCtx);
6594	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6595	}
6596
6597	HM_RESTORE_PREEMPT();
6598	VMMRZCallRing3Enable(pVCpu);
6599
6600	return VINF_SUCCESS;
6601	}
6602
6603
6604	/**
6605	* Saves the auto load/store'd guest MSRs from the current VMCS into
6606	* the guest-CPU context.
6607	*
6608	* @returns VBox status code.
6609	* @param pVCpu The cross context virtual CPU structure.
6610	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6611	* out-of-sync. Make sure to update the required fields
6612	* before using them.
6613	*
6614	* @remarks No-long-jump zone!!!
6615	*/
6616	static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6617	{
6618	if (HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS))
6619	return VINF_SUCCESS;
6620
6621	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS));
6622	PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
6623	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
6624	Log4(("hmR0VmxSaveGuestAutoLoadStoreMsrs: cMsrs=%u\n", cMsrs));
6625	for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
6626	{
6627	switch (pMsr->u32Msr)
6628	{
6629	case MSR_K8_TSC_AUX: CPUMR0SetGuestTscAux(pVCpu, pMsr->u64Value); break;
6630	case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
6631	case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
6632	case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
6633	case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
6634	case MSR_K6_EFER: /* Nothing to do here since we intercept writes, see hmR0VmxLoadGuestMsrs(). */
6635	break;
6636
6637	default:
6638	{
6639	AssertMsgFailed(("Unexpected MSR in auto-load/store area. uMsr=%#RX32 cMsrs=%u\n", pMsr->u32Msr, cMsrs));
6640	pVCpu->hm.s.u32HMError = pMsr->u32Msr;
6641	return VERR_HM_UNEXPECTED_LD_ST_MSR;
6642	}
6643	}
6644	}
6645
6646	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS);
6647	return VINF_SUCCESS;
6648	}
6649
6650
6651	/**
6652	* Saves the guest control registers from the current VMCS into the guest-CPU
6653	* context.
6654	*
6655	* @returns VBox status code.
6656	* @param pVCpu The cross context virtual CPU structure.
6657	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6658	* out-of-sync. Make sure to update the required fields
6659	* before using them.
6660	*
6661	* @remarks No-long-jump zone!!!
6662	*/
6663	static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6664	{
6665	/* Guest CR0. Guest FPU. */
6666	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6667	AssertRCReturn(rc, rc);
6668
6669	/* Guest CR4. */
6670	rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
6671	AssertRCReturn(rc, rc);
6672
6673	/* Guest CR2 - updated always during the world-switch or in #PF. */
6674	/* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
6675	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3))
6676	{
6677	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3));
6678	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
6679	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4));
6680
6681	PVM pVM = pVCpu->CTX_SUFF(pVM);
6682	if ( pVM->hm.s.vmx.fUnrestrictedGuest
6683	\|\| ( pVM->hm.s.fNestedPaging
6684	&& CPUMIsGuestPagingEnabledEx(pMixedCtx)))
6685	{
6686	uint64_t u64Val = 0;
6687	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
6688	if (pMixedCtx->cr3 != u64Val)
6689	{
6690	CPUMSetGuestCR3(pVCpu, u64Val);
6691	if (VMMRZCallRing3IsEnabled(pVCpu))
6692	{
6693	PGMUpdateCR3(pVCpu, u64Val);
6694	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6695	}
6696	else
6697	{
6698	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
6699	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
6700	}
6701	}
6702
6703	/* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
6704	if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
6705	{
6706	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
6707	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
6708	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
6709	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
6710	AssertRCReturn(rc, rc);
6711
6712	if (VMMRZCallRing3IsEnabled(pVCpu))
6713	{
6714	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6715	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6716	}
6717	else
6718	{
6719	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
6720	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
6721	}
6722	}
6723	}
6724
6725	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3);
6726	}
6727
6728	/*
6729	* Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
6730	* -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
6731	* -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
6732	*
6733	* The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
6734	* if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
6735	* calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
6736	* -NOT- check if HMVMX_UPDATED_GUEST_CR3 is already set or not!
6737	*
6738	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6739	*/
6740	if (VMMRZCallRing3IsEnabled(pVCpu))
6741	{
6742	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6743	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6744
6745	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6746	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6747
6748	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6749	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6750	}
6751
6752	return rc;
6753	}
6754
6755
6756	/**
6757	* Reads a guest segment register from the current VMCS into the guest-CPU
6758	* context.
6759	*
6760	* @returns VBox status code.
6761	* @param pVCpu The cross context virtual CPU structure.
6762	* @param idxSel Index of the selector in the VMCS.
6763	* @param idxLimit Index of the segment limit in the VMCS.
6764	* @param idxBase Index of the segment base in the VMCS.
6765	* @param idxAccess Index of the access rights of the segment in the VMCS.
6766	* @param pSelReg Pointer to the segment selector.
6767	*
6768	* @remarks No-long-jump zone!!!
6769	* @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
6770	* macro as that takes care of whether to read from the VMCS cache or
6771	* not.
6772	*/
6773	DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
6774	PCPUMSELREG pSelReg)
6775	{
6776	NOREF(pVCpu);
6777
6778	uint32_t u32Val = 0;
6779	int rc = VMXReadVmcs32(idxSel, &u32Val);
6780	AssertRCReturn(rc, rc);
6781	pSelReg->Sel = (uint16_t)u32Val;
6782	pSelReg->ValidSel = (uint16_t)u32Val;
6783	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
6784
6785	rc = VMXReadVmcs32(idxLimit, &u32Val);
6786	AssertRCReturn(rc, rc);
6787	pSelReg->u32Limit = u32Val;
6788
6789	uint64_t u64Val = 0;
6790	rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
6791	AssertRCReturn(rc, rc);
6792	pSelReg->u64Base = u64Val;
6793
6794	rc = VMXReadVmcs32(idxAccess, &u32Val);
6795	AssertRCReturn(rc, rc);
6796	pSelReg->Attr.u = u32Val;
6797
6798	/*
6799	* If VT-x marks the segment as unusable, most other bits remain undefined:
6800	* - For CS the L, D and G bits have meaning.
6801	* - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
6802	* - For the remaining data segments no bits are defined.
6803	*
6804	* The present bit and the unusable bit has been observed to be set at the
6805	* same time (the selector was supposed to be invalid as we started executing
6806	* a V8086 interrupt in ring-0).
6807	*
6808	* What should be important for the rest of the VBox code, is that the P bit is
6809	* cleared. Some of the other VBox code recognizes the unusable bit, but
6810	* AMD-V certainly don't, and REM doesn't really either. So, to be on the
6811	* safe side here, we'll strip off P and other bits we don't care about. If
6812	* any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
6813	*
6814	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
6815	*/
6816	if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6817	{
6818	Assert(idxSel != VMX_VMCS16_GUEST_TR_SEL); /* TR is the only selector that can never be unusable. */
6819
6820	/* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6821	pSelReg->Attr.u &= X86DESCATTR_UNUSABLE \| X86DESCATTR_L \| X86DESCATTR_D \| X86DESCATTR_G
6822	\| X86DESCATTR_DPL \| X86DESCATTR_TYPE \| X86DESCATTR_DT;
6823
6824	Log4(("hmR0VmxReadSegmentReg: Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Val, pSelReg->Attr.u));
6825	#ifdef DEBUG_bird
6826	AssertMsg((u32Val & ~X86DESCATTR_P) == pSelReg->Attr.u,
6827	("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6828	idxSel, u32Val, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6829	#endif
6830	}
6831	return VINF_SUCCESS;
6832	}
6833
6834
6835	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
6836	# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6837	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6838	VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6839	#else
6840	# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6841	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6842	VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6843	#endif
6844
6845
6846	/**
6847	* Saves the guest segment registers from the current VMCS into the guest-CPU
6848	* context.
6849	*
6850	* @returns VBox status code.
6851	* @param pVCpu The cross context virtual CPU structure.
6852	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6853	* out-of-sync. Make sure to update the required fields
6854	* before using them.
6855	*
6856	* @remarks No-long-jump zone!!!
6857	*/
6858	static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6859	{
6860	/* Guest segment registers. */
6861	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS))
6862	{
6863	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS));
6864	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6865	AssertRCReturn(rc, rc);
6866
6867	rc = VMXLOCAL_READ_SEG(CS, cs);
6868	rc \|= VMXLOCAL_READ_SEG(SS, ss);
6869	rc \|= VMXLOCAL_READ_SEG(DS, ds);
6870	rc \|= VMXLOCAL_READ_SEG(ES, es);
6871	rc \|= VMXLOCAL_READ_SEG(FS, fs);
6872	rc \|= VMXLOCAL_READ_SEG(GS, gs);
6873	AssertRCReturn(rc, rc);
6874
6875	/* Restore segment attributes for real-on-v86 mode hack. */
6876	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6877	{
6878	pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6879	pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6880	pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6881	pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6882	pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6883	pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6884	}
6885	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS);
6886	}
6887
6888	return VINF_SUCCESS;
6889	}
6890
6891
6892	/**
6893	* Saves the guest descriptor table registers and task register from the current
6894	* VMCS into the guest-CPU context.
6895	*
6896	* @returns VBox status code.
6897	* @param pVCpu The cross context virtual CPU structure.
6898	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6899	* out-of-sync. Make sure to update the required fields
6900	* before using them.
6901	*
6902	* @remarks No-long-jump zone!!!
6903	*/
6904	static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6905	{
6906	int rc = VINF_SUCCESS;
6907
6908	/* Guest LDTR. */
6909	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR))
6910	{
6911	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR));
6912	rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
6913	AssertRCReturn(rc, rc);
6914	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR);
6915	}
6916
6917	/* Guest GDTR. */
6918	uint64_t u64Val = 0;
6919	uint32_t u32Val = 0;
6920	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR))
6921	{
6922	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR));
6923	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
6924	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6925	pMixedCtx->gdtr.pGdt = u64Val;
6926	pMixedCtx->gdtr.cbGdt = u32Val;
6927	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR);
6928	}
6929
6930	/* Guest IDTR. */
6931	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR))
6932	{
6933	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR));
6934	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
6935	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6936	pMixedCtx->idtr.pIdt = u64Val;
6937	pMixedCtx->idtr.cbIdt = u32Val;
6938	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR);
6939	}
6940
6941	/* Guest TR. */
6942	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR))
6943	{
6944	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR));
6945	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6946	AssertRCReturn(rc, rc);
6947
6948	/* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
6949	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6950	{
6951	rc = VMXLOCAL_READ_SEG(TR, tr);
6952	AssertRCReturn(rc, rc);
6953	}
6954	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR);
6955	}
6956	return rc;
6957	}
6958
6959	#undef VMXLOCAL_READ_SEG
6960
6961
6962	/**
6963	* Saves the guest debug-register DR7 from the current VMCS into the guest-CPU
6964	* context.
6965	*
6966	* @returns VBox status code.
6967	* @param pVCpu The cross context virtual CPU structure.
6968	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6969	* out-of-sync. Make sure to update the required fields
6970	* before using them.
6971	*
6972	* @remarks No-long-jump zone!!!
6973	*/
6974	static int hmR0VmxSaveGuestDR7(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6975	{
6976	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DR7))
6977	{
6978	if (!pVCpu->hm.s.fUsingHyperDR7)
6979	{
6980	/* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6981	uint32_t u32Val;
6982	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
6983	pMixedCtx->dr[7] = u32Val;
6984	}
6985
6986	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DR7);
6987	}
6988	return VINF_SUCCESS;
6989	}
6990
6991
6992	/**
6993	* Saves the guest APIC state from the current VMCS into the guest-CPU context.
6994	*
6995	* @returns VBox status code.
6996	* @param pVCpu The cross context virtual CPU structure.
6997	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6998	* out-of-sync. Make sure to update the required fields
6999	* before using them.
7000	*
7001	* @remarks No-long-jump zone!!!
7002	*/
7003	static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7004	{
7005	NOREF(pMixedCtx);
7006
7007	/* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
7008	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_APIC_STATE);
7009	return VINF_SUCCESS;
7010	}
7011
7012
7013	/**
7014	* Saves the entire guest state from the currently active VMCS into the
7015	* guest-CPU context.
7016	*
7017	* This essentially VMREADs all guest-data.
7018	*
7019	* @returns VBox status code.
7020	* @param pVCpu The cross context virtual CPU structure.
7021	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7022	* out-of-sync. Make sure to update the required fields
7023	* before using them.
7024	*/
7025	static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7026	{
7027	Assert(pVCpu);
7028	Assert(pMixedCtx);
7029
7030	if (HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL)
7031	return VINF_SUCCESS;
7032
7033	/* Though we can longjmp to ring-3 due to log-flushes here and get recalled
7034	again on the ring-3 callback path, there is no real need to. */
7035	if (VMMRZCallRing3IsEnabled(pVCpu))
7036	VMMR0LogFlushDisable(pVCpu);
7037	else
7038	Assert(VMMR0IsLogFlushDisabled(pVCpu));
7039	Log4Func(("vcpu[%RU32]\n", pVCpu->idCpu));
7040
7041	int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
7042	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7043
7044	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
7045	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7046
7047	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7048	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7049
7050	rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
7051	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7052
7053	rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
7054	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDR7 failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7055
7056	rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
7057	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7058
7059	rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
7060	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestLazyMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7061
7062	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
7063	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7064
7065	rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
7066	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7067
7068	rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
7069	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestApicState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7070
7071	AssertMsg(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL,
7072	("Missed guest state bits while saving state; missing %RX32 (got %RX32, want %RX32) - check log for any previous errors!\n",
7073	HMVMX_UPDATED_GUEST_ALL ^ HMVMXCPU_GST_VALUE(pVCpu), HMVMXCPU_GST_VALUE(pVCpu), HMVMX_UPDATED_GUEST_ALL));
7074
7075	if (VMMRZCallRing3IsEnabled(pVCpu))
7076	VMMR0LogFlushEnable(pVCpu);
7077
7078	return VINF_SUCCESS;
7079	}
7080
7081
7082	/**
7083	* Saves basic guest registers needed for IEM instruction execution.
7084	*
7085	* @returns VBox status code (OR-able).
7086	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
7087	* @param pMixedCtx Pointer to the CPU context of the guest.
7088	* @param fMemory Whether the instruction being executed operates on
7089	* memory or not. Only CR0 is synced up if clear.
7090	* @param fNeedRsp Need RSP (any instruction working on GPRs or stack).
7091	*/
7092	static int hmR0VmxSaveGuestRegsForIemExec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fMemory, bool fNeedRsp)
7093	{
7094	/*
7095	* We assume all general purpose registers other than RSP are available.
7096	*
7097	* - RIP is a must, as it will be incremented or otherwise changed.
7098	* - RFLAGS are always required to figure the CPL.
7099	* - RSP isn't always required, however it's a GPR, so frequently required.
7100	* - SS and CS are the only segment register needed if IEM doesn't do memory
7101	* access (CPL + 16/32/64-bit mode), but we can only get all segment registers.
7102	* - CR0 is always required by IEM for the CPL, while CR3 and CR4 will only
7103	* be required for memory accesses.
7104	*
7105	* Note! Before IEM dispatches an exception, it will call us to sync in everything.
7106	*/
7107	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
7108	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7109	if (fNeedRsp)
7110	rc \|= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
7111	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7112	if (!fMemory)
7113	rc \|= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7114	else
7115	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
7116	AssertRCReturn(rc, rc);
7117	return rc;
7118	}
7119
7120
7121	/**
7122	* Ensures that we've got a complete basic guest-context.
7123	*
7124	* This excludes the FPU, SSE, AVX, and similar extended state. The interface
7125	* is for the interpreter.
7126	*
7127	* @returns VBox status code.
7128	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
7129	* @param pMixedCtx Pointer to the guest-CPU context which may have data
7130	* needing to be synced in.
7131	* @thread EMT(pVCpu)
7132	*/
7133	VMMR0_INT_DECL(int) HMR0EnsureCompleteBasicContext(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7134	{
7135	/* Note! Since this is only applicable to VT-x, the implementation is placed
7136	in the VT-x part of the sources instead of the generic stuff. */
7137	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported)
7138	{
7139	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7140	/*
7141	* For now, imply that the caller might change everything too. Do this after
7142	* saving the guest state so as to not trigger assertions.
7143	*/
7144	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7145	return rc;
7146	}
7147	return VINF_SUCCESS;
7148	}
7149
7150
7151	/**
7152	* Check per-VM and per-VCPU force flag actions that require us to go back to
7153	* ring-3 for one reason or another.
7154	*
7155	* @returns Strict VBox status code (i.e. informational status codes too)
7156	* @retval VINF_SUCCESS if we don't have any actions that require going back to
7157	* ring-3.
7158	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
7159	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
7160	* interrupts)
7161	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
7162	* all EMTs to be in ring-3.
7163	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
7164	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
7165	* to the EM loop.
7166	*
7167	* @param pVM The cross context VM structure.
7168	* @param pVCpu The cross context virtual CPU structure.
7169	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7170	* out-of-sync. Make sure to update the required fields
7171	* before using them.
7172	* @param fStepping Running in hmR0VmxRunGuestCodeStep().
7173	*/
7174	static VBOXSTRICTRC hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping)
7175	{
7176	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7177
7178	/*
7179	* Anything pending? Should be more likely than not if we're doing a good job.
7180	*/
7181	if ( !fStepping
7182	? !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_MASK)
7183	&& !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
7184	: !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
7185	&& !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
7186	return VINF_SUCCESS;
7187
7188	/* We need the control registers now, make sure the guest-CPU context is updated. */
7189	int rc3 = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
7190	AssertRCReturn(rc3, rc3);
7191
7192	/* Pending HM CR3 sync. */
7193	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
7194	{
7195	int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
7196	AssertMsgReturn(rc2 == VINF_SUCCESS \|\| rc2 == VINF_PGM_SYNC_CR3,
7197	("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
7198	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
7199	}
7200
7201	/* Pending HM PAE PDPEs. */
7202	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
7203	{
7204	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
7205	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
7206	}
7207
7208	/* Pending PGM C3 sync. */
7209	if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
7210	{
7211	VBOXSTRICTRC rcStrict2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
7212	VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
7213	if (rcStrict2 != VINF_SUCCESS)
7214	{
7215	AssertRC(VBOXSTRICTRC_VAL(rcStrict2));
7216	Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
7217	return rcStrict2;
7218	}
7219	}
7220
7221	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
7222	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
7223	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
7224	{
7225	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
7226	int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
7227	Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
7228	return rc2;
7229	}
7230
7231	/* Pending VM request packets, such as hardware interrupts. */
7232	if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
7233	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
7234	{
7235	Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
7236	return VINF_EM_PENDING_REQUEST;
7237	}
7238
7239	/* Pending PGM pool flushes. */
7240	if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
7241	{
7242	Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
7243	return VINF_PGM_POOL_FLUSH_PENDING;
7244	}
7245
7246	/* Pending DMA requests. */
7247	if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
7248	{
7249	Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
7250	return VINF_EM_RAW_TO_R3;
7251	}
7252
7253	return VINF_SUCCESS;
7254	}
7255
7256
7257	/**
7258	* Converts any TRPM trap into a pending HM event. This is typically used when
7259	* entering from ring-3 (not longjmp returns).
7260	*
7261	* @param pVCpu The cross context virtual CPU structure.
7262	*/
7263	static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
7264	{
7265	Assert(TRPMHasTrap(pVCpu));
7266	Assert(!pVCpu->hm.s.Event.fPending);
7267
7268	uint8_t uVector;
7269	TRPMEVENT enmTrpmEvent;
7270	RTGCUINT uErrCode;
7271	RTGCUINTPTR GCPtrFaultAddress;
7272	uint8_t cbInstr;
7273
7274	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
7275	AssertRC(rc);
7276
7277	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
7278	uint32_t u32IntInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7279	if (enmTrpmEvent == TRPM_TRAP)
7280	{
7281	switch (uVector)
7282	{
7283	case X86_XCPT_NMI:
7284	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7285	break;
7286
7287	case X86_XCPT_BP:
7288	case X86_XCPT_OF:
7289	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7290	break;
7291
7292	case X86_XCPT_PF:
7293	case X86_XCPT_DF:
7294	case X86_XCPT_TS:
7295	case X86_XCPT_NP:
7296	case X86_XCPT_SS:
7297	case X86_XCPT_GP:
7298	case X86_XCPT_AC:
7299	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7300	RT_FALL_THRU();
7301	default:
7302	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7303	break;
7304	}
7305	}
7306	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
7307	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7308	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
7309	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7310	else
7311	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
7312
7313	rc = TRPMResetTrap(pVCpu);
7314	AssertRC(rc);
7315	Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
7316	u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
7317
7318	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
7319	}
7320
7321
7322	/**
7323	* Converts the pending HM event into a TRPM trap.
7324	*
7325	* @param pVCpu The cross context virtual CPU structure.
7326	*/
7327	static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
7328	{
7329	Assert(pVCpu->hm.s.Event.fPending);
7330
7331	uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7332	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
7333	bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntInfo);
7334	uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
7335
7336	/* If a trap was already pending, we did something wrong! */
7337	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
7338
7339	TRPMEVENT enmTrapType;
7340	switch (uVectorType)
7341	{
7342	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
7343	enmTrapType = TRPM_HARDWARE_INT;
7344	break;
7345
7346	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
7347	enmTrapType = TRPM_SOFTWARE_INT;
7348	break;
7349
7350	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
7351	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
7352	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
7353	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
7354	enmTrapType = TRPM_TRAP;
7355	break;
7356
7357	default:
7358	AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
7359	enmTrapType = TRPM_32BIT_HACK;
7360	break;
7361	}
7362
7363	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
7364
7365	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
7366	AssertRC(rc);
7367
7368	if (fErrorCodeValid)
7369	TRPMSetErrorCode(pVCpu, uErrorCode);
7370
7371	if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
7372	&& uVector == X86_XCPT_PF)
7373	{
7374	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
7375	}
7376	else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
7377	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
7378	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
7379	{
7380	AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
7381	\|\| (uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF),
7382	("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
7383	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
7384	}
7385
7386	/* Clear any pending events from the VMCS. */
7387	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0); AssertRC(rc);
7388	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0); AssertRC(rc);
7389
7390	/* We're now done converting the pending event. */
7391	pVCpu->hm.s.Event.fPending = false;
7392	}
7393
7394
7395	/**
7396	* Does the necessary state syncing before returning to ring-3 for any reason
7397	* (longjmp, preemption, voluntary exits to ring-3) from VT-x.
7398	*
7399	* @returns VBox status code.
7400	* @param pVCpu The cross context virtual CPU structure.
7401	* @param pMixedCtx Pointer to the guest-CPU context. The data may
7402	* be out-of-sync. Make sure to update the required
7403	* fields before using them.
7404	* @param fSaveGuestState Whether to save the guest state or not.
7405	*
7406	* @remarks No-long-jmp zone!!!
7407	*/
7408	static int hmR0VmxLeave(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fSaveGuestState)
7409	{
7410	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7411	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7412
7413	RTCPUID idCpu = RTMpCpuId();
7414	Log4Func(("HostCpuId=%u\n", idCpu));
7415
7416	/*
7417	* !!! IMPORTANT !!!
7418	* If you modify code here, check whether hmR0VmxCallRing3Callback() needs to be updated too.
7419	*/
7420
7421	/* Save the guest state if necessary. */
7422	if ( fSaveGuestState
7423	&& HMVMXCPU_GST_VALUE(pVCpu) != HMVMX_UPDATED_GUEST_ALL)
7424	{
7425	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7426	AssertRCReturn(rc, rc);
7427	Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
7428	}
7429
7430	/* Restore host FPU state if necessary and resync on next R0 reentry .*/
7431	if (CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu))
7432	{
7433	/* We shouldn't reload CR0 without saving it first. */
7434	if (!fSaveGuestState)
7435	{
7436	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7437	AssertRCReturn(rc, rc);
7438	}
7439	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
7440	}
7441
7442	/* Restore host debug registers if necessary and resync on next R0 reentry. */
7443	#ifdef VBOX_STRICT
7444	if (CPUMIsHyperDebugStateActive(pVCpu))
7445	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
7446	#endif
7447	if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */))
7448	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
7449	Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
7450	Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
7451
7452	#if HC_ARCH_BITS == 64
7453	/* Restore host-state bits that VT-x only restores partially. */
7454	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7455	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7456	{
7457	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
7458	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7459	}
7460	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7461	#endif
7462
7463	/* Restore the lazy host MSRs as we're leaving VT-x context. */
7464	if (pVCpu->hm.s.vmx.fLazyMsrs)
7465	{
7466	/* We shouldn't reload the guest MSRs without saving it first. */
7467	if (!fSaveGuestState)
7468	{
7469	int rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
7470	AssertRCReturn(rc, rc);
7471	}
7472	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS));
7473	hmR0VmxLazyRestoreHostMsrs(pVCpu);
7474	Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
7475	}
7476
7477	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7478	pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7479
7480	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
7481	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
7482	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
7483	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
7484	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
7485	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
7486	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
7487	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7488
7489	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7490
7491	/** @todo This partially defeats the purpose of having preemption hooks.
7492	* The problem is, deregistering the hooks should be moved to a place that
7493	* lasts until the EMT is about to be destroyed not everytime while leaving HM
7494	* context.
7495	*/
7496	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7497	{
7498	int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7499	AssertRCReturn(rc, rc);
7500
7501	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7502	Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
7503	}
7504	Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
7505	NOREF(idCpu);
7506
7507	return VINF_SUCCESS;
7508	}
7509
7510
7511	/**
7512	* Leaves the VT-x session.
7513	*
7514	* @returns VBox status code.
7515	* @param pVCpu The cross context virtual CPU structure.
7516	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7517	* out-of-sync. Make sure to update the required fields
7518	* before using them.
7519	*
7520	* @remarks No-long-jmp zone!!!
7521	*/
7522	DECLINLINE(int) hmR0VmxLeaveSession(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7523	{
7524	HM_DISABLE_PREEMPT();
7525	HMVMX_ASSERT_CPU_SAFE();
7526	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7527	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7528
7529	/* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
7530	and done this from the VMXR0ThreadCtxCallback(). */
7531	if (!pVCpu->hm.s.fLeaveDone)
7532	{
7533	int rc2 = hmR0VmxLeave(pVCpu, pMixedCtx, true /* fSaveGuestState */);
7534	AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
7535	pVCpu->hm.s.fLeaveDone = true;
7536	}
7537	Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
7538
7539	/*
7540	* !!! IMPORTANT !!!
7541	* If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
7542	*/
7543
7544	/* Deregister hook now that we've left HM context before re-enabling preemption. */
7545	/** @todo Deregistering here means we need to VMCLEAR always
7546	* (longjmp/exit-to-r3) in VT-x which is not efficient. */
7547	/** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7548	VMMR0ThreadCtxHookDisable(pVCpu);
7549
7550	/* Leave HM context. This takes care of local init (term). */
7551	int rc = HMR0LeaveCpu(pVCpu);
7552
7553	HM_RESTORE_PREEMPT();
7554	return rc;
7555	}
7556
7557
7558	/**
7559	* Does the necessary state syncing before doing a longjmp to ring-3.
7560	*
7561	* @returns VBox status code.
7562	* @param pVCpu The cross context virtual CPU structure.
7563	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7564	* out-of-sync. Make sure to update the required fields
7565	* before using them.
7566	*
7567	* @remarks No-long-jmp zone!!!
7568	*/
7569	DECLINLINE(int) hmR0VmxLongJmpToRing3(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7570	{
7571	return hmR0VmxLeaveSession(pVCpu, pMixedCtx);
7572	}
7573
7574
7575	/**
7576	* Take necessary actions before going back to ring-3.
7577	*
7578	* An action requires us to go back to ring-3. This function does the necessary
7579	* steps before we can safely return to ring-3. This is not the same as longjmps
7580	* to ring-3, this is voluntary and prepares the guest so it may continue
7581	* executing outside HM (recompiler/IEM).
7582	*
7583	* @returns VBox status code.
7584	* @param pVM The cross context VM structure.
7585	* @param pVCpu The cross context virtual CPU structure.
7586	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7587	* out-of-sync. Make sure to update the required fields
7588	* before using them.
7589	* @param rcExit The reason for exiting to ring-3. Can be
7590	* VINF_VMM_UNKNOWN_RING3_CALL.
7591	*/
7592	static int hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, VBOXSTRICTRC rcExit)
7593	{
7594	Assert(pVM);
7595	Assert(pVCpu);
7596	Assert(pMixedCtx);
7597	HMVMX_ASSERT_PREEMPT_SAFE();
7598
7599	if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
7600	{
7601	VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
7602	pVCpu->hm.s.vmx.LastError.u32VMCSRevision = (uint32_t )pVCpu->hm.s.vmx.pvVmcs;
7603	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
7604	/* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
7605	}
7606
7607	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
7608	VMMRZCallRing3Disable(pVCpu);
7609	Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, VBOXSTRICTRC_VAL(rcExit)));
7610
7611	/* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
7612	if (pVCpu->hm.s.Event.fPending)
7613	{
7614	hmR0VmxPendingEventToTrpmTrap(pVCpu);
7615	Assert(!pVCpu->hm.s.Event.fPending);
7616	}
7617
7618	/* Clear interrupt-window and NMI-window controls as we re-evaluate it when we return from ring-3. */
7619	hmR0VmxClearIntNmiWindowsVmcs(pVCpu);
7620
7621	/* If we're emulating an instruction, we shouldn't have any TRPM traps pending
7622	and if we're injecting an event we should have a TRPM trap pending. */
7623	AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT \|\| TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
7624	#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a tripple fault in progress. */
7625	AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR \|\| !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
7626	#endif
7627
7628	/* Save guest state and restore host state bits. */
7629	int rc = hmR0VmxLeaveSession(pVCpu, pMixedCtx);
7630	AssertRCReturn(rc, rc);
7631	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7632	/* Thread-context hooks are unregistered at this point!!! */
7633
7634	/* Sync recompiler state. */
7635	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
7636	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
7637	\| CPUM_CHANGED_LDTR
7638	\| CPUM_CHANGED_GDTR
7639	\| CPUM_CHANGED_IDTR
7640	\| CPUM_CHANGED_TR
7641	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
7642	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
7643	if ( pVM->hm.s.fNestedPaging
7644	&& CPUMIsGuestPagingEnabledEx(pMixedCtx))
7645	{
7646	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
7647	}
7648
7649	Assert(!pVCpu->hm.s.fClearTrapFlag);
7650
7651	/* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
7652	if (rcExit != VINF_EM_RAW_INTERRUPT)
7653	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7654
7655	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
7656
7657	/* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
7658	VMMRZCallRing3RemoveNotification(pVCpu);
7659	VMMRZCallRing3Enable(pVCpu);
7660
7661	return rc;
7662	}
7663
7664
7665	/**
7666	* VMMRZCallRing3() callback wrapper which saves the guest state before we
7667	* longjump to ring-3 and possibly get preempted.
7668	*
7669	* @returns VBox status code.
7670	* @param pVCpu The cross context virtual CPU structure.
7671	* @param enmOperation The operation causing the ring-3 longjump.
7672	* @param pvUser Opaque pointer to the guest-CPU context. The data
7673	* may be out-of-sync. Make sure to update the required
7674	* fields before using them.
7675	*/
7676	static DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
7677	{
7678	if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
7679	{
7680	/*
7681	* !!! IMPORTANT !!!
7682	* If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
7683	* This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
7684	*/
7685	VMMRZCallRing3RemoveNotification(pVCpu);
7686	VMMRZCallRing3Disable(pVCpu);
7687	RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
7688	RTThreadPreemptDisable(&PreemptState);
7689
7690	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
7691	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
7692
7693	#if HC_ARCH_BITS == 64
7694	/* Restore host-state bits that VT-x only restores partially. */
7695	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7696	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7697	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7698	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7699	#endif
7700	/* Restore the lazy host MSRs as we're leaving VT-x context. */
7701	if (pVCpu->hm.s.vmx.fLazyMsrs)
7702	hmR0VmxLazyRestoreHostMsrs(pVCpu);
7703
7704	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7705	pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7706	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7707	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7708	{
7709	VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7710	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7711	}
7712
7713	/** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7714	VMMR0ThreadCtxHookDisable(pVCpu);
7715	HMR0LeaveCpu(pVCpu);
7716	RTThreadPreemptRestore(&PreemptState);
7717	return VINF_SUCCESS;
7718	}
7719
7720	Assert(pVCpu);
7721	Assert(pvUser);
7722	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7723	HMVMX_ASSERT_PREEMPT_SAFE();
7724
7725	VMMRZCallRing3Disable(pVCpu);
7726	Assert(VMMR0IsLogFlushDisabled(pVCpu));
7727
7728	Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32 enmOperation=%d\n", pVCpu, pVCpu->idCpu,
7729	enmOperation));
7730
7731	int rc = hmR0VmxLongJmpToRing3(pVCpu, (PCPUMCTX)pvUser);
7732	AssertRCReturn(rc, rc);
7733
7734	VMMRZCallRing3Enable(pVCpu);
7735	return VINF_SUCCESS;
7736	}
7737
7738
7739	/**
7740	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7741	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
7742	*
7743	* @param pVCpu The cross context virtual CPU structure.
7744	*/
7745	DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
7746	{
7747	if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7748	{
7749	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7750	{
7751	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7752	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7753	AssertRC(rc);
7754	Log4(("Setup interrupt-window exiting\n"));
7755	}
7756	} /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
7757	}
7758
7759
7760	/**
7761	* Clears the interrupt-window exiting control in the VMCS.
7762	*
7763	* @param pVCpu The cross context virtual CPU structure.
7764	*/
7765	DECLINLINE(void) hmR0VmxClearIntWindowExitVmcs(PVMCPU pVCpu)
7766	{
7767	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7768	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7769	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7770	AssertRC(rc);
7771	Log4(("Cleared interrupt-window exiting\n"));
7772	}
7773
7774
7775	/**
7776	* Sets the NMI-window exiting control in the VMCS which instructs VT-x to
7777	* cause a VM-exit as soon as the guest is in a state to receive NMIs.
7778	*
7779	* @param pVCpu The cross context virtual CPU structure.
7780	*/
7781	DECLINLINE(void) hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu)
7782	{
7783	if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7784	{
7785	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7786	{
7787	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7788	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7789	AssertRC(rc);
7790	Log4(("Setup NMI-window exiting\n"));
7791	}
7792	} /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
7793	}
7794
7795
7796	/**
7797	* Clears the NMI-window exiting control in the VMCS.
7798	*
7799	* @param pVCpu The cross context virtual CPU structure.
7800	*/
7801	DECLINLINE(void) hmR0VmxClearNmiWindowExitVmcs(PVMCPU pVCpu)
7802	{
7803	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT);
7804	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7805	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7806	AssertRC(rc);
7807	Log4(("Cleared NMI-window exiting\n"));
7808	}
7809
7810
7811	/**
7812	* Evaluates the event to be delivered to the guest and sets it as the pending
7813	* event.
7814	*
7815	* @returns The VT-x guest-interruptibility state.
7816	* @param pVCpu The cross context virtual CPU structure.
7817	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7818	* out-of-sync. Make sure to update the required fields
7819	* before using them.
7820	*/
7821	static uint32_t hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7822	{
7823	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7824	uint32_t const uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7825	bool const fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7826	bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7827	bool const fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7828
7829	Assert(!fBlockSti \|\| HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7830	Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7831	Assert(!fBlockSti \|\| pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7832	Assert(!TRPMHasTrap(pVCpu));
7833
7834	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
7835	APICUpdatePendingInterrupts(pVCpu);
7836
7837	/*
7838	* Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
7839	* to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
7840	*/
7841	/** @todo SMI. SMIs take priority over NMIs. */
7842	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
7843	{
7844	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
7845	if ( !pVCpu->hm.s.Event.fPending
7846	&& !fBlockNmi
7847	&& !fBlockSti
7848	&& !fBlockMovSS)
7849	{
7850	Log4(("Pending NMI vcpu[%RU32]\n", pVCpu->idCpu));
7851	uint32_t u32IntInfo = X86_XCPT_NMI \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7852	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7853
7854	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7855	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
7856	}
7857	else
7858	hmR0VmxSetNmiWindowExitVmcs(pVCpu);
7859	}
7860	/*
7861	* Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt() returns
7862	* a valid interrupt we must- deliver the interrupt. We can no longer re-request it from the APIC.
7863	*/
7864	else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC))
7865	&& !pVCpu->hm.s.fSingleInstruction)
7866	{
7867	Assert(!DBGFIsStepping(pVCpu));
7868	int rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7869	AssertRC(rc);
7870	bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7871	if ( !pVCpu->hm.s.Event.fPending
7872	&& !fBlockInt
7873	&& !fBlockSti
7874	&& !fBlockMovSS)
7875	{
7876	uint8_t u8Interrupt;
7877	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
7878	if (RT_SUCCESS(rc))
7879	{
7880	Log4(("Pending interrupt vcpu[%RU32] u8Interrupt=%#x \n", pVCpu->idCpu, u8Interrupt));
7881	uint32_t u32IntInfo = u8Interrupt \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7882	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7883
7884	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrfaultAddress */);
7885	}
7886	else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
7887	{
7888	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
7889	hmR0VmxApicSetTprThreshold(pVCpu, u8Interrupt >> 4);
7890	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
7891
7892	/*
7893	* If the CPU doesn't have TPR shadowing, we will always get a VM-exit on TPR changes and
7894	* APICSetTpr() will end up setting the VMCPU_FF_INTERRUPT_APIC if required, so there is no
7895	* need to re-set this force-flag here.
7896	*/
7897	}
7898	else
7899	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
7900	}
7901	else
7902	hmR0VmxSetIntWindowExitVmcs(pVCpu);
7903	}
7904
7905	return uIntrState;
7906	}
7907
7908
7909	/**
7910	* Sets a pending-debug exception to be delivered to the guest if the guest is
7911	* single-stepping in the VMCS.
7912	*
7913	* @param pVCpu The cross context virtual CPU structure.
7914	*/
7915	DECLINLINE(void) hmR0VmxSetPendingDebugXcptVmcs(PVMCPU pVCpu)
7916	{
7917	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS)); NOREF(pVCpu);
7918	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
7919	AssertRC(rc);
7920	}
7921
7922
7923	/**
7924	* Injects any pending events into the guest if the guest is in a state to
7925	* receive them.
7926	*
7927	* @returns Strict VBox status code (i.e. informational status codes too).
7928	* @param pVCpu The cross context virtual CPU structure.
7929	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7930	* out-of-sync. Make sure to update the required fields
7931	* before using them.
7932	* @param uIntrState The VT-x guest-interruptibility state.
7933	* @param fStepping Running in hmR0VmxRunGuestCodeStep() and we should
7934	* return VINF_EM_DBG_STEPPED if the event was
7935	* dispatched directly.
7936	*/
7937	static VBOXSTRICTRC hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t uIntrState, bool fStepping)
7938	{
7939	HMVMX_ASSERT_PREEMPT_SAFE();
7940	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7941
7942	bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7943	bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7944
7945	Assert(!fBlockSti \|\| HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7946	Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7947	Assert(!fBlockSti \|\| pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7948	Assert(!TRPMHasTrap(pVCpu));
7949
7950	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
7951	if (pVCpu->hm.s.Event.fPending)
7952	{
7953	/*
7954	* Do -not- clear any interrupt-window exiting control here. We might have an interrupt
7955	* pending even while injecting an event and in this case, we want a VM-exit as soon as
7956	* the guest is ready for the next interrupt, see @bugref{6208#c45}.
7957	*
7958	* See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
7959	*/
7960	uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7961	#ifdef VBOX_STRICT
7962	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7963	{
7964	bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7965	Assert(!fBlockInt);
7966	Assert(!fBlockSti);
7967	Assert(!fBlockMovSS);
7968	}
7969	else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7970	{
7971	bool const fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7972	Assert(!fBlockSti);
7973	Assert(!fBlockMovSS);
7974	Assert(!fBlockNmi);
7975	}
7976	#endif
7977	Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#x\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
7978	(uint8_t)uIntType));
7979	rcStrict = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
7980	pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress,
7981	fStepping, &uIntrState);
7982	AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
7983
7984	/* Update the interruptibility-state as it could have been changed by
7985	hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
7986	fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7987	fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7988
7989	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7990	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
7991	else
7992	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
7993	}
7994
7995	/* Deliver pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
7996	if ( fBlockSti
7997	\|\| fBlockMovSS)
7998	{
7999	if (!pVCpu->hm.s.fSingleInstruction)
8000	{
8001	/*
8002	* The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
8003	* VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
8004	* See Intel spec. 27.3.4 "Saving Non-Register State".
8005	*/
8006	Assert(!DBGFIsStepping(pVCpu));
8007	int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8008	AssertRCReturn(rc2, rc2);
8009	if (pMixedCtx->eflags.Bits.u1TF)
8010	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
8011	}
8012	else if (pMixedCtx->eflags.Bits.u1TF)
8013	{
8014	/*
8015	* We are single-stepping in the hypervisor debugger using EFLAGS.TF. Clear interrupt inhibition as setting the
8016	* BS bit would mean delivering a #DB to the guest upon VM-entry when it shouldn't be.
8017	*/
8018	Assert(!(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG));
8019	uIntrState = 0;
8020	}
8021	}
8022
8023	/*
8024	* There's no need to clear the VM-entry interruption-information field here if we're not injecting anything.
8025	* VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
8026	*/
8027	int rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
8028	AssertRC(rc2);
8029
8030	Assert(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
8031	NOREF(fBlockMovSS); NOREF(fBlockSti);
8032	return rcStrict;
8033	}
8034
8035
8036	/**
8037	* Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
8038	*
8039	* @param pVCpu The cross context virtual CPU structure.
8040	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8041	* out-of-sync. Make sure to update the required fields
8042	* before using them.
8043	*/
8044	DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8045	{
8046	NOREF(pMixedCtx);
8047	uint32_t u32IntInfo = X86_XCPT_UD \| VMX_EXIT_INTERRUPTION_INFO_VALID;
8048	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
8049	}
8050
8051
8052	/**
8053	* Injects a double-fault (\#DF) exception into the VM.
8054	*
8055	* @returns Strict VBox status code (i.e. informational status codes too).
8056	* @param pVCpu The cross context virtual CPU structure.
8057	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8058	* out-of-sync. Make sure to update the required fields
8059	* before using them.
8060	* @param fStepping Whether we're running in hmR0VmxRunGuestCodeStep()
8061	* and should return VINF_EM_DBG_STEPPED if the event
8062	* is injected directly (register modified by us, not
8063	* by hardware on VM-entry).
8064	* @param puIntrState Pointer to the current guest interruptibility-state.
8065	* This interruptibility-state will be updated if
8066	* necessary. This cannot not be NULL.
8067	*/
8068	DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping, uint32_t *puIntrState)
8069	{
8070	uint32_t u32IntInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
8071	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8072	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
8073	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */,
8074	fStepping, puIntrState);
8075	}
8076
8077
8078	/**
8079	* Sets a debug (\#DB) exception as pending-for-injection into the VM.
8080	*
8081	* @param pVCpu The cross context virtual CPU structure.
8082	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8083	* out-of-sync. Make sure to update the required fields
8084	* before using them.
8085	*/
8086	DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8087	{
8088	NOREF(pMixedCtx);
8089	uint32_t u32IntInfo = X86_XCPT_DB \| VMX_EXIT_INTERRUPTION_INFO_VALID;
8090	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8091	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
8092	}
8093
8094
8095	/**
8096	* Sets an overflow (\#OF) exception as pending-for-injection into the VM.
8097	*
8098	* @param pVCpu The cross context virtual CPU structure.
8099	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8100	* out-of-sync. Make sure to update the required fields
8101	* before using them.
8102	* @param cbInstr The value of RIP that is to be pushed on the guest
8103	* stack.
8104	*/
8105	DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
8106	{
8107	NOREF(pMixedCtx);
8108	uint32_t u32IntInfo = X86_XCPT_OF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
8109	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8110	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
8111	}
8112
8113
8114	/**
8115	* Injects a general-protection (\#GP) fault into the VM.
8116	*
8117	* @returns Strict VBox status code (i.e. informational status codes too).
8118	* @param pVCpu The cross context virtual CPU structure.
8119	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8120	* out-of-sync. Make sure to update the required fields
8121	* before using them.
8122	* @param fErrorCodeValid Whether the error code is valid (depends on the CPU
8123	* mode, i.e. in real-mode it's not valid).
8124	* @param u32ErrorCode The error code associated with the \#GP.
8125	* @param fStepping Whether we're running in
8126	* hmR0VmxRunGuestCodeStep() and should return
8127	* VINF_EM_DBG_STEPPED if the event is injected
8128	* directly (register modified by us, not by
8129	* hardware on VM-entry).
8130	* @param puIntrState Pointer to the current guest interruptibility-state.
8131	* This interruptibility-state will be updated if
8132	* necessary. This cannot not be NULL.
8133	*/
8134	DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
8135	bool fStepping, uint32_t *puIntrState)
8136	{
8137	uint32_t u32IntInfo = X86_XCPT_GP \| VMX_EXIT_INTERRUPTION_INFO_VALID;
8138	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8139	if (fErrorCodeValid)
8140	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
8141	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr /, u32ErrorCode, 0 / GCPtrFaultAddress */,
8142	fStepping, puIntrState);
8143	}
8144
8145
8146	#if 0 /* unused */
8147	/**
8148	* Sets a general-protection (\#GP) exception as pending-for-injection into the
8149	* VM.
8150	*
8151	* @param pVCpu The cross context virtual CPU structure.
8152	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8153	* out-of-sync. Make sure to update the required fields
8154	* before using them.
8155	* @param u32ErrorCode The error code associated with the \#GP.
8156	*/
8157	DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t u32ErrorCode)
8158	{
8159	NOREF(pMixedCtx);
8160	uint32_t u32IntInfo = X86_XCPT_GP \| VMX_EXIT_INTERRUPTION_INFO_VALID;
8161	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8162	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
8163	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, u32ErrorCode, 0 / GCPtrFaultAddress */);
8164	}
8165	#endif /* unused */
8166
8167
8168	/**
8169	* Sets a software interrupt (INTn) as pending-for-injection into the VM.
8170	*
8171	* @param pVCpu The cross context virtual CPU structure.
8172	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8173	* out-of-sync. Make sure to update the required fields
8174	* before using them.
8175	* @param uVector The software interrupt vector number.
8176	* @param cbInstr The value of RIP that is to be pushed on the guest
8177	* stack.
8178	*/
8179	DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
8180	{
8181	NOREF(pMixedCtx);
8182	uint32_t u32IntInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
8183	if ( uVector == X86_XCPT_BP
8184	\|\| uVector == X86_XCPT_OF)
8185	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8186	else
8187	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8188	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
8189	}
8190
8191
8192	/**
8193	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
8194	* stack.
8195	*
8196	* @returns Strict VBox status code (i.e. informational status codes too).
8197	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
8198	* @param pVM The cross context VM structure.
8199	* @param pMixedCtx Pointer to the guest-CPU context.
8200	* @param uValue The value to push to the guest stack.
8201	*/
8202	DECLINLINE(VBOXSTRICTRC) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
8203	{
8204	/*
8205	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
8206	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
8207	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
8208	*/
8209	if (pMixedCtx->sp == 1)
8210	return VINF_EM_RESET;
8211	pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
8212	int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
8213	AssertRC(rc);
8214	return rc;
8215	}
8216
8217
8218	/**
8219	* Injects an event into the guest upon VM-entry by updating the relevant fields
8220	* in the VM-entry area in the VMCS.
8221	*
8222	* @returns Strict VBox status code (i.e. informational status codes too).
8223	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
8224	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
8225	*
8226	* @param pVCpu The cross context virtual CPU structure.
8227	* @param pMixedCtx Pointer to the guest-CPU context. The data may
8228	* be out-of-sync. Make sure to update the required
8229	* fields before using them.
8230	* @param u64IntInfo The VM-entry interruption-information field.
8231	* @param cbInstr The VM-entry instruction length in bytes (for
8232	* software interrupts, exceptions and privileged
8233	* software exceptions).
8234	* @param u32ErrCode The VM-entry exception error code.
8235	* @param GCPtrFaultAddress The page-fault address for \#PF exceptions.
8236	* @param puIntrState Pointer to the current guest interruptibility-state.
8237	* This interruptibility-state will be updated if
8238	* necessary. This cannot not be NULL.
8239	* @param fStepping Whether we're running in
8240	* hmR0VmxRunGuestCodeStep() and should return
8241	* VINF_EM_DBG_STEPPED if the event is injected
8242	* directly (register modified by us, not by
8243	* hardware on VM-entry).
8244	*
8245	* @remarks Requires CR0!
8246	*/
8247	static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
8248	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, bool fStepping,
8249	uint32_t *puIntrState)
8250	{
8251	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
8252	AssertMsg(u64IntInfo >> 32 == 0, ("%#RX64\n", u64IntInfo));
8253	Assert(puIntrState);
8254	uint32_t u32IntInfo = (uint32_t)u64IntInfo;
8255
8256	uint32_t const uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntInfo);
8257	uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo);
8258
8259	#ifdef VBOX_STRICT
8260	/*
8261	* Validate the error-code-valid bit for hardware exceptions.
8262	* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling"
8263	*/
8264	if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
8265	&& !CPUMIsGuestInRealModeEx(pMixedCtx))
8266	{
8267	switch (uVector)
8268	{
8269	case X86_XCPT_PF:
8270	case X86_XCPT_DF:
8271	case X86_XCPT_TS:
8272	case X86_XCPT_NP:
8273	case X86_XCPT_SS:
8274	case X86_XCPT_GP:
8275	case X86_XCPT_AC:
8276	AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo),
8277	("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
8278	RT_FALL_THRU();
8279	default:
8280	break;
8281	}
8282	}
8283	#endif
8284
8285	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
8286	Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
8287	\|\| !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
8288
8289	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
8290
8291	/* We require CR0 to check if the guest is in real-mode. */
8292	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8293	AssertRCReturn(rc, rc);
8294
8295	/*
8296	* Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
8297	* mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
8298	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
8299	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
8300	*/
8301	if (CPUMIsGuestInRealModeEx(pMixedCtx))
8302	{
8303	PVM pVM = pVCpu->CTX_SUFF(pVM);
8304	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
8305	{
8306	Assert(PDMVmmDevHeapIsEnabled(pVM));
8307	Assert(pVM->hm.s.vmx.pRealModeTSS);
8308
8309	/* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
8310	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8311	rc \|= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
8312	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8313	AssertRCReturn(rc, rc);
8314	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP));
8315
8316	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
8317	size_t const cbIdtEntry = sizeof(X86IDTR16);
8318	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
8319	{
8320	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
8321	if (uVector == X86_XCPT_DF)
8322	return VINF_EM_RESET;
8323
8324	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
8325	if (uVector == X86_XCPT_GP)
8326	return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, fStepping, puIntrState);
8327
8328	/* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
8329	/* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
8330	return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid /, 0 / u32ErrCode */,
8331	fStepping, puIntrState);
8332	}
8333
8334	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
8335	uint16_t uGuestIp = pMixedCtx->ip;
8336	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
8337	{
8338	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
8339	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
8340	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
8341	}
8342	else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
8343	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
8344
8345	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
8346	X86IDTR16 IdtEntry;
8347	RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
8348	rc = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
8349	AssertRCReturn(rc, rc);
8350
8351	/* Construct the stack frame for the interrupt/exception handler. */
8352	VBOXSTRICTRC rcStrict;
8353	rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
8354	if (rcStrict == VINF_SUCCESS)
8355	rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
8356	if (rcStrict == VINF_SUCCESS)
8357	rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
8358
8359	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
8360	if (rcStrict == VINF_SUCCESS)
8361	{
8362	pMixedCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
8363	pMixedCtx->rip = IdtEntry.offSel;
8364	pMixedCtx->cs.Sel = IdtEntry.uSel;
8365	pMixedCtx->cs.ValidSel = IdtEntry.uSel;
8366	pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
8367	if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
8368	&& uVector == X86_XCPT_PF)
8369	pMixedCtx->cr2 = GCPtrFaultAddress;
8370
8371	/* If any other guest-state bits are changed here, make sure to update
8372	hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
8373	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS
8374	\| HM_CHANGED_GUEST_RIP
8375	\| HM_CHANGED_GUEST_RFLAGS
8376	\| HM_CHANGED_GUEST_RSP);
8377
8378	/* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
8379	if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
8380	{
8381	Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
8382	&& uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
8383	Log4(("Clearing inhibition due to STI.\n"));
8384	*puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
8385	}
8386	Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
8387	u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->eflags.u, pMixedCtx->cs.Sel, pMixedCtx->eip));
8388
8389	/* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
8390	it, if we are returning to ring-3 before executing guest code. */
8391	pVCpu->hm.s.Event.fPending = false;
8392
8393	/* Make hmR0VmxPreRunGuest return if we're stepping since we've changed cs:rip. */
8394	if (fStepping)
8395	rcStrict = VINF_EM_DBG_STEPPED;
8396	}
8397	AssertMsg(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8398	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8399	return rcStrict;
8400	}
8401
8402	/*
8403	* For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
8404	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
8405	*/
8406	u32IntInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
8407	}
8408
8409	/* Validate. */
8410	Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
8411	Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(u32IntInfo)); /* Bit 12 MBZ. */
8412	Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
8413
8414	/* Inject. */
8415	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
8416	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo))
8417	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
8418	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
8419
8420	if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
8421	&& uVector == X86_XCPT_PF)
8422	pMixedCtx->cr2 = GCPtrFaultAddress;
8423
8424	Log4(("Injecting vcpu[%RU32] u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", pVCpu->idCpu,
8425	u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
8426
8427	AssertRCReturn(rc, rc);
8428	return VINF_SUCCESS;
8429	}
8430
8431
8432	/**
8433	* Clears the interrupt-window exiting control in the VMCS and if necessary
8434	* clears the current event in the VMCS as well.
8435	*
8436	* @returns VBox status code.
8437	* @param pVCpu The cross context virtual CPU structure.
8438	*
8439	* @remarks Use this function only to clear events that have not yet been
8440	* delivered to the guest but are injected in the VMCS!
8441	* @remarks No-long-jump zone!!!
8442	*/
8443	static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu)
8444	{
8445	Log4Func(("vcpu[%d]\n", pVCpu->idCpu));
8446
8447	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
8448	hmR0VmxClearIntWindowExitVmcs(pVCpu);
8449
8450	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)
8451	hmR0VmxClearNmiWindowExitVmcs(pVCpu);
8452	}
8453
8454
8455	/**
8456	* Enters the VT-x session.
8457	*
8458	* @returns VBox status code.
8459	* @param pVM The cross context VM structure.
8460	* @param pVCpu The cross context virtual CPU structure.
8461	* @param pCpu Pointer to the CPU info struct.
8462	*/
8463	VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
8464	{
8465	AssertPtr(pVM);
8466	AssertPtr(pVCpu);
8467	Assert(pVM->hm.s.vmx.fSupported);
8468	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8469	NOREF(pCpu); NOREF(pVM);
8470
8471	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8472	Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE));
8473
8474	#ifdef VBOX_STRICT
8475	/* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
8476	RTCCUINTREG uHostCR4 = ASMGetCR4();
8477	if (!(uHostCR4 & X86_CR4_VMXE))
8478	{
8479	LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
8480	return VERR_VMX_X86_CR4_VMXE_CLEARED;
8481	}
8482	#endif
8483
8484	/*
8485	* Load the VCPU's VMCS as the current (and active) one.
8486	*/
8487	Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
8488	int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8489	if (RT_FAILURE(rc))
8490	return rc;
8491
8492	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8493	pVCpu->hm.s.fLeaveDone = false;
8494	Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8495
8496	return VINF_SUCCESS;
8497	}
8498
8499
8500	/**
8501	* The thread-context callback (only on platforms which support it).
8502	*
8503	* @param enmEvent The thread-context event.
8504	* @param pVCpu The cross context virtual CPU structure.
8505	* @param fGlobalInit Whether global VT-x/AMD-V init. was used.
8506	* @thread EMT(pVCpu)
8507	*/
8508	VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
8509	{
8510	NOREF(fGlobalInit);
8511
8512	switch (enmEvent)
8513	{
8514	case RTTHREADCTXEVENT_OUT:
8515	{
8516	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8517	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8518	VMCPU_ASSERT_EMT(pVCpu);
8519
8520	PCPUMCTX pMixedCtx = CPUMQueryGuestCtxPtr(pVCpu);
8521
8522	/* No longjmps (logger flushes, locks) in this fragile context. */
8523	VMMRZCallRing3Disable(pVCpu);
8524	Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
8525
8526	/*
8527	* Restore host-state (FPU, debug etc.)
8528	*/
8529	if (!pVCpu->hm.s.fLeaveDone)
8530	{
8531	/* Do -not- save guest-state here as we might already be in the middle of saving it (esp. bad if we are
8532	holding the PGM lock while saving the guest state (see hmR0VmxSaveGuestControlRegs()). */
8533	hmR0VmxLeave(pVCpu, pMixedCtx, false /* fSaveGuestState */);
8534	pVCpu->hm.s.fLeaveDone = true;
8535	}
8536
8537	/* Leave HM context, takes care of local init (term). */
8538	int rc = HMR0LeaveCpu(pVCpu);
8539	AssertRC(rc); NOREF(rc);
8540
8541	/* Restore longjmp state. */
8542	VMMRZCallRing3Enable(pVCpu);
8543	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
8544	break;
8545	}
8546
8547	case RTTHREADCTXEVENT_IN:
8548	{
8549	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8550	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8551	VMCPU_ASSERT_EMT(pVCpu);
8552
8553	/* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
8554	VMMRZCallRing3Disable(pVCpu);
8555	Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
8556
8557	/* Initialize the bare minimum state required for HM. This takes care of
8558	initializing VT-x if necessary (onlined CPUs, local init etc.) */
8559	int rc = HMR0EnterCpu(pVCpu);
8560	AssertRC(rc);
8561	Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE));
8562
8563	/* Load the active VMCS as the current one. */
8564	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
8565	{
8566	rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8567	AssertRC(rc); NOREF(rc);
8568	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8569	Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8570	}
8571	pVCpu->hm.s.fLeaveDone = false;
8572
8573	/* Restore longjmp state. */
8574	VMMRZCallRing3Enable(pVCpu);
8575	break;
8576	}
8577
8578	default:
8579	break;
8580	}
8581	}
8582
8583
8584	/**
8585	* Saves the host state in the VMCS host-state.
8586	* Sets up the VM-exit MSR-load area.
8587	*
8588	* The CPU state will be loaded from these fields on every successful VM-exit.
8589	*
8590	* @returns VBox status code.
8591	* @param pVM The cross context VM structure.
8592	* @param pVCpu The cross context virtual CPU structure.
8593	*
8594	* @remarks No-long-jump zone!!!
8595	*/
8596	static int hmR0VmxSaveHostState(PVM pVM, PVMCPU pVCpu)
8597	{
8598	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8599
8600	int rc = VINF_SUCCESS;
8601	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8602	{
8603	rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
8604	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8605
8606	rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
8607	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8608
8609	rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
8610	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8611
8612	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_HOST_CONTEXT);
8613	}
8614	return rc;
8615	}
8616
8617
8618	/**
8619	* Saves the host state in the VMCS host-state.
8620	*
8621	* @returns VBox status code.
8622	* @param pVM The cross context VM structure.
8623	* @param pVCpu The cross context virtual CPU structure.
8624	*
8625	* @remarks No-long-jump zone!!!
8626	*/
8627	VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
8628	{
8629	AssertPtr(pVM);
8630	AssertPtr(pVCpu);
8631
8632	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8633
8634	/* Save the host state here while entering HM context. When thread-context hooks are used, we might get preempted
8635	and have to resave the host state but most of the time we won't be, so do it here before we disable interrupts. */
8636	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8637	return hmR0VmxSaveHostState(pVM, pVCpu);
8638	}
8639
8640
8641	/**
8642	* Loads the guest state into the VMCS guest-state area.
8643	*
8644	* The will typically be done before VM-entry when the guest-CPU state and the
8645	* VMCS state may potentially be out of sync.
8646	*
8647	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
8648	* VM-entry controls.
8649	* Sets up the appropriate VMX non-root function to execute guest code based on
8650	* the guest CPU mode.
8651	*
8652	* @returns VBox strict status code.
8653	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8654	* without unrestricted guest access and the VMMDev is not presently
8655	* mapped (e.g. EFI32).
8656	*
8657	* @param pVM The cross context VM structure.
8658	* @param pVCpu The cross context virtual CPU structure.
8659	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8660	* out-of-sync. Make sure to update the required fields
8661	* before using them.
8662	*
8663	* @remarks No-long-jump zone!!!
8664	*/
8665	static VBOXSTRICTRC hmR0VmxLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8666	{
8667	AssertPtr(pVM);
8668	AssertPtr(pVCpu);
8669	AssertPtr(pMixedCtx);
8670	HMVMX_ASSERT_PREEMPT_SAFE();
8671
8672	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8673
8674	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
8675
8676	/* Determine real-on-v86 mode. */
8677	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
8678	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
8679	&& CPUMIsGuestInRealModeEx(pMixedCtx))
8680	{
8681	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
8682	}
8683
8684	/*
8685	* Load the guest-state into the VMCS.
8686	* Any ordering dependency among the sub-functions below must be explicitly stated using comments.
8687	* Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
8688	*/
8689	int rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
8690	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8691
8692	/* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
8693	rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
8694	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8695
8696	/* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
8697	rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
8698	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8699
8700	rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
8701	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8702
8703	VBOXSTRICTRC rcStrict = hmR0VmxLoadGuestCR3AndCR4(pVCpu, pMixedCtx);
8704	if (rcStrict == VINF_SUCCESS)
8705	{ /* likely */ }
8706	else
8707	{
8708	Assert(rcStrict == VINF_EM_RESCHEDULE_REM \|\| RT_FAILURE_NP(rcStrict));
8709	return rcStrict;
8710	}
8711
8712	/* Assumes pMixedCtx->cr0 is up-to-date (strict builds require CR0 for segment register validation checks). */
8713	rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
8714	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8715
8716	/* This needs to be done after hmR0VmxLoadGuestEntryCtls() and hmR0VmxLoadGuestExitCtls() as it may alter controls if we
8717	determine we don't have to swap EFER after all. */
8718	rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
8719	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8720
8721	rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
8722	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8723
8724	rc = hmR0VmxLoadGuestXcptIntercepts(pVCpu, pMixedCtx);
8725	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestXcptIntercepts! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8726
8727	/*
8728	* Loading Rflags here is fine, even though Rflags.TF might depend on guest debug state (which is not loaded here).
8729	* It is re-evaluated and updated if necessary in hmR0VmxLoadSharedState().
8730	*/
8731	rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
8732	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestRipRspRflags! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8733
8734	/* Clear any unused and reserved bits. */
8735	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR2);
8736
8737	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
8738	return rc;
8739	}
8740
8741
8742	/**
8743	* Loads the state shared between the host and guest into the VMCS.
8744	*
8745	* @param pVM The cross context VM structure.
8746	* @param pVCpu The cross context virtual CPU structure.
8747	* @param pCtx Pointer to the guest-CPU context.
8748	*
8749	* @remarks No-long-jump zone!!!
8750	*/
8751	static void hmR0VmxLoadSharedState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8752	{
8753	NOREF(pVM);
8754
8755	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8756	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8757
8758	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
8759	{
8760	int rc = hmR0VmxLoadSharedCR0(pVCpu, pCtx);
8761	AssertRC(rc);
8762	}
8763
8764	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
8765	{
8766	int rc = hmR0VmxLoadSharedDebugState(pVCpu, pCtx);
8767	AssertRC(rc);
8768
8769	/* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
8770	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
8771	{
8772	rc = hmR0VmxLoadGuestRflags(pVCpu, pCtx);
8773	AssertRC(rc);
8774	}
8775	}
8776
8777	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS))
8778	{
8779	hmR0VmxLazyLoadGuestMsrs(pVCpu, pCtx);
8780	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
8781	}
8782
8783	/* Loading CR0, debug state might have changed intercepts, update VMCS. */
8784	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS))
8785	{
8786	Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_AC));
8787	Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
8788	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
8789	AssertRC(rc);
8790	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
8791	}
8792
8793	AssertMsg(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE),
8794	("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8795	}
8796
8797
8798	/**
8799	* Worker for loading the guest-state bits in the inner VT-x execution loop.
8800	*
8801	* @returns Strict VBox status code (i.e. informational status codes too).
8802	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8803	* without unrestricted guest access and the VMMDev is not presently
8804	* mapped (e.g. EFI32).
8805	*
8806	* @param pVM The cross context VM structure.
8807	* @param pVCpu The cross context virtual CPU structure.
8808	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8809	* out-of-sync. Make sure to update the required fields
8810	* before using them.
8811	*
8812	* @remarks No-long-jump zone!!!
8813	*/
8814	static VBOXSTRICTRC hmR0VmxLoadGuestStateOptimal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8815	{
8816	HMVMX_ASSERT_PREEMPT_SAFE();
8817	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8818	Assert(VMMR0IsLogFlushDisabled(pVCpu));
8819
8820	Log5(("LoadFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8821	#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
8822	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
8823	#endif
8824
8825	/*
8826	* RIP is what changes the most often and hence if it's the only bit needing to be
8827	* updated, we shall handle it early for performance reasons.
8828	*/
8829	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
8830	if (HMCPU_CF_IS_SET_ONLY(pVCpu, HM_CHANGED_GUEST_RIP))
8831	{
8832	rcStrict = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
8833	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8834	{ /* likely */}
8835	else
8836	{
8837	AssertMsgFailedReturn(("hmR0VmxLoadGuestStateOptimal: hmR0VmxLoadGuestRip failed! rc=%Rrc\n",
8838	VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
8839	}
8840	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
8841	}
8842	else if (HMCPU_CF_VALUE(pVCpu))
8843	{
8844	rcStrict = hmR0VmxLoadGuestState(pVM, pVCpu, pMixedCtx);
8845	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8846	{ /* likely */}
8847	else
8848	{
8849	AssertMsg(rcStrict == VINF_EM_RESCHEDULE_REM,
8850	("hmR0VmxLoadGuestStateOptimal: hmR0VmxLoadGuestState failed! rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8851	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8852	return rcStrict;
8853	}
8854	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
8855	}
8856
8857	/* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
8858	AssertMsg( !HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_ALL_GUEST)
8859	\|\| HMCPU_CF_IS_PENDING_ONLY(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE),
8860	("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8861	return rcStrict;
8862	}
8863
8864
8865	/**
8866	* Does the preparations before executing guest code in VT-x.
8867	*
8868	* This may cause longjmps to ring-3 and may even result in rescheduling to the
8869	* recompiler/IEM. We must be cautious what we do here regarding committing
8870	* guest-state information into the VMCS assuming we assuredly execute the
8871	* guest in VT-x mode.
8872	*
8873	* If we fall back to the recompiler/IEM after updating the VMCS and clearing
8874	* the common-state (TRPM/forceflags), we must undo those changes so that the
8875	* recompiler/IEM can (and should) use them when it resumes guest execution.
8876	* Otherwise such operations must be done when we can no longer exit to ring-3.
8877	*
8878	* @returns Strict VBox status code (i.e. informational status codes too).
8879	* @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
8880	* have been disabled.
8881	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a
8882	* double-fault into the guest.
8883	* @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
8884	* dispatched directly.
8885	* @retval VINF_* scheduling changes, we have to go back to ring-3.
8886	*
8887	* @param pVM The cross context VM structure.
8888	* @param pVCpu The cross context virtual CPU structure.
8889	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8890	* out-of-sync. Make sure to update the required fields
8891	* before using them.
8892	* @param pVmxTransient Pointer to the VMX transient structure.
8893	* @param fStepping Set if called from hmR0VmxRunGuestCodeStep(). Makes
8894	* us ignore some of the reasons for returning to
8895	* ring-3, and return VINF_EM_DBG_STEPPED if event
8896	* dispatching took place.
8897	*/
8898	static VBOXSTRICTRC hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, bool fStepping)
8899	{
8900	Assert(VMMRZCallRing3IsEnabled(pVCpu));
8901
8902	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
8903	PGMRZDynMapFlushAutoSet(pVCpu);
8904	#endif
8905
8906	/* Check force flag actions that might require us to go back to ring-3. */
8907	VBOXSTRICTRC rcStrict = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx, fStepping);
8908	if (rcStrict == VINF_SUCCESS)
8909	{ /* FFs doesn't get set all the time. */ }
8910	else
8911	return rcStrict;
8912
8913	#ifndef IEM_VERIFICATION_MODE_FULL
8914	/*
8915	* Setup the virtualized-APIC accesses.
8916	*
8917	* Note! This can cause a longjumps to R3 due to the acquisition of the PGM lock
8918	* in both PGMHandlerPhysicalReset() and IOMMMIOMapMMIOHCPage(), see @bugref{8721}.
8919	*
8920	* This is the reason we do it here and not in hmR0VmxLoadGuestState().
8921	*/
8922	if ( !pVCpu->hm.s.vmx.u64MsrApicBase
8923	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
8924	&& PDMHasApic(pVM))
8925	{
8926	uint64_t const u64MsrApicBase = APICGetBaseMsrNoCheck(pVCpu);
8927	Assert(u64MsrApicBase);
8928	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
8929
8930	RTGCPHYS const GCPhysApicBase = u64MsrApicBase & PAGE_BASE_GC_MASK;
8931
8932	/* Unalias any existing mapping. */
8933	int rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
8934	AssertRCReturn(rc, rc);
8935
8936	/* Map the HC APIC-access page in place of the MMIO page, also updates the shadow page tables if necessary. */
8937	Log4(("hmR0VmxPreRunGuest: VCPU%u: Mapped HC APIC-access page at %#RGp\n", pVCpu->idCpu, GCPhysApicBase));
8938	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
8939	AssertRCReturn(rc, rc);
8940
8941	/* Update the per-VCPU cache of the APIC base MSR. */
8942	pVCpu->hm.s.vmx.u64MsrApicBase = u64MsrApicBase;
8943	}
8944	#endif /* !IEM_VERIFICATION_MODE_FULL */
8945
8946	if (TRPMHasTrap(pVCpu))
8947	hmR0VmxTrpmTrapToPendingEvent(pVCpu);
8948	uint32_t uIntrState = hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
8949
8950	/*
8951	* Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus needs to be done with
8952	* longjmps or interrupts + preemption enabled. Event injection might also result in triple-faulting the VM.
8953	*/
8954	rcStrict = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx, uIntrState, fStepping);
8955	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8956	{ /* likely */ }
8957	else
8958	{
8959	AssertMsg(rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8960	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8961	return rcStrict;
8962	}
8963
8964	/*
8965	* No longjmps to ring-3 from this point on!!!
8966	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
8967	* This also disables flushing of the R0-logger instance (if any).
8968	*/
8969	VMMRZCallRing3Disable(pVCpu);
8970
8971	/*
8972	* Load the guest state bits.
8973	*
8974	* We cannot perform longjmps while loading the guest state because we do not preserve the
8975	* host/guest state (although the VMCS will be preserved) across longjmps which can cause
8976	* CPU migration.
8977	*
8978	* If we are injecting events to a real-on-v86 mode guest, we will have to update
8979	* RIP and some segment registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
8980	* Hence, loading of the guest state needs to be done -after- injection of events.
8981	*/
8982	rcStrict = hmR0VmxLoadGuestStateOptimal(pVM, pVCpu, pMixedCtx);
8983	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8984	{ /* likely */ }
8985	else
8986	{
8987	VMMRZCallRing3Enable(pVCpu);
8988	return rcStrict;
8989	}
8990
8991	/*
8992	* We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.)
8993	* when thread-context hooks aren't used and we've been running with preemption disabled for a while.
8994	*
8995	* We need to check for force-flags that could've possible been altered since we last checked them (e.g.
8996	* by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
8997	*
8998	* We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
8999	* executing guest code.
9000	*/
9001	pVmxTransient->fEFlags = ASMIntDisableFlags();
9002
9003	if ( ( !VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS \| VM_FF_TM_VIRTUAL_SYNC)
9004	&& !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
9005	\|\| ( fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
9006	&& !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER \| VMCPU_FF_PDM_CRITSECT))) )
9007	{
9008	if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
9009	{
9010	pVCpu->hm.s.Event.fPending = false;
9011
9012	/*
9013	* We've injected any pending events. This is really the point of no return (to ring-3).
9014	*
9015	* Note! The caller expects to continue with interrupts & longjmps disabled on successful
9016	* returns from this function, so don't enable them here.
9017	*/
9018	return VINF_SUCCESS;
9019	}
9020
9021	STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
9022	rcStrict = VINF_EM_RAW_INTERRUPT;
9023	}
9024	else
9025	{
9026	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
9027	rcStrict = VINF_EM_RAW_TO_R3;
9028	}
9029
9030	ASMSetFlags(pVmxTransient->fEFlags);
9031	VMMRZCallRing3Enable(pVCpu);
9032
9033	return rcStrict;
9034	}
9035
9036
9037	/**
9038	* Prepares to run guest code in VT-x and we've committed to doing so. This
9039	* means there is no backing out to ring-3 or anywhere else at this
9040	* point.
9041	*
9042	* @param pVM The cross context VM structure.
9043	* @param pVCpu The cross context virtual CPU structure.
9044	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
9045	* out-of-sync. Make sure to update the required fields
9046	* before using them.
9047	* @param pVmxTransient Pointer to the VMX transient structure.
9048	*
9049	* @remarks Called with preemption disabled.
9050	* @remarks No-long-jump zone!!!
9051	*/
9052	static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9053	{
9054	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9055	Assert(VMMR0IsLogFlushDisabled(pVCpu));
9056	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9057
9058	/*
9059	* Indicate start of guest execution and where poking EMT out of guest-context is recognized.
9060	*/
9061	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
9062	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
9063
9064	#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
9065	if (!CPUMIsGuestFPUStateActive(pVCpu))
9066	if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
9067	HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT);
9068	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9069	#endif
9070
9071	if ( pVCpu->hm.s.fPreloadGuestFpu
9072	&& !CPUMIsGuestFPUStateActive(pVCpu))
9073	{
9074	if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
9075	HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT);
9076	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
9077	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9078	}
9079
9080	/*
9081	* Lazy-update of the host MSRs values in the auto-load/store MSR area.
9082	*/
9083	if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
9084	&& pVCpu->hm.s.vmx.cMsrs > 0)
9085	{
9086	hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
9087	}
9088
9089	/*
9090	* Load the host state bits as we may've been preempted (only happens when
9091	* thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
9092	* Note that the 64-on-32 switcher saves the (64-bit) host state into the VMCS and
9093	* if we change the switcher back to 32-bit, we must save the 32-bit host state here.
9094	* See @bugref{8432}.
9095	*/
9096	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
9097	{
9098	int rc = hmR0VmxSaveHostState(pVM, pVCpu);
9099	AssertRC(rc);
9100	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreemptSaveHostState);
9101	}
9102	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT));
9103
9104	/*
9105	* Load the state shared between host and guest (FPU, debug, lazy MSRs).
9106	*/
9107	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE))
9108	hmR0VmxLoadSharedState(pVM, pVCpu, pMixedCtx);
9109	AssertMsg(!HMCPU_CF_VALUE(pVCpu), ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
9110
9111	/* Store status of the shared guest-host state at the time of VM-entry. */
9112	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
9113	if (CPUMIsGuestInLongModeEx(pMixedCtx))
9114	{
9115	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
9116	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
9117	}
9118	else
9119	#endif
9120	{
9121	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
9122	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
9123	}
9124	pVmxTransient->fWasGuestFPUStateActive = CPUMIsGuestFPUStateActive(pVCpu);
9125
9126	/*
9127	* Cache the TPR-shadow for checking on every VM-exit if it might have changed.
9128	*/
9129	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
9130	pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR];
9131
9132	PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
9133	RTCPUID idCurrentCpu = pCpu->idCpu;
9134	if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
9135	\|\| idCurrentCpu != pVCpu->hm.s.idLastCpu)
9136	{
9137	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVM, pVCpu);
9138	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
9139	}
9140
9141	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
9142	hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
9143	Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
9144	pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
9145
9146	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
9147
9148	TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
9149	to start executing. */
9150
9151	/*
9152	* Load the TSC_AUX MSR when we are not intercepting RDTSCP.
9153	*/
9154	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
9155	{
9156	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
9157	{
9158	bool fMsrUpdated;
9159	int rc2 = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
9160	AssertRC(rc2);
9161	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS));
9162
9163	rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, CPUMR0GetGuestTscAux(pVCpu), true /* fUpdateHostMsr */,
9164	&fMsrUpdated);
9165	AssertRC(rc2);
9166	Assert(fMsrUpdated \|\| pVCpu->hm.s.vmx.fUpdatedHostMsrs);
9167
9168	/* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
9169	pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
9170	}
9171	else
9172	{
9173	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
9174	Assert(!pVCpu->hm.s.vmx.cMsrs \|\| pVCpu->hm.s.vmx.fUpdatedHostMsrs);
9175	}
9176	}
9177
9178	#ifdef VBOX_STRICT
9179	hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
9180	hmR0VmxCheckHostEferMsr(pVCpu);
9181	AssertRC(hmR0VmxCheckVmcsCtls(pVCpu));
9182	#endif
9183	#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
9184	uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVM, pVCpu, pMixedCtx);
9185	if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
9186	Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
9187	#endif
9188	}
9189
9190
9191	/**
9192	* Performs some essential restoration of state after running guest code in
9193	* VT-x.
9194	*
9195	* @param pVM The cross context VM structure.
9196	* @param pVCpu The cross context virtual CPU structure.
9197	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
9198	* out-of-sync. Make sure to update the required fields
9199	* before using them.
9200	* @param pVmxTransient Pointer to the VMX transient structure.
9201	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
9202	*
9203	* @remarks Called with interrupts disabled, and returns with interrupts enabled!
9204	*
9205	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
9206	* unconditionally when it is safe to do so.
9207	*/
9208	static void hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
9209	{
9210	NOREF(pVM);
9211
9212	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9213
9214	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
9215	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
9216	HMVMXCPU_GST_RESET_TO(pVCpu, 0); /* Exits/longjmps to ring-3 requires saving the guest state. */
9217	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
9218	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
9219	pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
9220
9221	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
9222	TMCpuTickSetLastSeen(pVCpu, ASMReadTSC() + pVCpu->hm.s.vmx.u64TSCOffset);
9223
9224	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
9225	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
9226	Assert(!ASMIntAreEnabled());
9227	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
9228
9229	#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
9230	if (CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVM, pVCpu))
9231	{
9232	hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9233	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9234	}
9235	#endif
9236
9237	#if HC_ARCH_BITS == 64
9238	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
9239	#endif
9240	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
9241	/* The 64-on-32 switcher maintains uVmcsState on its own and we need to leave it alone here. */
9242	if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
9243	pVCpu->hm.s.vmx.uVmcsState \|= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
9244	#else
9245	pVCpu->hm.s.vmx.uVmcsState \|= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
9246	#endif
9247	#ifdef VBOX_STRICT
9248	hmR0VmxCheckHostEferMsr(pVCpu); /* Verify that VMRUN/VMLAUNCH didn't modify host EFER. */
9249	#endif
9250	ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
9251
9252	/* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
9253	uint32_t uExitReason;
9254	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
9255	rc \|= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
9256	AssertRC(rc);
9257	pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
9258	pVmxTransient->fVMEntryFailed = VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uEntryIntInfo);
9259
9260	if (rcVMRun == VINF_SUCCESS)
9261	{
9262	/*
9263	* Update the VM-exit history array here even if the VM-entry failed due to:
9264	* - Invalid guest state.
9265	* - MSR loading.
9266	* - Machine-check event.
9267	*
9268	* In any of the above cases we will still have a "valid" VM-exit reason
9269	* despite @a fVMEntryFailed being false.
9270	*
9271	* See Intel spec. 26.7 "VM-Entry failures during or after loading guest state".
9272	*/
9273	HMCPU_EXIT_HISTORY_ADD(pVCpu, pVmxTransient->uExitReason);
9274
9275	if (!pVmxTransient->fVMEntryFailed)
9276	{
9277	/** @todo We can optimize this by only syncing with our force-flags when
9278	* really needed and keeping the VMCS state as it is for most
9279	* VM-exits. */
9280	/* Update the guest interruptibility-state from the VMCS. */
9281	hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
9282
9283	/*
9284	* Allow longjmps to ring-3 -after- saving the guest-interruptibility state
9285	* as it's not part of hmR0VmxSaveGuestState() and thus would trigger an assertion
9286	* on the longjmp path to ring-3 while saving the (rest of) the guest state,
9287	* see @bugref{6208#c63}.
9288	*/
9289	VMMRZCallRing3Enable(pVCpu);
9290
9291	#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) \|\| defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
9292	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9293	AssertRC(rc);
9294	#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
9295	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9296	AssertRC(rc);
9297	#endif
9298
9299	/*
9300	* Sync the TPR shadow with our APIC state.
9301	*/
9302	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
9303	&& pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR])
9304	{
9305	rc = APICSetTpr(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR]);
9306	AssertRC(rc);
9307	HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
9308	}
9309
9310	return;
9311	}
9312	}
9313	else
9314	{
9315	Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
9316	pVmxTransient->fVMEntryFailed));
9317	}
9318
9319	VMMRZCallRing3Enable(pVCpu);
9320	}
9321
9322
9323	/**
9324	* Runs the guest code using VT-x the normal way.
9325	*
9326	* @returns VBox status code.
9327	* @param pVM The cross context VM structure.
9328	* @param pVCpu The cross context virtual CPU structure.
9329	* @param pCtx Pointer to the guest-CPU context.
9330	*
9331	* @note Mostly the same as hmR0VmxRunGuestCodeStep().
9332	*/
9333	static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
9334	{
9335	VMXTRANSIENT VmxTransient;
9336	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
9337	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
9338	uint32_t cLoops = 0;
9339
9340	for (;; cLoops++)
9341	{
9342	Assert(!HMR0SuspendPending());
9343	HMVMX_ASSERT_CPU_SAFE();
9344
9345	/* Preparatory work for running guest code, this may force us to return
9346	to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
9347	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
9348	rcStrict = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, false /* fStepping */);
9349	if (rcStrict != VINF_SUCCESS)
9350	break;
9351
9352	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
9353	int rcRun = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
9354	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
9355
9356	/* Restore any residual host-state and save any bits shared between host
9357	and guest into the guest-CPU state. Re-enables interrupts! */
9358	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rcRun);
9359
9360	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
9361	if (RT_SUCCESS(rcRun))
9362	{ /* very likely */ }
9363	else
9364	{
9365	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
9366	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rcRun, pCtx, &VmxTransient);
9367	return rcRun;
9368	}
9369
9370	/* Profile the VM-exit. */
9371	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
9372	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
9373	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
9374	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
9375	HMVMX_START_EXIT_DISPATCH_PROF();
9376
9377	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
9378
9379	/* Handle the VM-exit. */
9380	#ifdef HMVMX_USE_FUNCTION_TABLE
9381	rcStrict = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
9382	#else
9383	rcStrict = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
9384	#endif
9385	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
9386	if (rcStrict == VINF_SUCCESS)
9387	{
9388	if (cLoops <= pVM->hm.s.cMaxResumeLoops)
9389	continue; /* likely */
9390	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
9391	rcStrict = VINF_EM_RAW_INTERRUPT;
9392	}
9393	break;
9394	}
9395
9396	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
9397	return rcStrict;
9398	}
9399
9400
9401
9402	/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
9403	* probes.
9404	*
9405	* The following few functions and associated structure contains the bloat
9406	* necessary for providing detailed debug events and dtrace probes as well as
9407	* reliable host side single stepping. This works on the principle of
9408	* "subclassing" the normal execution loop and workers. We replace the loop
9409	* method completely and override selected helpers to add necessary adjustments
9410	* to their core operation.
9411	*
9412	* The goal is to keep the "parent" code lean and mean, so as not to sacrifice
9413	* any performance for debug and analysis features.
9414	*
9415	* @{
9416	*/
9417
9418	/**
9419	* Transient per-VCPU debug state of VMCS and related info. we save/restore in
9420	* the debug run loop.
9421	*/
9422	typedef struct VMXRUNDBGSTATE
9423	{
9424	/** The RIP we started executing at. This is for detecting that we stepped. */
9425	uint64_t uRipStart;
9426	/** The CS we started executing with. */
9427	uint16_t uCsStart;
9428
9429	/** Whether we've actually modified the 1st execution control field. */
9430	bool fModifiedProcCtls : 1;
9431	/** Whether we've actually modified the 2nd execution control field. */
9432	bool fModifiedProcCtls2 : 1;
9433	/** Whether we've actually modified the exception bitmap. */
9434	bool fModifiedXcptBitmap : 1;
9435
9436	/** We desire the modified the CR0 mask to be cleared. */
9437	bool fClearCr0Mask : 1;
9438	/** We desire the modified the CR4 mask to be cleared. */
9439	bool fClearCr4Mask : 1;
9440	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
9441	uint32_t fCpe1Extra;
9442	/** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
9443	uint32_t fCpe1Unwanted;
9444	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
9445	uint32_t fCpe2Extra;
9446	/** Extra stuff we need in VMX_VMCS32_CTRL_EXCEPTION_BITMAP. */
9447	uint32_t bmXcptExtra;
9448	/** The sequence number of the Dtrace provider settings the state was
9449	* configured against. */
9450	uint32_t uDtraceSettingsSeqNo;
9451	/** VM-exits to check (one bit per VM-exit). */
9452	uint32_t bmExitsToCheck[3];
9453
9454	/** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
9455	uint32_t fProcCtlsInitial;
9456	/** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
9457	uint32_t fProcCtls2Initial;
9458	/** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
9459	uint32_t bmXcptInitial;
9460	} VMXRUNDBGSTATE;
9461	AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
9462	typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
9463
9464
9465	/**
9466	* Initializes the VMXRUNDBGSTATE structure.
9467	*
9468	* @param pVCpu The cross context virtual CPU structure of the
9469	* calling EMT.
9470	* @param pCtx The CPU register context to go with @a pVCpu.
9471	* @param pDbgState The structure to initialize.
9472	*/
9473	DECLINLINE(void) hmR0VmxRunDebugStateInit(PVMCPU pVCpu, PCCPUMCTX pCtx, PVMXRUNDBGSTATE pDbgState)
9474	{
9475	pDbgState->uRipStart = pCtx->rip;
9476	pDbgState->uCsStart = pCtx->cs.Sel;
9477
9478	pDbgState->fModifiedProcCtls = false;
9479	pDbgState->fModifiedProcCtls2 = false;
9480	pDbgState->fModifiedXcptBitmap = false;
9481	pDbgState->fClearCr0Mask = false;
9482	pDbgState->fClearCr4Mask = false;
9483	pDbgState->fCpe1Extra = 0;
9484	pDbgState->fCpe1Unwanted = 0;
9485	pDbgState->fCpe2Extra = 0;
9486	pDbgState->bmXcptExtra = 0;
9487	pDbgState->fProcCtlsInitial = pVCpu->hm.s.vmx.u32ProcCtls;
9488	pDbgState->fProcCtls2Initial = pVCpu->hm.s.vmx.u32ProcCtls2;
9489	pDbgState->bmXcptInitial = pVCpu->hm.s.vmx.u32XcptBitmap;
9490	}
9491
9492
9493	/**
9494	* Updates the VMSC fields with changes requested by @a pDbgState.
9495	*
9496	* This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
9497	* immediately before executing guest code, i.e. when interrupts are disabled.
9498	* We don't check status codes here as we cannot easily assert or return in the
9499	* latter case.
9500	*
9501	* @param pVCpu The cross context virtual CPU structure.
9502	* @param pDbgState The debug state.
9503	*/
9504	DECLINLINE(void) hmR0VmxPreRunGuestDebugStateApply(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState)
9505	{
9506	/*
9507	* Ensure desired flags in VMCS control fields are set.
9508	* (Ignoring write failure here, as we're committed and it's just debug extras.)
9509	*
9510	* Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
9511	* there should be no stale data in pCtx at this point.
9512	*/
9513	if ( (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
9514	\|\| (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Unwanted))
9515	{
9516	pVCpu->hm.s.vmx.u32ProcCtls \|= pDbgState->fCpe1Extra;
9517	pVCpu->hm.s.vmx.u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
9518	VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
9519	Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls));
9520	pDbgState->fModifiedProcCtls = true;
9521	}
9522
9523	if ((pVCpu->hm.s.vmx.u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
9524	{
9525	pVCpu->hm.s.vmx.u32ProcCtls2 \|= pDbgState->fCpe2Extra;
9526	VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pVCpu->hm.s.vmx.u32ProcCtls2);
9527	Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls2));
9528	pDbgState->fModifiedProcCtls2 = true;
9529	}
9530
9531	if ((pVCpu->hm.s.vmx.u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
9532	{
9533	pVCpu->hm.s.vmx.u32XcptBitmap \|= pDbgState->bmXcptExtra;
9534	VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
9535	Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVCpu->hm.s.vmx.u32XcptBitmap));
9536	pDbgState->fModifiedXcptBitmap = true;
9537	}
9538
9539	if (pDbgState->fClearCr0Mask && pVCpu->hm.s.vmx.u32CR0Mask != 0)
9540	{
9541	pVCpu->hm.s.vmx.u32CR0Mask = 0;
9542	VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, 0);
9543	Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR0_MASK: 0\n"));
9544	}
9545
9546	if (pDbgState->fClearCr4Mask && pVCpu->hm.s.vmx.u32CR4Mask != 0)
9547	{
9548	pVCpu->hm.s.vmx.u32CR4Mask = 0;
9549	VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, 0);
9550	Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR4_MASK: 0\n"));
9551	}
9552	}
9553
9554
9555	DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugStateRevert(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState, VBOXSTRICTRC rcStrict)
9556	{
9557	/*
9558	* Restore VM-exit control settings as we may not reenter this function the
9559	* next time around.
9560	*/
9561	/* We reload the initial value, trigger what we can of recalculations the
9562	next time around. From the looks of things, that's all that's required atm. */
9563	if (pDbgState->fModifiedProcCtls)
9564	{
9565	if (!(pDbgState->fProcCtlsInitial & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
9566	pDbgState->fProcCtlsInitial \|= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
9567	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
9568	AssertRCReturn(rc2, rc2);
9569	pVCpu->hm.s.vmx.u32ProcCtls = pDbgState->fProcCtlsInitial;
9570	}
9571
9572	/* We're currently the only ones messing with this one, so just restore the
9573	cached value and reload the field. */
9574	if ( pDbgState->fModifiedProcCtls2
9575	&& pVCpu->hm.s.vmx.u32ProcCtls2 != pDbgState->fProcCtls2Initial)
9576	{
9577	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
9578	AssertRCReturn(rc2, rc2);
9579	pVCpu->hm.s.vmx.u32ProcCtls2 = pDbgState->fProcCtls2Initial;
9580	}
9581
9582	/* If we've modified the exception bitmap, we restore it and trigger
9583	reloading and partial recalculation the next time around. */
9584	if (pDbgState->fModifiedXcptBitmap)
9585	pVCpu->hm.s.vmx.u32XcptBitmap = pDbgState->bmXcptInitial;
9586
9587	return rcStrict;
9588	}
9589
9590
9591	/**
9592	* Configures VM-exit controls for current DBGF and DTrace settings.
9593	*
9594	* This updates @a pDbgState and the VMCS execution control fields to reflect
9595	* the necessary VM-exits demanded by DBGF and DTrace.
9596	*
9597	* @param pVM The cross context VM structure.
9598	* @param pVCpu The cross context virtual CPU structure.
9599	* @param pCtx Pointer to the guest-CPU context.
9600	* @param pDbgState The debug state.
9601	* @param pVmxTransient Pointer to the VMX transient structure. May update
9602	* fUpdateTscOffsettingAndPreemptTimer.
9603	*/
9604	static void hmR0VmxPreRunGuestDebugStateUpdate(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx,
9605	PVMXRUNDBGSTATE pDbgState, PVMXTRANSIENT pVmxTransient)
9606	{
9607	/*
9608	* Take down the dtrace serial number so we can spot changes.
9609	*/
9610	pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
9611	ASMCompilerBarrier();
9612
9613	/*
9614	* We'll rebuild most of the middle block of data members (holding the
9615	* current settings) as we go along here, so start by clearing it all.
9616	*/
9617	pDbgState->bmXcptExtra = 0;
9618	pDbgState->fCpe1Extra = 0;
9619	pDbgState->fCpe1Unwanted = 0;
9620	pDbgState->fCpe2Extra = 0;
9621	for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
9622	pDbgState->bmExitsToCheck[i] = 0;
9623
9624	/*
9625	* Software interrupts (INT XXh) - no idea how to trigger these...
9626	*/
9627	if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
9628	\|\| VBOXVMM_INT_SOFTWARE_ENABLED())
9629	{
9630	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9631	}
9632
9633	/*
9634	* INT3 breakpoints - triggered by #BP exceptions.
9635	*/
9636	if (pVM->dbgf.ro.cEnabledInt3Breakpoints > 0)
9637	pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
9638
9639	/*
9640	* Exception bitmap and XCPT events+probes.
9641	*/
9642	for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
9643	if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
9644	pDbgState->bmXcptExtra \|= RT_BIT_32(iXcpt);
9645
9646	if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DE);
9647	if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DB);
9648	if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
9649	if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_OF);
9650	if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BR);
9651	if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_UD);
9652	if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NM);
9653	if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DF);
9654	if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_TS);
9655	if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NP);
9656	if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SS);
9657	if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_GP);
9658	if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_PF);
9659	if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_MF);
9660	if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_AC);
9661	if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_XF);
9662	if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_VE);
9663	if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SX);
9664
9665	if (pDbgState->bmXcptExtra)
9666	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9667
9668	/*
9669	* Process events and probes for VM-exits, making sure we get the wanted VM-exits.
9670	*
9671	* Note! This is the reverse of waft hmR0VmxHandleExitDtraceEvents does.
9672	* So, when adding/changing/removing please don't forget to update it.
9673	*
9674	* Some of the macros are picking up local variables to save horizontal space,
9675	* (being able to see it in a table is the lesser evil here).
9676	*/
9677	#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
9678	( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
9679	\|\| RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
9680	#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
9681	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9682	{ AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9683	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9684	} else do { } while (0)
9685	#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
9686	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9687	{ \
9688	(pDbgState)->fCpe1Extra \|= (a_fCtrlProcExec); \
9689	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9690	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9691	} else do { } while (0)
9692	#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
9693	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9694	{ \
9695	(pDbgState)->fCpe1Unwanted \|= (a_fUnwantedCtrlProcExec); \
9696	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9697	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9698	} else do { } while (0)
9699	#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
9700	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9701	{ \
9702	(pDbgState)->fCpe2Extra \|= (a_fCtrlProcExec2); \
9703	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9704	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9705	} else do { } while (0)
9706
9707	SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
9708	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
9709	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
9710	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
9711	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
9712
9713	SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
9714	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
9715	SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
9716	SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
9717	SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT); /* paranoia */
9718	SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
9719	SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
9720	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
9721	SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9722	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
9723	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT);
9724	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
9725	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9726	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
9727	SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
9728	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
9729	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
9730	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
9731	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
9732	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
9733	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
9734	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
9735	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
9736	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
9737	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
9738	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
9739	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
9740	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
9741	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
9742	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
9743	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
9744	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
9745	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
9746	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
9747	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
9748	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
9749
9750	if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
9751	\|\| IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9752	{
9753	int rc2 = hmR0VmxSaveGuestCR0(pVCpu, pCtx);
9754	rc2 \|= hmR0VmxSaveGuestCR4(pVCpu, pCtx);
9755	rc2 \|= hmR0VmxSaveGuestApicState(pVCpu, pCtx);
9756	AssertRC(rc2);
9757
9758	#if 0 /** @todo fix me */
9759	pDbgState->fClearCr0Mask = true;
9760	pDbgState->fClearCr4Mask = true;
9761	#endif
9762	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
9763	pDbgState->fCpe1Extra \|= VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT \| VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT;
9764	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9765	pDbgState->fCpe1Extra \|= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT \| VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT;
9766	pDbgState->fCpe1Unwanted \|= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* risky? */
9767	/* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
9768	require clearing here and in the loop if we start using it. */
9769	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
9770	}
9771	else
9772	{
9773	if (pDbgState->fClearCr0Mask)
9774	{
9775	pDbgState->fClearCr0Mask = false;
9776	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9777	}
9778	if (pDbgState->fClearCr4Mask)
9779	{
9780	pDbgState->fClearCr4Mask = false;
9781	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
9782	}
9783	}
9784	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
9785	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
9786
9787	if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
9788	\|\| IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
9789	{
9790	/** @todo later, need to fix handler as it assumes this won't usually happen. */
9791	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
9792	}
9793	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
9794	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
9795
9796	SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS); /* risky clearing this? */
9797	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
9798	SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS);
9799	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
9800	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT); /* paranoia */
9801	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
9802	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT); /* paranoia */
9803	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
9804	#if 0 /** @todo too slow, fix handler. */
9805	SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT);
9806	#endif
9807	SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
9808
9809	if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
9810	\|\| IS_EITHER_ENABLED(pVM, INSTR_SIDT)
9811	\|\| IS_EITHER_ENABLED(pVM, INSTR_LGDT)
9812	\|\| IS_EITHER_ENABLED(pVM, INSTR_LIDT))
9813	{
9814	pDbgState->fCpe2Extra \|= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9815	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XDTR_ACCESS);
9816	}
9817	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_XDTR_ACCESS);
9818	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_XDTR_ACCESS);
9819	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_XDTR_ACCESS);
9820	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_XDTR_ACCESS);
9821
9822	if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
9823	\|\| IS_EITHER_ENABLED(pVM, INSTR_STR)
9824	\|\| IS_EITHER_ENABLED(pVM, INSTR_LLDT)
9825	\|\| IS_EITHER_ENABLED(pVM, INSTR_LTR))
9826	{
9827	pDbgState->fCpe2Extra \|= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9828	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_TR_ACCESS);
9829	}
9830	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_TR_ACCESS);
9831	SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_TR_ACCESS);
9832	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_TR_ACCESS);
9833	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_TR_ACCESS);
9834
9835	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
9836	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
9837	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9838	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
9839	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
9840	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
9841	SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT);
9842	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
9843	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
9844	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
9845	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT);
9846	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
9847	SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9848	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
9849	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
9850	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
9851	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_VMCS_CTRL_PROC_EXEC2_RDSEED_EXIT);
9852	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
9853	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
9854	SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
9855	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
9856	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
9857
9858	#undef IS_EITHER_ENABLED
9859	#undef SET_ONLY_XBM_IF_EITHER_EN
9860	#undef SET_CPE1_XBM_IF_EITHER_EN
9861	#undef SET_CPEU_XBM_IF_EITHER_EN
9862	#undef SET_CPE2_XBM_IF_EITHER_EN
9863
9864	/*
9865	* Sanitize the control stuff.
9866	*/
9867	pDbgState->fCpe2Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1;
9868	if (pDbgState->fCpe2Extra)
9869	pDbgState->fCpe1Extra \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
9870	pDbgState->fCpe1Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1;
9871	pDbgState->fCpe1Unwanted &= ~pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0;
9872	if (pVCpu->hm.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
9873	{
9874	pVCpu->hm.s.fDebugWantRdTscExit ^= true;
9875	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9876	}
9877
9878	Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
9879	pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
9880	pDbgState->fClearCr0Mask ? " clr-cr0" : "",
9881	pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
9882	}
9883
9884
9885	/**
9886	* Fires off DBGF events and dtrace probes for a VM-exit, when it's
9887	* appropriate.
9888	*
9889	* The caller has checked the VM-exit against the
9890	* VMXRUNDBGSTATE::bmExitsToCheck bitmap. The caller has checked for NMIs
9891	* already, so we don't have to do that either.
9892	*
9893	* @returns Strict VBox status code (i.e. informational status codes too).
9894	* @param pVM The cross context VM structure.
9895	* @param pVCpu The cross context virtual CPU structure.
9896	* @param pMixedCtx Pointer to the guest-CPU context.
9897	* @param pVmxTransient Pointer to the VMX-transient structure.
9898	* @param uExitReason The VM-exit reason.
9899	*
9900	* @remarks The name of this function is displayed by dtrace, so keep it short
9901	* and to the point. No longer than 33 chars long, please.
9902	*/
9903	static VBOXSTRICTRC hmR0VmxHandleExitDtraceEvents(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx,
9904	PVMXTRANSIENT pVmxTransient, uint32_t uExitReason)
9905	{
9906	/*
9907	* Translate the event into a DBGF event (enmEvent + uEventArg) and at the
9908	* same time check whether any corresponding Dtrace event is enabled (fDtrace).
9909	*
9910	* Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
9911	* does. Must add/change/remove both places. Same ordering, please.
9912	*
9913	* Added/removed events must also be reflected in the next section
9914	* where we dispatch dtrace events.
9915	*/
9916	bool fDtrace1 = false;
9917	bool fDtrace2 = false;
9918	DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
9919	DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
9920	uint32_t uEventArg = 0;
9921	#define SET_EXIT(a_EventSubName) \
9922	do { \
9923	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9924	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9925	} while (0)
9926	#define SET_BOTH(a_EventSubName) \
9927	do { \
9928	enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
9929	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9930	fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
9931	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9932	} while (0)
9933	switch (uExitReason)
9934	{
9935	case VMX_EXIT_MTF:
9936	return hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient);
9937
9938	case VMX_EXIT_XCPT_OR_NMI:
9939	{
9940	uint8_t const idxVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
9941	switch (VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo))
9942	{
9943	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
9944	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT:
9945	case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT:
9946	if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
9947	{
9948	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uExitIntInfo))
9949	{
9950	hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
9951	uEventArg = pVmxTransient->uExitIntErrorCode;
9952	}
9953	enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
9954	switch (enmEvent1)
9955	{
9956	case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
9957	case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
9958	case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
9959	case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
9960	case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
9961	case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
9962	case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
9963	case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
9964	case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
9965	case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
9966	case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
9967	case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
9968	case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
9969	case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
9970	case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
9971	case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
9972	case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
9973	case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
9974	default: break;
9975	}
9976	}
9977	else
9978	AssertFailed();
9979	break;
9980
9981	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT:
9982	uEventArg = idxVector;
9983	enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
9984	fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
9985	break;
9986	}
9987	break;
9988	}
9989
9990	case VMX_EXIT_TRIPLE_FAULT:
9991	enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
9992	//fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
9993	break;
9994	case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
9995	case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
9996	case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
9997	case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
9998	case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
9999
10000	/* Instruction specific VM-exits: */
10001	case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
10002	case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
10003	case VMX_EXIT_HLT: SET_BOTH(HALT); break;
10004	case VMX_EXIT_INVD: SET_BOTH(INVD); break;
10005	case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
10006	case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
10007	case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
10008	case VMX_EXIT_RSM: SET_BOTH(RSM); break;
10009	case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
10010	case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
10011	case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
10012	case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
10013	case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
10014	case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
10015	case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
10016	case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
10017	case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
10018	case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
10019	case VMX_EXIT_MOV_CRX:
10020	hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10021	/** @todo r=bird: I feel these macros aren't very descriptive and needs to be at least 30 chars longer! ;-)
10022	* Sensible abbreviations strongly recommended here because even with 130 columns this stuff get too wide! */
10023	if ( VMX_EXIT_QUALIFICATION_CRX_ACCESS(pVmxTransient->uExitQualification)
10024	== VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ)
10025	SET_BOTH(CRX_READ);
10026	else
10027	SET_BOTH(CRX_WRITE);
10028	uEventArg = VMX_EXIT_QUALIFICATION_CRX_REGISTER(pVmxTransient->uExitQualification);
10029	break;
10030	case VMX_EXIT_MOV_DRX:
10031	hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10032	if ( VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification)
10033	== VMX_EXIT_QUALIFICATION_DRX_DIRECTION_READ)
10034	SET_BOTH(DRX_READ);
10035	else
10036	SET_BOTH(DRX_WRITE);
10037	uEventArg = VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification);
10038	break;
10039	case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
10040	case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
10041	case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
10042	case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
10043	case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
10044	case VMX_EXIT_XDTR_ACCESS:
10045	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
10046	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_XDTR_INSINFO_INSTR_ID))
10047	{
10048	case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
10049	case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
10050	case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
10051	case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
10052	}
10053	break;
10054
10055	case VMX_EXIT_TR_ACCESS:
10056	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
10057	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_YYTR_INSINFO_INSTR_ID))
10058	{
10059	case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
10060	case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
10061	case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
10062	case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
10063	}
10064	break;
10065
10066	case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
10067	case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
10068	case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
10069	case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
10070	case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
10071	case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
10072	case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
10073	case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
10074	case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
10075	case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
10076	case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
10077
10078	/* Events that aren't relevant at this point. */
10079	case VMX_EXIT_EXT_INT:
10080	case VMX_EXIT_INT_WINDOW:
10081	case VMX_EXIT_NMI_WINDOW:
10082	case VMX_EXIT_TPR_BELOW_THRESHOLD:
10083	case VMX_EXIT_PREEMPT_TIMER:
10084	case VMX_EXIT_IO_INSTR:
10085	break;
10086
10087	/* Errors and unexpected events. */
10088	case VMX_EXIT_INIT_SIGNAL:
10089	case VMX_EXIT_SIPI:
10090	case VMX_EXIT_IO_SMI:
10091	case VMX_EXIT_SMI:
10092	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
10093	case VMX_EXIT_ERR_MSR_LOAD:
10094	case VMX_EXIT_ERR_MACHINE_CHECK:
10095	break;
10096
10097	default:
10098	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
10099	break;
10100	}
10101	#undef SET_BOTH
10102	#undef SET_EXIT
10103
10104	/*
10105	* Dtrace tracepoints go first. We do them here at once so we don't
10106	* have to copy the guest state saving and stuff a few dozen times.
10107	* Down side is that we've got to repeat the switch, though this time
10108	* we use enmEvent since the probes are a subset of what DBGF does.
10109	*/
10110	if (fDtrace1 \|\| fDtrace2)
10111	{
10112	hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10113	hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10114	switch (enmEvent1)
10115	{
10116	/** @todo consider which extra parameters would be helpful for each probe. */
10117	case DBGFEVENT_END: break;
10118	case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pMixedCtx); break;
10119	case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pMixedCtx, pMixedCtx->dr[6]); break;
10120	case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pMixedCtx); break;
10121	case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pMixedCtx); break;
10122	case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pMixedCtx); break;
10123	case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pMixedCtx); break;
10124	case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pMixedCtx); break;
10125	case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pMixedCtx); break;
10126	case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pMixedCtx, uEventArg); break;
10127	case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pMixedCtx, uEventArg); break;
10128	case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pMixedCtx, uEventArg); break;
10129	case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pMixedCtx, uEventArg); break;
10130	case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pMixedCtx, uEventArg, pMixedCtx->cr2); break;
10131	case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pMixedCtx); break;
10132	case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pMixedCtx); break;
10133	case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pMixedCtx); break;
10134	case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pMixedCtx); break;
10135	case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pMixedCtx, uEventArg); break;
10136	case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10137	case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pMixedCtx, pMixedCtx->eax, pMixedCtx->ecx); break;
10138	case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pMixedCtx); break;
10139	case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pMixedCtx); break;
10140	case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pMixedCtx); break;
10141	case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pMixedCtx); break;
10142	case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pMixedCtx); break;
10143	case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pMixedCtx); break;
10144	case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pMixedCtx); break;
10145	case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10146	case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10147	case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10148	case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10149	case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pMixedCtx, pMixedCtx->ecx); break;
10150	case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pMixedCtx, pMixedCtx->ecx,
10151	RT_MAKE_U64(pMixedCtx->eax, pMixedCtx->edx)); break;
10152	case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pMixedCtx); break;
10153	case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pMixedCtx); break;
10154	case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pMixedCtx); break;
10155	case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pMixedCtx); break;
10156	case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pMixedCtx); break;
10157	case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pMixedCtx); break;
10158	case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pMixedCtx); break;
10159	case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pMixedCtx); break;
10160	case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pMixedCtx); break;
10161	case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pMixedCtx); break;
10162	case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pMixedCtx); break;
10163	case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pMixedCtx); break;
10164	case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pMixedCtx); break;
10165	case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pMixedCtx); break;
10166	case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pMixedCtx); break;
10167	case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pMixedCtx); break;
10168	case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pMixedCtx); break;
10169	case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pMixedCtx); break;
10170	case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pMixedCtx); break;
10171	case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pMixedCtx); break;
10172	case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pMixedCtx); break;
10173	case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pMixedCtx); break;
10174	case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pMixedCtx); break;
10175	case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pMixedCtx); break;
10176	case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pMixedCtx); break;
10177	case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pMixedCtx); break;
10178	case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pMixedCtx); break;
10179	case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pMixedCtx); break;
10180	case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pMixedCtx); break;
10181	case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pMixedCtx); break;
10182	case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pMixedCtx); break;
10183	case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pMixedCtx); break;
10184	default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
10185	}
10186	switch (enmEvent2)
10187	{
10188	/** @todo consider which extra parameters would be helpful for each probe. */
10189	case DBGFEVENT_END: break;
10190	case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pMixedCtx); break;
10191	case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pMixedCtx, pMixedCtx->eax, pMixedCtx->ecx); break;
10192	case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pMixedCtx); break;
10193	case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pMixedCtx); break;
10194	case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pMixedCtx); break;
10195	case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pMixedCtx); break;
10196	case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pMixedCtx); break;
10197	case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pMixedCtx); break;
10198	case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pMixedCtx); break;
10199	case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10200	case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10201	case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10202	case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10203	case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pMixedCtx, pMixedCtx->ecx); break;
10204	case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pMixedCtx, pMixedCtx->ecx,
10205	RT_MAKE_U64(pMixedCtx->eax, pMixedCtx->edx)); break;
10206	case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pMixedCtx); break;
10207	case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pMixedCtx); break;
10208	case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pMixedCtx); break;
10209	case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pMixedCtx); break;
10210	case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pMixedCtx); break;
10211	case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pMixedCtx); break;
10212	case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pMixedCtx); break;
10213	case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pMixedCtx); break;
10214	case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pMixedCtx); break;
10215	case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pMixedCtx); break;
10216	case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pMixedCtx); break;
10217	case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pMixedCtx); break;
10218	case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pMixedCtx); break;
10219	case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pMixedCtx); break;
10220	case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pMixedCtx); break;
10221	case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pMixedCtx); break;
10222	case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pMixedCtx); break;
10223	case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pMixedCtx); break;
10224	case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pMixedCtx); break;
10225	case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pMixedCtx); break;
10226	case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pMixedCtx); break;
10227	case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pMixedCtx); break;
10228	case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pMixedCtx); break;
10229	case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pMixedCtx); break;
10230	case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pMixedCtx); break;
10231	case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pMixedCtx); break;
10232	case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pMixedCtx); break;
10233	case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pMixedCtx); break;
10234	case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pMixedCtx); break;
10235	case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pMixedCtx); break;
10236	case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pMixedCtx); break;
10237	case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pMixedCtx); break;
10238	case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pMixedCtx); break;
10239	case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pMixedCtx); break;
10240	case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pMixedCtx); break;
10241	case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pMixedCtx); break;
10242	default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
10243	}
10244	}
10245
10246	/*
10247	* Fire of the DBGF event, if enabled (our check here is just a quick one,
10248	* the DBGF call will do a full check).
10249	*
10250	* Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
10251	* Note! If we have to events, we prioritize the first, i.e. the instruction
10252	* one, in order to avoid event nesting.
10253	*/
10254	if ( enmEvent1 != DBGFEVENT_END
10255	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
10256	{
10257	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent1, uEventArg, DBGFEVENTCTX_HM);
10258	if (rcStrict != VINF_SUCCESS)
10259	return rcStrict;
10260	}
10261	else if ( enmEvent2 != DBGFEVENT_END
10262	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
10263	{
10264	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent2, uEventArg, DBGFEVENTCTX_HM);
10265	if (rcStrict != VINF_SUCCESS)
10266	return rcStrict;
10267	}
10268
10269	return VINF_SUCCESS;
10270	}
10271
10272
10273	/**
10274	* Single-stepping VM-exit filtering.
10275	*
10276	* This is preprocessing the VM-exits and deciding whether we've gotten far
10277	* enough to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit
10278	* handling is performed.
10279	*
10280	* @returns Strict VBox status code (i.e. informational status codes too).
10281	* @param pVM The cross context VM structure.
10282	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
10283	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
10284	* out-of-sync. Make sure to update the required
10285	* fields before using them.
10286	* @param pVmxTransient Pointer to the VMX-transient structure.
10287	* @param uExitReason The VM-exit reason.
10288	* @param pDbgState The debug state.
10289	*/
10290	DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugHandleExit(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient,
10291	uint32_t uExitReason, PVMXRUNDBGSTATE pDbgState)
10292	{
10293	/*
10294	* Expensive (saves context) generic dtrace VM-exit probe.
10295	*/
10296	if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
10297	{ /* more likely */ }
10298	else
10299	{
10300	hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10301	hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10302	VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, pMixedCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQualification);
10303	}
10304
10305	/*
10306	* Check for host NMI, just to get that out of the way.
10307	*/
10308	if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
10309	{ /* normally likely */ }
10310	else
10311	{
10312	int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
10313	AssertRCReturn(rc2, rc2);
10314	uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
10315	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
10316	return hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient);
10317	}
10318
10319	/*
10320	* Check for single stepping event if we're stepping.
10321	*/
10322	if (pVCpu->hm.s.fSingleInstruction)
10323	{
10324	switch (uExitReason)
10325	{
10326	case VMX_EXIT_MTF:
10327	return hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient);
10328
10329	/* Various events: */
10330	case VMX_EXIT_XCPT_OR_NMI:
10331	case VMX_EXIT_EXT_INT:
10332	case VMX_EXIT_TRIPLE_FAULT:
10333	case VMX_EXIT_INT_WINDOW:
10334	case VMX_EXIT_NMI_WINDOW:
10335	case VMX_EXIT_TASK_SWITCH:
10336	case VMX_EXIT_TPR_BELOW_THRESHOLD:
10337	case VMX_EXIT_APIC_ACCESS:
10338	case VMX_EXIT_EPT_VIOLATION:
10339	case VMX_EXIT_EPT_MISCONFIG:
10340	case VMX_EXIT_PREEMPT_TIMER:
10341
10342	/* Instruction specific VM-exits: */
10343	case VMX_EXIT_CPUID:
10344	case VMX_EXIT_GETSEC:
10345	case VMX_EXIT_HLT:
10346	case VMX_EXIT_INVD:
10347	case VMX_EXIT_INVLPG:
10348	case VMX_EXIT_RDPMC:
10349	case VMX_EXIT_RDTSC:
10350	case VMX_EXIT_RSM:
10351	case VMX_EXIT_VMCALL:
10352	case VMX_EXIT_VMCLEAR:
10353	case VMX_EXIT_VMLAUNCH:
10354	case VMX_EXIT_VMPTRLD:
10355	case VMX_EXIT_VMPTRST:
10356	case VMX_EXIT_VMREAD:
10357	case VMX_EXIT_VMRESUME:
10358	case VMX_EXIT_VMWRITE:
10359	case VMX_EXIT_VMXOFF:
10360	case VMX_EXIT_VMXON:
10361	case VMX_EXIT_MOV_CRX:
10362	case VMX_EXIT_MOV_DRX:
10363	case VMX_EXIT_IO_INSTR:
10364	case VMX_EXIT_RDMSR:
10365	case VMX_EXIT_WRMSR:
10366	case VMX_EXIT_MWAIT:
10367	case VMX_EXIT_MONITOR:
10368	case VMX_EXIT_PAUSE:
10369	case VMX_EXIT_XDTR_ACCESS:
10370	case VMX_EXIT_TR_ACCESS:
10371	case VMX_EXIT_INVEPT:
10372	case VMX_EXIT_RDTSCP:
10373	case VMX_EXIT_INVVPID:
10374	case VMX_EXIT_WBINVD:
10375	case VMX_EXIT_XSETBV:
10376	case VMX_EXIT_RDRAND:
10377	case VMX_EXIT_INVPCID:
10378	case VMX_EXIT_VMFUNC:
10379	case VMX_EXIT_RDSEED:
10380	case VMX_EXIT_XSAVES:
10381	case VMX_EXIT_XRSTORS:
10382	{
10383	int rc2 = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10384	rc2 \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10385	AssertRCReturn(rc2, rc2);
10386	if ( pMixedCtx->rip != pDbgState->uRipStart
10387	\|\| pMixedCtx->cs.Sel != pDbgState->uCsStart)
10388	return VINF_EM_DBG_STEPPED;
10389	break;
10390	}
10391
10392	/* Errors and unexpected events: */
10393	case VMX_EXIT_INIT_SIGNAL:
10394	case VMX_EXIT_SIPI:
10395	case VMX_EXIT_IO_SMI:
10396	case VMX_EXIT_SMI:
10397	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
10398	case VMX_EXIT_ERR_MSR_LOAD:
10399	case VMX_EXIT_ERR_MACHINE_CHECK:
10400	case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
10401	break;
10402
10403	default:
10404	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
10405	break;
10406	}
10407	}
10408
10409	/*
10410	* Check for debugger event breakpoints and dtrace probes.
10411	*/
10412	if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
10413	&& ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
10414	{
10415	VBOXSTRICTRC rcStrict = hmR0VmxHandleExitDtraceEvents(pVM, pVCpu, pMixedCtx, pVmxTransient, uExitReason);
10416	if (rcStrict != VINF_SUCCESS)
10417	return rcStrict;
10418	}
10419
10420	/*
10421	* Normal processing.
10422	*/
10423	#ifdef HMVMX_USE_FUNCTION_TABLE
10424	return g_apfnVMExitHandlers[uExitReason](pVCpu, pMixedCtx, pVmxTransient);
10425	#else
10426	return hmR0VmxHandleExit(pVCpu, pMixedCtx, pVmxTransient, uExitReason);
10427	#endif
10428	}
10429
10430
10431	/**
10432	* Single steps guest code using VT-x.
10433	*
10434	* @returns Strict VBox status code (i.e. informational status codes too).
10435	* @param pVM The cross context VM structure.
10436	* @param pVCpu The cross context virtual CPU structure.
10437	* @param pCtx Pointer to the guest-CPU context.
10438	*
10439	* @note Mostly the same as hmR0VmxRunGuestCodeNormal().
10440	*/
10441	static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10442	{
10443	VMXTRANSIENT VmxTransient;
10444	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
10445
10446	/* Set HMCPU indicators. */
10447	bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
10448	pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction \|\| DBGFIsStepping(pVCpu);
10449	pVCpu->hm.s.fDebugWantRdTscExit = false;
10450	pVCpu->hm.s.fUsingDebugLoop = true;
10451
10452	/* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
10453	VMXRUNDBGSTATE DbgState;
10454	hmR0VmxRunDebugStateInit(pVCpu, pCtx, &DbgState);
10455	hmR0VmxPreRunGuestDebugStateUpdate(pVM, pVCpu, pCtx, &DbgState, &VmxTransient);
10456
10457	/*
10458	* The loop.
10459	*/
10460	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
10461	for (uint32_t cLoops = 0; ; cLoops++)
10462	{
10463	Assert(!HMR0SuspendPending());
10464	HMVMX_ASSERT_CPU_SAFE();
10465	bool fStepping = pVCpu->hm.s.fSingleInstruction;
10466
10467	/*
10468	* Preparatory work for running guest code, this may force us to return
10469	* to ring-3. This bugger disables interrupts on VINF_SUCCESS!
10470	*/
10471	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
10472	hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Set up execute controls the next to can respond to. */
10473	rcStrict = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, fStepping);
10474	if (rcStrict != VINF_SUCCESS)
10475	break;
10476
10477	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
10478	hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Override any obnoxious code in the above two calls. */
10479
10480	/*
10481	* Now we can run the guest code.
10482	*/
10483	int rcRun = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
10484
10485	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
10486
10487	/*
10488	* Restore any residual host-state and save any bits shared between host
10489	* and guest into the guest-CPU state. Re-enables interrupts!
10490	*/
10491	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rcRun);
10492
10493	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
10494	if (RT_SUCCESS(rcRun))
10495	{ /* very likely */ }
10496	else
10497	{
10498	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
10499	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rcRun, pCtx, &VmxTransient);
10500	return rcRun;
10501	}
10502
10503	/* Profile the VM-exit. */
10504	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
10505	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
10506	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
10507	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
10508	HMVMX_START_EXIT_DISPATCH_PROF();
10509
10510	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
10511
10512	/*
10513	* Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitDebug().
10514	*/
10515	rcStrict = hmR0VmxRunDebugHandleExit(pVM, pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason, &DbgState);
10516	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
10517	if (rcStrict != VINF_SUCCESS)
10518	break;
10519	if (cLoops > pVM->hm.s.cMaxResumeLoops)
10520	{
10521	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
10522	rcStrict = VINF_EM_RAW_INTERRUPT;
10523	break;
10524	}
10525
10526	/*
10527	* Stepping: Did the RIP change, if so, consider it a single step.
10528	* Otherwise, make sure one of the TFs gets set.
10529	*/
10530	if (fStepping)
10531	{
10532	int rc2 = hmR0VmxSaveGuestRip(pVCpu, pCtx);
10533	rc2 \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pCtx);
10534	AssertRCReturn(rc2, rc2);
10535	if ( pCtx->rip != DbgState.uRipStart
10536	\|\| pCtx->cs.Sel != DbgState.uCsStart)
10537	{
10538	rcStrict = VINF_EM_DBG_STEPPED;
10539	break;
10540	}
10541	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
10542	}
10543
10544	/*
10545	* Update when dtrace settings changes (DBGF kicks us, so no need to check).
10546	*/
10547	if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
10548	hmR0VmxPreRunGuestDebugStateUpdate(pVM, pVCpu, pCtx, &DbgState, &VmxTransient);
10549	}
10550
10551	/*
10552	* Clear the X86_EFL_TF if necessary.
10553	*/
10554	if (pVCpu->hm.s.fClearTrapFlag)
10555	{
10556	int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pCtx);
10557	AssertRCReturn(rc2, rc2);
10558	pVCpu->hm.s.fClearTrapFlag = false;
10559	pCtx->eflags.Bits.u1TF = 0;
10560	}
10561	/** @todo there seems to be issues with the resume flag when the monitor trap
10562	* flag is pending without being used. Seen early in bios init when
10563	* accessing APIC page in protected mode. */
10564
10565	/*
10566	* Restore VM-exit control settings as we may not reenter this function the
10567	* next time around.
10568	*/
10569	rcStrict = hmR0VmxRunDebugStateRevert(pVCpu, &DbgState, rcStrict);
10570
10571	/* Restore HMCPU indicators. */
10572	pVCpu->hm.s.fUsingDebugLoop = false;
10573	pVCpu->hm.s.fDebugWantRdTscExit = false;
10574	pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
10575
10576	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
10577	return rcStrict;
10578	}
10579
10580
10581	/** @} */
10582
10583
10584	/**
10585	* Checks if any expensive dtrace probes are enabled and we should go to the
10586	* debug loop.
10587	*
10588	* @returns true if we should use debug loop, false if not.
10589	*/
10590	static bool hmR0VmxAnyExpensiveProbesEnabled(void)
10591	{
10592	/* It's probably faster to OR the raw 32-bit counter variables together.
10593	Since the variables are in an array and the probes are next to one
10594	another (more or less), we have good locality. So, better read
10595	eight-nine cache lines ever time and only have one conditional, than
10596	128+ conditionals, right? */
10597	return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED_RAW() /* expensive too due to context */
10598	\| VBOXVMM_XCPT_DE_ENABLED_RAW()
10599	\| VBOXVMM_XCPT_DB_ENABLED_RAW()
10600	\| VBOXVMM_XCPT_BP_ENABLED_RAW()
10601	\| VBOXVMM_XCPT_OF_ENABLED_RAW()
10602	\| VBOXVMM_XCPT_BR_ENABLED_RAW()
10603	\| VBOXVMM_XCPT_UD_ENABLED_RAW()
10604	\| VBOXVMM_XCPT_NM_ENABLED_RAW()
10605	\| VBOXVMM_XCPT_DF_ENABLED_RAW()
10606	\| VBOXVMM_XCPT_TS_ENABLED_RAW()
10607	\| VBOXVMM_XCPT_NP_ENABLED_RAW()
10608	\| VBOXVMM_XCPT_SS_ENABLED_RAW()
10609	\| VBOXVMM_XCPT_GP_ENABLED_RAW()
10610	\| VBOXVMM_XCPT_PF_ENABLED_RAW()
10611	\| VBOXVMM_XCPT_MF_ENABLED_RAW()
10612	\| VBOXVMM_XCPT_AC_ENABLED_RAW()
10613	\| VBOXVMM_XCPT_XF_ENABLED_RAW()
10614	\| VBOXVMM_XCPT_VE_ENABLED_RAW()
10615	\| VBOXVMM_XCPT_SX_ENABLED_RAW()
10616	\| VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
10617	\| VBOXVMM_INT_HARDWARE_ENABLED_RAW()
10618	) != 0
10619	\|\| ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
10620	\| VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
10621	\| VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
10622	\| VBOXVMM_INSTR_CPUID_ENABLED_RAW()
10623	\| VBOXVMM_INSTR_INVD_ENABLED_RAW()
10624	\| VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
10625	\| VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
10626	\| VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
10627	\| VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
10628	\| VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
10629	\| VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
10630	\| VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
10631	\| VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
10632	\| VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
10633	\| VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
10634	\| VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
10635	\| VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
10636	\| VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
10637	\| VBOXVMM_INSTR_SIDT_ENABLED_RAW()
10638	\| VBOXVMM_INSTR_LIDT_ENABLED_RAW()
10639	\| VBOXVMM_INSTR_SGDT_ENABLED_RAW()
10640	\| VBOXVMM_INSTR_LGDT_ENABLED_RAW()
10641	\| VBOXVMM_INSTR_SLDT_ENABLED_RAW()
10642	\| VBOXVMM_INSTR_LLDT_ENABLED_RAW()
10643	\| VBOXVMM_INSTR_STR_ENABLED_RAW()
10644	\| VBOXVMM_INSTR_LTR_ENABLED_RAW()
10645	\| VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
10646	\| VBOXVMM_INSTR_RSM_ENABLED_RAW()
10647	\| VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
10648	\| VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
10649	\| VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
10650	\| VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
10651	\| VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
10652	\| VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED_RAW()
10653	\| VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED_RAW()
10654	\| VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED_RAW()
10655	\| VBOXVMM_INSTR_VMX_VMPTRST_ENABLED_RAW()
10656	\| VBOXVMM_INSTR_VMX_VMREAD_ENABLED_RAW()
10657	\| VBOXVMM_INSTR_VMX_VMRESUME_ENABLED_RAW()
10658	\| VBOXVMM_INSTR_VMX_VMWRITE_ENABLED_RAW()
10659	\| VBOXVMM_INSTR_VMX_VMXOFF_ENABLED_RAW()
10660	\| VBOXVMM_INSTR_VMX_VMXON_ENABLED_RAW()
10661	\| VBOXVMM_INSTR_VMX_VMFUNC_ENABLED_RAW()
10662	\| VBOXVMM_INSTR_VMX_INVEPT_ENABLED_RAW()
10663	\| VBOXVMM_INSTR_VMX_INVVPID_ENABLED_RAW()
10664	\| VBOXVMM_INSTR_VMX_INVPCID_ENABLED_RAW()
10665	) != 0
10666	\|\| ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
10667	\| VBOXVMM_EXIT_HALT_ENABLED_RAW()
10668	\| VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
10669	\| VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
10670	\| VBOXVMM_EXIT_CPUID_ENABLED_RAW()
10671	\| VBOXVMM_EXIT_INVD_ENABLED_RAW()
10672	\| VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
10673	\| VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
10674	\| VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
10675	\| VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
10676	\| VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
10677	\| VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
10678	\| VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
10679	\| VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
10680	\| VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
10681	\| VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
10682	\| VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
10683	\| VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
10684	\| VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
10685	\| VBOXVMM_EXIT_SIDT_ENABLED_RAW()
10686	\| VBOXVMM_EXIT_LIDT_ENABLED_RAW()
10687	\| VBOXVMM_EXIT_SGDT_ENABLED_RAW()
10688	\| VBOXVMM_EXIT_LGDT_ENABLED_RAW()
10689	\| VBOXVMM_EXIT_SLDT_ENABLED_RAW()
10690	\| VBOXVMM_EXIT_LLDT_ENABLED_RAW()
10691	\| VBOXVMM_EXIT_STR_ENABLED_RAW()
10692	\| VBOXVMM_EXIT_LTR_ENABLED_RAW()
10693	\| VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
10694	\| VBOXVMM_EXIT_RSM_ENABLED_RAW()
10695	\| VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
10696	\| VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
10697	\| VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
10698	\| VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
10699	\| VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
10700	\| VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED_RAW()
10701	\| VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED_RAW()
10702	\| VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED_RAW()
10703	\| VBOXVMM_EXIT_VMX_VMPTRST_ENABLED_RAW()
10704	\| VBOXVMM_EXIT_VMX_VMREAD_ENABLED_RAW()
10705	\| VBOXVMM_EXIT_VMX_VMRESUME_ENABLED_RAW()
10706	\| VBOXVMM_EXIT_VMX_VMWRITE_ENABLED_RAW()
10707	\| VBOXVMM_EXIT_VMX_VMXOFF_ENABLED_RAW()
10708	\| VBOXVMM_EXIT_VMX_VMXON_ENABLED_RAW()
10709	\| VBOXVMM_EXIT_VMX_VMFUNC_ENABLED_RAW()
10710	\| VBOXVMM_EXIT_VMX_INVEPT_ENABLED_RAW()
10711	\| VBOXVMM_EXIT_VMX_INVVPID_ENABLED_RAW()
10712	\| VBOXVMM_EXIT_VMX_INVPCID_ENABLED_RAW()
10713	\| VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED_RAW()
10714	\| VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED_RAW()
10715	\| VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED_RAW()
10716	\| VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED_RAW()
10717	) != 0;
10718	}
10719
10720
10721	/**
10722	* Runs the guest code using VT-x.
10723	*
10724	* @returns Strict VBox status code (i.e. informational status codes too).
10725	* @param pVM The cross context VM structure.
10726	* @param pVCpu The cross context virtual CPU structure.
10727	* @param pCtx Pointer to the guest-CPU context.
10728	*/
10729	VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10730	{
10731	Assert(VMMRZCallRing3IsEnabled(pVCpu));
10732	Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
10733	HMVMX_ASSERT_PREEMPT_SAFE();
10734
10735	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
10736
10737	VBOXSTRICTRC rcStrict;
10738	if ( !pVCpu->hm.s.fUseDebugLoop
10739	&& (!VBOXVMM_ANY_PROBES_ENABLED() \|\| !hmR0VmxAnyExpensiveProbesEnabled())
10740	&& !DBGFIsStepping(pVCpu)
10741	&& !pVM->dbgf.ro.cEnabledInt3Breakpoints)
10742	rcStrict = hmR0VmxRunGuestCodeNormal(pVM, pVCpu, pCtx);
10743	else
10744	rcStrict = hmR0VmxRunGuestCodeDebug(pVM, pVCpu, pCtx);
10745
10746	if (rcStrict == VERR_EM_INTERPRETER)
10747	rcStrict = VINF_EM_RAW_EMULATE_INSTR;
10748	else if (rcStrict == VINF_EM_RESET)
10749	rcStrict = VINF_EM_TRIPLE_FAULT;
10750
10751	int rc2 = hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rcStrict);
10752	if (RT_FAILURE(rc2))
10753	{
10754	pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
10755	rcStrict = rc2;
10756	}
10757	Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
10758	return rcStrict;
10759	}
10760
10761
10762	#ifndef HMVMX_USE_FUNCTION_TABLE
10763	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
10764	{
10765	# ifdef DEBUG_ramshankar
10766	# define RETURN_EXIT_CALL(a_CallExpr) \
10767	do { \
10768	int rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx); AssertRC(rc2); \
10769	VBOXSTRICTRC rcStrict = a_CallExpr; \
10770	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST); \
10771	return rcStrict; \
10772	} while (0)
10773	# else
10774	# define RETURN_EXIT_CALL(a_CallExpr) return a_CallExpr
10775	# endif
10776	switch (rcReason)
10777	{
10778	case VMX_EXIT_EPT_MISCONFIG: RETURN_EXIT_CALL(hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient));
10779	case VMX_EXIT_EPT_VIOLATION: RETURN_EXIT_CALL(hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient));
10780	case VMX_EXIT_IO_INSTR: RETURN_EXIT_CALL(hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient));
10781	case VMX_EXIT_CPUID: RETURN_EXIT_CALL(hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient));
10782	case VMX_EXIT_RDTSC: RETURN_EXIT_CALL(hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient));
10783	case VMX_EXIT_RDTSCP: RETURN_EXIT_CALL(hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient));
10784	case VMX_EXIT_APIC_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient));
10785	case VMX_EXIT_XCPT_OR_NMI: RETURN_EXIT_CALL(hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient));
10786	case VMX_EXIT_MOV_CRX: RETURN_EXIT_CALL(hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient));
10787	case VMX_EXIT_EXT_INT: RETURN_EXIT_CALL(hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient));
10788	case VMX_EXIT_INT_WINDOW: RETURN_EXIT_CALL(hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient));
10789	case VMX_EXIT_MWAIT: RETURN_EXIT_CALL(hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient));
10790	case VMX_EXIT_MONITOR: RETURN_EXIT_CALL(hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient));
10791	case VMX_EXIT_TASK_SWITCH: RETURN_EXIT_CALL(hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient));
10792	case VMX_EXIT_PREEMPT_TIMER: RETURN_EXIT_CALL(hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient));
10793	case VMX_EXIT_RDMSR: RETURN_EXIT_CALL(hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient));
10794	case VMX_EXIT_WRMSR: RETURN_EXIT_CALL(hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient));
10795	case VMX_EXIT_MOV_DRX: RETURN_EXIT_CALL(hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient));
10796	case VMX_EXIT_TPR_BELOW_THRESHOLD: RETURN_EXIT_CALL(hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient));
10797	case VMX_EXIT_HLT: RETURN_EXIT_CALL(hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient));
10798	case VMX_EXIT_INVD: RETURN_EXIT_CALL(hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient));
10799	case VMX_EXIT_INVLPG: RETURN_EXIT_CALL(hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient));
10800	case VMX_EXIT_RSM: RETURN_EXIT_CALL(hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient));
10801	case VMX_EXIT_MTF: RETURN_EXIT_CALL(hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient));
10802	case VMX_EXIT_PAUSE: RETURN_EXIT_CALL(hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient));
10803	case VMX_EXIT_XDTR_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient));
10804	case VMX_EXIT_TR_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient));
10805	case VMX_EXIT_WBINVD: RETURN_EXIT_CALL(hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient));
10806	case VMX_EXIT_XSETBV: RETURN_EXIT_CALL(hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient));
10807	case VMX_EXIT_RDRAND: RETURN_EXIT_CALL(hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient));
10808	case VMX_EXIT_INVPCID: RETURN_EXIT_CALL(hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient));
10809	case VMX_EXIT_GETSEC: RETURN_EXIT_CALL(hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient));
10810	case VMX_EXIT_RDPMC: RETURN_EXIT_CALL(hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient));
10811	case VMX_EXIT_VMCALL: RETURN_EXIT_CALL(hmR0VmxExitVmcall(pVCpu, pMixedCtx, pVmxTransient));
10812
10813	case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient);
10814	case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient);
10815	case VMX_EXIT_INIT_SIGNAL: return hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient);
10816	case VMX_EXIT_SIPI: return hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient);
10817	case VMX_EXIT_IO_SMI: return hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient);
10818	case VMX_EXIT_SMI: return hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient);
10819	case VMX_EXIT_ERR_MSR_LOAD: return hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient);
10820	case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient);
10821	case VMX_EXIT_ERR_MACHINE_CHECK: return hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient);
10822
10823	case VMX_EXIT_VMCLEAR:
10824	case VMX_EXIT_VMLAUNCH:
10825	case VMX_EXIT_VMPTRLD:
10826	case VMX_EXIT_VMPTRST:
10827	case VMX_EXIT_VMREAD:
10828	case VMX_EXIT_VMRESUME:
10829	case VMX_EXIT_VMWRITE:
10830	case VMX_EXIT_VMXOFF:
10831	case VMX_EXIT_VMXON:
10832	case VMX_EXIT_INVEPT:
10833	case VMX_EXIT_INVVPID:
10834	case VMX_EXIT_VMFUNC:
10835	case VMX_EXIT_XSAVES:
10836	case VMX_EXIT_XRSTORS:
10837	return hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
10838	case VMX_EXIT_ENCLS:
10839	case VMX_EXIT_RDSEED: /* only spurious VM-exits, so undefined */
10840	case VMX_EXIT_PML_FULL:
10841	default:
10842	return hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
10843	}
10844	#undef RETURN_EXIT_CALL
10845	}
10846	#endif /* !HMVMX_USE_FUNCTION_TABLE */
10847
10848
10849	#ifdef VBOX_STRICT
10850	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
10851	# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
10852	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
10853
10854	# define HMVMX_ASSERT_PREEMPT_CPUID() \
10855	do { \
10856	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
10857	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
10858	} while (0)
10859
10860	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
10861	do { \
10862	AssertPtr(pVCpu); \
10863	AssertPtr(pMixedCtx); \
10864	AssertPtr(pVmxTransient); \
10865	Assert(pVmxTransient->fVMEntryFailed == false); \
10866	Assert(ASMIntAreEnabled()); \
10867	HMVMX_ASSERT_PREEMPT_SAFE(); \
10868	HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
10869	Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
10870	HMVMX_ASSERT_PREEMPT_SAFE(); \
10871	if (VMMR0IsLogFlushDisabled(pVCpu)) \
10872	HMVMX_ASSERT_PREEMPT_CPUID(); \
10873	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10874	} while (0)
10875
10876	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
10877	do { \
10878	Log4Func(("\n")); \
10879	} while (0)
10880	#else /* nonstrict builds: */
10881	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
10882	do { \
10883	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10884	NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient); \
10885	} while (0)
10886	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while (0)
10887	#endif
10888
10889
10890	/**
10891	* Advances the guest RIP by the specified number of bytes.
10892	*
10893	* @param pVCpu The cross context virtual CPU structure.
10894	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
10895	* out-of-sync. Make sure to update the required fields
10896	* before using them.
10897	* @param cbInstr Number of bytes to advance the RIP by.
10898	*
10899	* @remarks No-long-jump zone!!!
10900	*/
10901	DECLINLINE(void) hmR0VmxAdvanceGuestRipBy(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
10902	{
10903	/* Advance the RIP. */
10904	pMixedCtx->rip += cbInstr;
10905	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10906
10907	/* Update interrupt inhibition. */
10908	if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
10909	&& pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
10910	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
10911	}
10912
10913
10914	/**
10915	* Advances the guest RIP after reading it from the VMCS.
10916	*
10917	* @returns VBox status code, no informational status codes.
10918	* @param pVCpu The cross context virtual CPU structure.
10919	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
10920	* out-of-sync. Make sure to update the required fields
10921	* before using them.
10922	* @param pVmxTransient Pointer to the VMX transient structure.
10923	*
10924	* @remarks No-long-jump zone!!!
10925	*/
10926	static int hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10927	{
10928	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10929	rc \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10930	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10931	AssertRCReturn(rc, rc);
10932
10933	hmR0VmxAdvanceGuestRipBy(pVCpu, pMixedCtx, pVmxTransient->cbInstr);
10934
10935	/*
10936	* Deliver a debug exception to the guest if it is single-stepping. Don't directly inject a #DB but use the
10937	* pending debug exception field as it takes care of priority of events.
10938	*
10939	* See Intel spec. 32.2.1 "Debug Exceptions".
10940	*/
10941	if ( !pVCpu->hm.s.fSingleInstruction
10942	&& pMixedCtx->eflags.Bits.u1TF)
10943	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
10944
10945	return VINF_SUCCESS;
10946	}
10947
10948
10949	/**
10950	* Tries to determine what part of the guest-state VT-x has deemed as invalid
10951	* and update error record fields accordingly.
10952	*
10953	* @return VMX_IGS_* return codes.
10954	* @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
10955	* wrong with the guest state.
10956	*
10957	* @param pVM The cross context VM structure.
10958	* @param pVCpu The cross context virtual CPU structure.
10959	* @param pCtx Pointer to the guest-CPU state.
10960	*
10961	* @remarks This function assumes our cache of the VMCS controls
10962	* are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
10963	*/
10964	static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10965	{
10966	#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
10967	#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
10968	uError = (err); \
10969	break; \
10970	} else do { } while (0)
10971
10972	int rc;
10973	uint32_t uError = VMX_IGS_ERROR;
10974	uint32_t u32Val;
10975	bool fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
10976
10977	do
10978	{
10979	/*
10980	* CR0.
10981	*/
10982	uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10983	uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10984	/* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
10985	See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
10986	if (fUnrestrictedGuest)
10987	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
10988
10989	uint32_t u32GuestCR0;
10990	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCR0);
10991	AssertRCBreak(rc);
10992	HMVMX_CHECK_BREAK((u32GuestCR0 & uSetCR0) == uSetCR0, VMX_IGS_CR0_FIXED1);
10993	HMVMX_CHECK_BREAK(!(u32GuestCR0 & ~uZapCR0), VMX_IGS_CR0_FIXED0);
10994	if ( !fUnrestrictedGuest
10995	&& (u32GuestCR0 & X86_CR0_PG)
10996	&& !(u32GuestCR0 & X86_CR0_PE))
10997	{
10998	HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
10999	}
11000
11001	/*
11002	* CR4.
11003	*/
11004	uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
11005	uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
11006
11007	uint32_t u32GuestCR4;
11008	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCR4);
11009	AssertRCBreak(rc);
11010	HMVMX_CHECK_BREAK((u32GuestCR4 & uSetCR4) == uSetCR4, VMX_IGS_CR4_FIXED1);
11011	HMVMX_CHECK_BREAK(!(u32GuestCR4 & ~uZapCR4), VMX_IGS_CR4_FIXED0);
11012
11013	/*
11014	* IA32_DEBUGCTL MSR.
11015	*/
11016	uint64_t u64Val;
11017	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
11018	AssertRCBreak(rc);
11019	if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
11020	&& (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
11021	{
11022	HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
11023	}
11024	uint64_t u64DebugCtlMsr = u64Val;
11025
11026	#ifdef VBOX_STRICT
11027	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
11028	AssertRCBreak(rc);
11029	Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
11030	#endif
11031	bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
11032
11033	/*
11034	* RIP and RFLAGS.
11035	*/
11036	uint32_t u32Eflags;
11037	#if HC_ARCH_BITS == 64
11038	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
11039	AssertRCBreak(rc);
11040	/* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
11041	if ( !fLongModeGuest
11042	\|\| !pCtx->cs.Attr.n.u1Long)
11043	{
11044	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
11045	}
11046	/** @todo If the processor supports N < 64 linear-address bits, bits 63:N
11047	* must be identical if the "IA-32e mode guest" VM-entry
11048	* control is 1 and CS.L is 1. No check applies if the
11049	* CPU supports 64 linear-address bits. */
11050
11051	/* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
11052	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
11053	AssertRCBreak(rc);
11054	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
11055	VMX_IGS_RFLAGS_RESERVED);
11056	HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
11057	u32Eflags = u64Val;
11058	#else
11059	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
11060	AssertRCBreak(rc);
11061	HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
11062	HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
11063	#endif
11064
11065	if ( fLongModeGuest
11066	\|\| ( fUnrestrictedGuest
11067	&& !(u32GuestCR0 & X86_CR0_PE)))
11068	{
11069	HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
11070	}
11071
11072	uint32_t u32EntryInfo;
11073	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
11074	AssertRCBreak(rc);
11075	if ( VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
11076	&& VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
11077	{
11078	HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
11079	}
11080
11081	/*
11082	* 64-bit checks.
11083	*/
11084	#if HC_ARCH_BITS == 64
11085	if (fLongModeGuest)
11086	{
11087	HMVMX_CHECK_BREAK(u32GuestCR0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
11088	HMVMX_CHECK_BREAK(u32GuestCR4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
11089	}
11090
11091	if ( !fLongModeGuest
11092	&& (u32GuestCR4 & X86_CR4_PCIDE))
11093	{
11094	HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
11095	}
11096
11097	/** @todo CR3 field must be such that bits 63:52 and bits in the range
11098	* 51:32 beyond the processor's physical-address width are 0. */
11099
11100	if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
11101	&& (pCtx->dr[7] & X86_DR7_MBZ_MASK))
11102	{
11103	HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
11104	}
11105
11106	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
11107	AssertRCBreak(rc);
11108	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
11109
11110	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
11111	AssertRCBreak(rc);
11112	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
11113	#endif
11114
11115	/*
11116	* PERF_GLOBAL MSR.
11117	*/
11118	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
11119	{
11120	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
11121	AssertRCBreak(rc);
11122	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
11123	VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
11124	}
11125
11126	/*
11127	* PAT MSR.
11128	*/
11129	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
11130	{
11131	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
11132	AssertRCBreak(rc);
11133	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
11134	for (unsigned i = 0; i < 8; i++)
11135	{
11136	uint8_t u8Val = (u64Val & 0xff);
11137	if ( u8Val != 0 /* UC */
11138	&& u8Val != 1 /* WC */
11139	&& u8Val != 4 /* WT */
11140	&& u8Val != 5 /* WP */
11141	&& u8Val != 6 /* WB */
11142	&& u8Val != 7 /* UC- */)
11143	{
11144	HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
11145	}
11146	u64Val >>= 8;
11147	}
11148	}
11149
11150	/*
11151	* EFER MSR.
11152	*/
11153	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
11154	{
11155	Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
11156	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
11157	AssertRCBreak(rc);
11158	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
11159	VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
11160	HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVCpu->hm.s.vmx.u32EntryCtls
11161	& VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
11162	VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
11163	HMVMX_CHECK_BREAK( fUnrestrictedGuest
11164	\|\| !(u32GuestCR0 & X86_CR0_PG)
11165	\|\| RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
11166	VMX_IGS_EFER_LMA_LME_MISMATCH);
11167	}
11168
11169	/*
11170	* Segment registers.
11171	*/
11172	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
11173	\|\| !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
11174	if (!(u32Eflags & X86_EFL_VM))
11175	{
11176	/* CS */
11177	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
11178	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
11179	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
11180	HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
11181	\|\| !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
11182	HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
11183	\|\| (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
11184	/* CS cannot be loaded with NULL in protected mode. */
11185	HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
11186	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
11187	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
11188	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
11189	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
11190	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
11191	else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
11192	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
11193	else
11194	HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
11195
11196	/* SS */
11197	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
11198	\|\| (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
11199	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
11200	if ( !(pCtx->cr0 & X86_CR0_PE)
11201	\|\| pCtx->cs.Attr.n.u4Type == 3)
11202	{
11203	HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
11204	}
11205	if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
11206	{
11207	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
11208	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
11209	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
11210	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
11211	HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
11212	\|\| !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
11213	HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
11214	\|\| (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
11215	}
11216
11217	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
11218	if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
11219	{
11220	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
11221	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
11222	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
11223	\|\| pCtx->ds.Attr.n.u4Type > 11
11224	\|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
11225	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
11226	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
11227	HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
11228	\|\| !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
11229	HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
11230	\|\| (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
11231	HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
11232	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
11233	}
11234	if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
11235	{
11236	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
11237	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
11238	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
11239	\|\| pCtx->es.Attr.n.u4Type > 11
11240	\|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
11241	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
11242	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
11243	HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
11244	\|\| !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
11245	HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
11246	\|\| (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
11247	HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
11248	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
11249	}
11250	if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
11251	{
11252	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
11253	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
11254	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
11255	\|\| pCtx->fs.Attr.n.u4Type > 11
11256	\|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
11257	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
11258	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
11259	HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
11260	\|\| !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
11261	HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
11262	\|\| (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
11263	HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
11264	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
11265	}
11266	if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
11267	{
11268	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
11269	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
11270	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
11271	\|\| pCtx->gs.Attr.n.u4Type > 11
11272	\|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
11273	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
11274	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
11275	HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
11276	\|\| !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
11277	HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
11278	\|\| (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
11279	HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
11280	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
11281	}
11282	/* 64-bit capable CPUs. */
11283	#if HC_ARCH_BITS == 64
11284	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
11285	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
11286	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
11287	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
11288	HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
11289	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ss.u64Base >> 32),
11290	VMX_IGS_LONGMODE_SS_BASE_INVALID);
11291	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ds.u64Base >> 32),
11292	VMX_IGS_LONGMODE_DS_BASE_INVALID);
11293	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->es.u64Base >> 32),
11294	VMX_IGS_LONGMODE_ES_BASE_INVALID);
11295	#endif
11296	}
11297	else
11298	{
11299	/* V86 mode checks. */
11300	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
11301	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11302	{
11303	u32CSAttr = 0xf3; u32SSAttr = 0xf3;
11304	u32DSAttr = 0xf3; u32ESAttr = 0xf3;
11305	u32FSAttr = 0xf3; u32GSAttr = 0xf3;
11306	}
11307	else
11308	{
11309	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
11310	u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
11311	u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
11312	}
11313
11314	/* CS */
11315	HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
11316	HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
11317	HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
11318	/* SS */
11319	HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
11320	HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
11321	HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
11322	/* DS */
11323	HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
11324	HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
11325	HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
11326	/* ES */
11327	HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
11328	HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
11329	HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
11330	/* FS */
11331	HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
11332	HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
11333	HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
11334	/* GS */
11335	HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
11336	HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
11337	HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
11338	/* 64-bit capable CPUs. */
11339	#if HC_ARCH_BITS == 64
11340	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
11341	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
11342	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
11343	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
11344	HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
11345	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ss.u64Base >> 32),
11346	VMX_IGS_LONGMODE_SS_BASE_INVALID);
11347	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ds.u64Base >> 32),
11348	VMX_IGS_LONGMODE_DS_BASE_INVALID);
11349	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->es.u64Base >> 32),
11350	VMX_IGS_LONGMODE_ES_BASE_INVALID);
11351	#endif
11352	}
11353
11354	/*
11355	* TR.
11356	*/
11357	HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
11358	/* 64-bit capable CPUs. */
11359	#if HC_ARCH_BITS == 64
11360	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
11361	#endif
11362	if (fLongModeGuest)
11363	{
11364	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
11365	VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
11366	}
11367	else
11368	{
11369	HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
11370	\|\| pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
11371	VMX_IGS_TR_ATTR_TYPE_INVALID);
11372	}
11373	HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
11374	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
11375	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
11376	HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
11377	\|\| !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
11378	HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
11379	\|\| (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
11380	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
11381
11382	/*
11383	* GDTR and IDTR.
11384	*/
11385	#if HC_ARCH_BITS == 64
11386	rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
11387	AssertRCBreak(rc);
11388	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
11389
11390	rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
11391	AssertRCBreak(rc);
11392	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
11393	#endif
11394
11395	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
11396	AssertRCBreak(rc);
11397	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
11398
11399	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
11400	AssertRCBreak(rc);
11401	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
11402
11403	/*
11404	* Guest Non-Register State.
11405	*/
11406	/* Activity State. */
11407	uint32_t u32ActivityState;
11408	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
11409	AssertRCBreak(rc);
11410	HMVMX_CHECK_BREAK( !u32ActivityState
11411	\|\| (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
11412	VMX_IGS_ACTIVITY_STATE_INVALID);
11413	HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
11414	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
11415	uint32_t u32IntrState;
11416	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
11417	AssertRCBreak(rc);
11418	if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
11419	\|\| u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
11420	{
11421	HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
11422	}
11423
11424	/** @todo Activity state and injecting interrupts. Left as a todo since we
11425	* currently don't use activity states but ACTIVE. */
11426
11427	HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
11428	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
11429
11430	/* Guest interruptibility-state. */
11431	HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
11432	HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
11433	\| VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
11434	!= ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
11435	\| VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
11436	VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
11437	HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
11438	\|\| !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
11439	VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
11440	if (VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo))
11441	{
11442	if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
11443	{
11444	HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
11445	&& !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
11446	VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
11447	}
11448	else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
11449	{
11450	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
11451	VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
11452	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
11453	VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
11454	}
11455	}
11456	/** @todo Assumes the processor is not in SMM. */
11457	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
11458	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
11459	HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
11460	\|\| (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
11461	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
11462	if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
11463	&& VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
11464	&& VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
11465	{
11466	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
11467	VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
11468	}
11469
11470	/* Pending debug exceptions. */
11471	#if HC_ARCH_BITS == 64
11472	rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
11473	AssertRCBreak(rc);
11474	/* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
11475	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
11476	u32Val = u64Val; /* For pending debug exceptions checks below. */
11477	#else
11478	rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
11479	AssertRCBreak(rc);
11480	/* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
11481	HMVMX_CHECK_BREAK(!(u32Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
11482	#endif
11483
11484	if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
11485	\|\| (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
11486	\|\| u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
11487	{
11488	if ( (u32Eflags & X86_EFL_TF)
11489	&& !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
11490	{
11491	/* Bit 14 is PendingDebug.BS. */
11492	HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
11493	}
11494	if ( !(u32Eflags & X86_EFL_TF)
11495	\|\| (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
11496	{
11497	/* Bit 14 is PendingDebug.BS. */
11498	HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
11499	}
11500	}
11501
11502	/* VMCS link pointer. */
11503	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
11504	AssertRCBreak(rc);
11505	if (u64Val != UINT64_C(0xffffffffffffffff))
11506	{
11507	HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
11508	/** @todo Bits beyond the processor's physical-address width MBZ. */
11509	/** @todo 32-bit located in memory referenced by value of this field (as a
11510	* physical address) must contain the processor's VMCS revision ID. */
11511	/** @todo SMM checks. */
11512	}
11513
11514	/** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
11515	* not using Nested Paging? */
11516	if ( pVM->hm.s.fNestedPaging
11517	&& !fLongModeGuest
11518	&& CPUMIsGuestInPAEModeEx(pCtx))
11519	{
11520	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
11521	AssertRCBreak(rc);
11522	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11523
11524	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
11525	AssertRCBreak(rc);
11526	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11527
11528	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
11529	AssertRCBreak(rc);
11530	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11531
11532	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
11533	AssertRCBreak(rc);
11534	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11535	}
11536
11537	/* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
11538	if (uError == VMX_IGS_ERROR)
11539	uError = VMX_IGS_REASON_NOT_FOUND;
11540	} while (0);
11541
11542	pVCpu->hm.s.u32HMError = uError;
11543	return uError;
11544
11545	#undef HMVMX_ERROR_BREAK
11546	#undef HMVMX_CHECK_BREAK
11547	}
11548
11549	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11550	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
11551	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11552
11553	/** @name VM-exit handlers.
11554	* @{
11555	*/
11556
11557	/**
11558	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
11559	*/
11560	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11561	{
11562	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11563	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
11564	/* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
11565	if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
11566	return VINF_SUCCESS;
11567	return VINF_EM_RAW_INTERRUPT;
11568	}
11569
11570
11571	/**
11572	* VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
11573	*/
11574	HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11575	{
11576	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11577	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
11578
11579	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11580	AssertRCReturn(rc, rc);
11581
11582	uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
11583	Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
11584	&& uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
11585	Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
11586
11587	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
11588	{
11589	/*
11590	* This cannot be a guest NMI as the only way for the guest to receive an NMI is if we injected it ourselves and
11591	* anything we inject is not going to cause a VM-exit directly for the event being injected.
11592	* See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
11593	*
11594	* Dispatch the NMI to the host. See Intel spec. 27.5.5 "Updating Non-Register State".
11595	*/
11596	VMXDispatchHostNmi();
11597	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
11598	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11599	return VINF_SUCCESS;
11600	}
11601
11602	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11603	VBOXSTRICTRC rcStrictRc1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11604	if (RT_UNLIKELY(rcStrictRc1 == VINF_SUCCESS))
11605	{ /* likely */ }
11606	else
11607	{
11608	if (rcStrictRc1 == VINF_HM_DOUBLE_FAULT)
11609	rcStrictRc1 = VINF_SUCCESS;
11610	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11611	return rcStrictRc1;
11612	}
11613
11614	uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
11615	uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntInfo);
11616	switch (uIntType)
11617	{
11618	case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
11619	Assert(uVector == X86_XCPT_DB);
11620	RT_FALL_THRU();
11621	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
11622	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF \|\| uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT);
11623	RT_FALL_THRU();
11624	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
11625	{
11626	/*
11627	* If there's any exception caused as a result of event injection, the resulting
11628	* secondary/final execption will be pending, we shall continue guest execution
11629	* after injecting the event. The page-fault case is complicated and we manually
11630	* handle any currently pending event in hmR0VmxExitXcptPF.
11631	*/
11632	if (!pVCpu->hm.s.Event.fPending)
11633	{ /* likely */ }
11634	else if (uVector != X86_XCPT_PF)
11635	{
11636	rc = VINF_SUCCESS;
11637	break;
11638	}
11639
11640	switch (uVector)
11641	{
11642	case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
11643	case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
11644	case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
11645	case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
11646	case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
11647	case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
11648	case X86_XCPT_AC: rc = hmR0VmxExitXcptAC(pVCpu, pMixedCtx, pVmxTransient); break;
11649
11650	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
11651	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11652	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
11653	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11654	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
11655	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11656	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
11657	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11658	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
11659	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11660	case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
11661	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11662	default:
11663	{
11664	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11665	AssertRCReturn(rc, rc);
11666
11667	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
11668	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11669	{
11670	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
11671	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
11672	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
11673
11674	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11675	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11676	AssertRCReturn(rc, rc);
11677	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
11678	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
11679	0 /* GCPtrFaultAddress */);
11680	AssertRCReturn(rc, rc);
11681	}
11682	else
11683	{
11684	AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
11685	pVCpu->hm.s.u32HMError = uVector;
11686	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
11687	}
11688	break;
11689	}
11690	}
11691	break;
11692	}
11693
11694	default:
11695	{
11696	pVCpu->hm.s.u32HMError = uExitIntInfo;
11697	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
11698	AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntInfo)));
11699	break;
11700	}
11701	}
11702	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11703	return rc;
11704	}
11705
11706
11707	/**
11708	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
11709	*/
11710	HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11711	{
11712	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11713
11714	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
11715	hmR0VmxClearIntWindowExitVmcs(pVCpu);
11716
11717	/* Deliver the pending interrupts via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11718	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
11719	return VINF_SUCCESS;
11720	}
11721
11722
11723	/**
11724	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
11725	*/
11726	HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11727	{
11728	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11729	if (RT_UNLIKELY(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)))
11730	{
11731	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
11732	HMVMX_RETURN_UNEXPECTED_EXIT();
11733	}
11734
11735	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS));
11736
11737	/*
11738	* If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
11739	* It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
11740	*/
11741	uint32_t uIntrState = 0;
11742	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
11743	AssertRCReturn(rc, rc);
11744
11745	bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
11746	if ( fBlockSti
11747	&& VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
11748	{
11749	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
11750	}
11751
11752	/* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
11753	hmR0VmxClearNmiWindowExitVmcs(pVCpu);
11754
11755	/* Deliver the pending NMI via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11756	return VINF_SUCCESS;
11757	}
11758
11759
11760	/**
11761	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
11762	*/
11763	HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11764	{
11765	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11766	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
11767	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11768	}
11769
11770
11771	/**
11772	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
11773	*/
11774	HMVMX_EXIT_NSRC_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11775	{
11776	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11777	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
11778	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11779	}
11780
11781
11782	/**
11783	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
11784	*/
11785	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11786	{
11787	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11788	PVM pVM = pVCpu->CTX_SUFF(pVM);
11789	int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11790	if (RT_LIKELY(rc == VINF_SUCCESS))
11791	{
11792	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11793	Assert(pVmxTransient->cbInstr == 2);
11794	}
11795	else
11796	{
11797	AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
11798	rc = VERR_EM_INTERPRETER;
11799	}
11800	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
11801	return rc;
11802	}
11803
11804
11805	/**
11806	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
11807	*/
11808	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11809	{
11810	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11811	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11812	AssertRCReturn(rc, rc);
11813
11814	if (pMixedCtx->cr4 & X86_CR4_SMXE)
11815	return VINF_EM_RAW_EMULATE_INSTR;
11816
11817	AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
11818	HMVMX_RETURN_UNEXPECTED_EXIT();
11819	}
11820
11821
11822	/**
11823	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
11824	*/
11825	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11826	{
11827	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11828	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11829	AssertRCReturn(rc, rc);
11830
11831	PVM pVM = pVCpu->CTX_SUFF(pVM);
11832	rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11833	if (RT_LIKELY(rc == VINF_SUCCESS))
11834	{
11835	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11836	Assert(pVmxTransient->cbInstr == 2);
11837	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
11838	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11839	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11840	}
11841	else
11842	rc = VERR_EM_INTERPRETER;
11843	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
11844	return rc;
11845	}
11846
11847
11848	/**
11849	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
11850	*/
11851	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11852	{
11853	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11854	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11855	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
11856	AssertRCReturn(rc, rc);
11857
11858	PVM pVM = pVCpu->CTX_SUFF(pVM);
11859	rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
11860	if (RT_SUCCESS(rc))
11861	{
11862	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11863	Assert(pVmxTransient->cbInstr == 3);
11864	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
11865	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11866	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11867	}
11868	else
11869	{
11870	AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
11871	rc = VERR_EM_INTERPRETER;
11872	}
11873	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
11874	return rc;
11875	}
11876
11877
11878	/**
11879	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
11880	*/
11881	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11882	{
11883	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11884	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11885	rc \|= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11886	AssertRCReturn(rc, rc);
11887
11888	PVM pVM = pVCpu->CTX_SUFF(pVM);
11889	rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11890	if (RT_LIKELY(rc == VINF_SUCCESS))
11891	{
11892	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11893	Assert(pVmxTransient->cbInstr == 2);
11894	}
11895	else
11896	{
11897	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
11898	rc = VERR_EM_INTERPRETER;
11899	}
11900	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
11901	return rc;
11902	}
11903
11904
11905	/**
11906	* VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
11907	*/
11908	HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11909	{
11910	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11911	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitVmcall);
11912
11913	VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
11914	if (pVCpu->hm.s.fHypercallsEnabled)
11915	{
11916	#if 0
11917	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11918	#else
11919	/* Aggressive state sync. for now. */
11920	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
11921	rc \|= hmR0VmxSaveGuestRflags(pVCpu,pMixedCtx); /* For CPL checks in gimHvHypercall() & gimKvmHypercall() */
11922	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* For long-mode checks in gimKvmHypercall(). */
11923	AssertRCReturn(rc, rc);
11924	#endif
11925
11926	/* Perform the hypercall. */
11927	rcStrict = GIMHypercall(pVCpu, pMixedCtx);
11928	if (rcStrict == VINF_SUCCESS)
11929	{
11930	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11931	AssertRCReturn(rc, rc);
11932	}
11933	else
11934	Assert( rcStrict == VINF_GIM_R3_HYPERCALL
11935	\|\| rcStrict == VINF_GIM_HYPERCALL_CONTINUING
11936	\|\| RT_FAILURE(VBOXSTRICTRC_VAL(rcStrict)));
11937
11938	/* If the hypercall changes anything other than guest's general-purpose registers,
11939	we would need to reload the guest changed bits here before VM-entry. */
11940	}
11941	else
11942	Log4(("hmR0VmxExitVmcall: Hypercalls not enabled\n"));
11943
11944	/* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
11945	if (RT_FAILURE(VBOXSTRICTRC_VAL(rcStrict)))
11946	{
11947	hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
11948	rcStrict = VINF_SUCCESS;
11949	}
11950
11951	return rcStrict;
11952	}
11953
11954
11955	/**
11956	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
11957	*/
11958	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11959	{
11960	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11961	PVM pVM = pVCpu->CTX_SUFF(pVM);
11962	Assert(!pVM->hm.s.fNestedPaging \|\| pVCpu->hm.s.fUsingDebugLoop);
11963
11964	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11965	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11966	AssertRCReturn(rc, rc);
11967
11968	VBOXSTRICTRC rcStrict = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
11969	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11970	rcStrict = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11971	else
11972	AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
11973	pVmxTransient->uExitQualification, VBOXSTRICTRC_VAL(rcStrict)));
11974	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
11975	return rcStrict;
11976	}
11977
11978
11979	/**
11980	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
11981	*/
11982	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11983	{
11984	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11985	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11986	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
11987	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11988	AssertRCReturn(rc, rc);
11989
11990	PVM pVM = pVCpu->CTX_SUFF(pVM);
11991	rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11992	if (RT_LIKELY(rc == VINF_SUCCESS))
11993	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11994	else
11995	{
11996	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
11997	rc = VERR_EM_INTERPRETER;
11998	}
11999	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
12000	return rc;
12001	}
12002
12003
12004	/**
12005	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
12006	*/
12007	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12008	{
12009	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12010	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
12011	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
12012	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12013	AssertRCReturn(rc, rc);
12014
12015	PVM pVM = pVCpu->CTX_SUFF(pVM);
12016	VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
12017	rc = VBOXSTRICTRC_VAL(rc2);
12018	if (RT_LIKELY( rc == VINF_SUCCESS
12019	\|\| rc == VINF_EM_HALT))
12020	{
12021	int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12022	AssertRCReturn(rc3, rc3);
12023
12024	if ( rc == VINF_EM_HALT
12025	&& EMMonitorWaitShouldContinue(pVCpu, pMixedCtx))
12026	{
12027	rc = VINF_SUCCESS;
12028	}
12029	}
12030	else
12031	{
12032	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
12033	rc = VERR_EM_INTERPRETER;
12034	}
12035	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_EM_HALT \|\| rc == VERR_EM_INTERPRETER,
12036	("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
12037	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
12038	return rc;
12039	}
12040
12041
12042	/**
12043	* VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
12044	*/
12045	HMVMX_EXIT_NSRC_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12046	{
12047	/*
12048	* Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
12049	* get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
12050	* executing VMCALL in VMX root operation. If we get here, something funny is going on.
12051	* See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
12052	*/
12053	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12054	AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12055	HMVMX_RETURN_UNEXPECTED_EXIT();
12056	}
12057
12058
12059	/**
12060	* VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
12061	*/
12062	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12063	{
12064	/*
12065	* This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
12066	* root operation. Only an STM (SMM transfer monitor) would get this VM-exit when we (the executive monitor) execute a VMCALL
12067	* in VMX root mode or receive an SMI. If we get here, something funny is going on.
12068	* See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
12069	*/
12070	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12071	AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12072	HMVMX_RETURN_UNEXPECTED_EXIT();
12073	}
12074
12075
12076	/**
12077	* VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
12078	*/
12079	HMVMX_EXIT_NSRC_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12080	{
12081	/* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
12082	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12083	AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12084	HMVMX_RETURN_UNEXPECTED_EXIT();
12085	}
12086
12087
12088	/**
12089	* VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
12090	*/
12091	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12092	{
12093	/*
12094	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
12095	* don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
12096	* See Intel spec. 25.3 "Other Causes of VM-exits".
12097	*/
12098	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12099	AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12100	HMVMX_RETURN_UNEXPECTED_EXIT();
12101	}
12102
12103
12104	/**
12105	* VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
12106	* VM-exit.
12107	*/
12108	HMVMX_EXIT_NSRC_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12109	{
12110	/*
12111	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
12112	* See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
12113	*
12114	* It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
12115	* See Intel spec. "23.8 Restrictions on VMX operation".
12116	*/
12117	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12118	return VINF_SUCCESS;
12119	}
12120
12121
12122	/**
12123	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
12124	* VM-exit.
12125	*/
12126	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12127	{
12128	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12129	return VINF_EM_RESET;
12130	}
12131
12132
12133	/**
12134	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
12135	*/
12136	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12137	{
12138	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12139	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
12140
12141	int rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12142	AssertRCReturn(rc, rc);
12143
12144	if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
12145	rc = VINF_SUCCESS;
12146	else
12147	rc = VINF_EM_HALT;
12148
12149	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
12150	if (rc != VINF_SUCCESS)
12151	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
12152	return rc;
12153	}
12154
12155
12156	/**
12157	* VM-exit handler for instructions that result in a \#UD exception delivered to
12158	* the guest.
12159	*/
12160	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12161	{
12162	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12163	hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
12164	return VINF_SUCCESS;
12165	}
12166
12167
12168	/**
12169	* VM-exit handler for expiry of the VMX preemption timer.
12170	*/
12171	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12172	{
12173	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12174
12175	/* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
12176	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
12177
12178	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
12179	PVM pVM = pVCpu->CTX_SUFF(pVM);
12180	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
12181	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
12182	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
12183	}
12184
12185
12186	/**
12187	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
12188	*/
12189	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12190	{
12191	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12192
12193	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12194	rc \|= hmR0VmxSaveGuestRegsForIemExec(pVCpu, pMixedCtx, false /fMemory/, false /fNeedRsp/);
12195	rc \|= hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
12196	AssertRCReturn(rc, rc);
12197
12198	VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbInstr);
12199	HMCPU_CF_SET(pVCpu, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS : HM_CHANGED_ALL_GUEST);
12200
12201	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
12202
12203	return rcStrict;
12204	}
12205
12206
12207	/**
12208	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
12209	*/
12210	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12211	{
12212	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12213	/** @todo Use VM-exit instruction information. */
12214	return VERR_EM_INTERPRETER;
12215	}
12216
12217
12218	/**
12219	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
12220	* Error VM-exit.
12221	*/
12222	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12223	{
12224	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12225	AssertRCReturn(rc, rc);
12226
12227	rc = hmR0VmxCheckVmcsCtls(pVCpu);
12228	AssertRCReturn(rc, rc);
12229
12230	uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
12231	NOREF(uInvalidReason);
12232
12233	#ifdef VBOX_STRICT
12234	uint32_t uIntrState;
12235	RTHCUINTREG uHCReg;
12236	uint64_t u64Val;
12237	uint32_t u32Val;
12238
12239	rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
12240	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
12241	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
12242	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
12243	AssertRCReturn(rc, rc);
12244
12245	Log4(("uInvalidReason %u\n", uInvalidReason));
12246	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
12247	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
12248	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
12249	Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
12250
12251	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
12252	Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
12253	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
12254	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
12255	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
12256	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
12257	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
12258	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
12259	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
12260	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
12261	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
12262	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
12263	#else
12264	NOREF(pVmxTransient);
12265	#endif
12266
12267	hmR0DumpRegs(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
12268	return VERR_VMX_INVALID_GUEST_STATE;
12269	}
12270
12271
12272	/**
12273	* VM-exit handler for VM-entry failure due to an MSR-load
12274	* (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
12275	*/
12276	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12277	{
12278	NOREF(pVmxTransient);
12279	AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
12280	HMVMX_RETURN_UNEXPECTED_EXIT();
12281	}
12282
12283
12284	/**
12285	* VM-exit handler for VM-entry failure due to a machine-check event
12286	* (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
12287	*/
12288	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12289	{
12290	NOREF(pVmxTransient);
12291	AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
12292	HMVMX_RETURN_UNEXPECTED_EXIT();
12293	}
12294
12295
12296	/**
12297	* VM-exit handler for all undefined reasons. Should never ever happen.. in
12298	* theory.
12299	*/
12300	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12301	{
12302	AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
12303	NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient);
12304	return VERR_VMX_UNDEFINED_EXIT_CODE;
12305	}
12306
12307
12308	/**
12309	* VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
12310	* (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
12311	* Conditional VM-exit.
12312	*/
12313	HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12314	{
12315	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12316
12317	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
12318	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
12319	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
12320	return VERR_EM_INTERPRETER;
12321	AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12322	HMVMX_RETURN_UNEXPECTED_EXIT();
12323	}
12324
12325
12326	/**
12327	* VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
12328	*/
12329	HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12330	{
12331	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12332
12333	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
12334	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
12335	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
12336	return VERR_EM_INTERPRETER;
12337	AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12338	HMVMX_RETURN_UNEXPECTED_EXIT();
12339	}
12340
12341
12342	/**
12343	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
12344	*/
12345	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12346	{
12347	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12348
12349	/* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
12350	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
12351	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
12352	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12353	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
12354	{
12355	rc \|= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
12356	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
12357	}
12358	AssertRCReturn(rc, rc);
12359	Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
12360
12361	#ifdef VBOX_STRICT
12362	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
12363	{
12364	if ( hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx)
12365	&& pMixedCtx->ecx != MSR_K6_EFER)
12366	{
12367	AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
12368	pMixedCtx->ecx));
12369	HMVMX_RETURN_UNEXPECTED_EXIT();
12370	}
12371	if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
12372	{
12373	VMXMSREXITREAD enmRead;
12374	VMXMSREXITWRITE enmWrite;
12375	int rc2 = hmR0VmxGetMsrPermission(pVCpu, pMixedCtx->ecx, &enmRead, &enmWrite);
12376	AssertRCReturn(rc2, rc2);
12377	if (enmRead == VMXMSREXIT_PASSTHRU_READ)
12378	{
12379	AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
12380	HMVMX_RETURN_UNEXPECTED_EXIT();
12381	}
12382	}
12383	}
12384	#endif
12385
12386	PVM pVM = pVCpu->CTX_SUFF(pVM);
12387	rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
12388	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER,
12389	("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
12390	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
12391	if (RT_SUCCESS(rc))
12392	{
12393	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12394	Assert(pVmxTransient->cbInstr == 2);
12395	}
12396	return rc;
12397	}
12398
12399
12400	/**
12401	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
12402	*/
12403	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12404	{
12405	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12406	PVM pVM = pVCpu->CTX_SUFF(pVM);
12407	int rc = VINF_SUCCESS;
12408
12409	/* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
12410	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
12411	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
12412	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12413	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
12414	{
12415	rc \|= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
12416	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
12417	}
12418	AssertRCReturn(rc, rc);
12419	Log4(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", pMixedCtx->ecx, pMixedCtx->edx, pMixedCtx->eax));
12420
12421	rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
12422	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
12423	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
12424
12425	if (RT_SUCCESS(rc))
12426	{
12427	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12428
12429	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
12430	if ( pMixedCtx->ecx == MSR_IA32_APICBASE
12431	\|\| ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
12432	&& pMixedCtx->ecx <= MSR_IA32_X2APIC_END))
12433	{
12434	/*
12435	* We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
12436	* virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
12437	* EMInterpretWrmsr() changes it.
12438	*/
12439	HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
12440	}
12441	else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
12442	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
12443	else if (pMixedCtx->ecx == MSR_K6_EFER)
12444	{
12445	/*
12446	* If the guest touches EFER we need to update the VM-Entry and VM-Exit controls as well,
12447	* even if it is -not- touching bits that cause paging mode changes (LMA/LME). We care about
12448	* the other bits as well, SCE and NXE. See @bugref{7368}.
12449	*/
12450	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_EFER_MSR \| HM_CHANGED_VMX_ENTRY_CTLS \| HM_CHANGED_VMX_EXIT_CTLS);
12451	}
12452
12453	/* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
12454	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
12455	{
12456	switch (pMixedCtx->ecx)
12457	{
12458	/*
12459	* For SYSENTER CS, EIP, ESP MSRs, we set both the flags here so we don't accidentally
12460	* overwrite the changed guest-CPU context value while going to ring-3, see @bufref{8745}.
12461	*/
12462	case MSR_IA32_SYSENTER_CS:
12463	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR);
12464	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR);
12465	break;
12466	case MSR_IA32_SYSENTER_EIP:
12467	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
12468	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR);
12469	break;
12470	case MSR_IA32_SYSENTER_ESP:
12471	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
12472	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR);
12473	break;
12474	case MSR_K8_FS_BASE: RT_FALL_THRU();
12475	case MSR_K8_GS_BASE: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS); break;
12476	case MSR_K6_EFER: /* already handled above */ break;
12477	default:
12478	{
12479	if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
12480	HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
12481	else if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
12482	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
12483	break;
12484	}
12485	}
12486	}
12487	#ifdef VBOX_STRICT
12488	else
12489	{
12490	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
12491	switch (pMixedCtx->ecx)
12492	{
12493	case MSR_IA32_SYSENTER_CS:
12494	case MSR_IA32_SYSENTER_EIP:
12495	case MSR_IA32_SYSENTER_ESP:
12496	case MSR_K8_FS_BASE:
12497	case MSR_K8_GS_BASE:
12498	{
12499	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
12500	HMVMX_RETURN_UNEXPECTED_EXIT();
12501	}
12502
12503	/* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
12504	default:
12505	{
12506	if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
12507	{
12508	/* EFER writes are always intercepted, see hmR0VmxLoadGuestMsrs(). */
12509	if (pMixedCtx->ecx != MSR_K6_EFER)
12510	{
12511	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
12512	pMixedCtx->ecx));
12513	HMVMX_RETURN_UNEXPECTED_EXIT();
12514	}
12515	}
12516
12517	if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
12518	{
12519	VMXMSREXITREAD enmRead;
12520	VMXMSREXITWRITE enmWrite;
12521	int rc2 = hmR0VmxGetMsrPermission(pVCpu, pMixedCtx->ecx, &enmRead, &enmWrite);
12522	AssertRCReturn(rc2, rc2);
12523	if (enmWrite == VMXMSREXIT_PASSTHRU_WRITE)
12524	{
12525	AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
12526	HMVMX_RETURN_UNEXPECTED_EXIT();
12527	}
12528	}
12529	break;
12530	}
12531	}
12532	}
12533	#endif /* VBOX_STRICT */
12534	}
12535	return rc;
12536	}
12537
12538
12539	/**
12540	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
12541	*/
12542	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12543	{
12544	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12545
12546	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
12547	return VINF_EM_RAW_INTERRUPT;
12548	}
12549
12550
12551	/**
12552	* VM-exit handler for when the TPR value is lowered below the specified
12553	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
12554	*/
12555	HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12556	{
12557	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12558	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
12559
12560	/*
12561	* The TPR shadow would've been synced with the APIC TPR in hmR0VmxPostRunGuest(). We'll re-evaluate
12562	* pending interrupts and inject them before the next VM-entry so we can just continue execution here.
12563	*/
12564	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
12565	return VINF_SUCCESS;
12566	}
12567
12568
12569	/**
12570	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
12571	* VM-exit.
12572	*
12573	* @retval VINF_SUCCESS when guest execution can continue.
12574	* @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
12575	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
12576	* @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
12577	* interpreter.
12578	*/
12579	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12580	{
12581	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12582	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
12583	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12584	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12585	AssertRCReturn(rc, rc);
12586
12587	RTGCUINTPTR const uExitQualification = pVmxTransient->uExitQualification;
12588	uint32_t const uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
12589	PVM pVM = pVCpu->CTX_SUFF(pVM);
12590	VBOXSTRICTRC rcStrict;
12591	rc = hmR0VmxSaveGuestRegsForIemExec(pVCpu, pMixedCtx, false /fMemory/, true /fNeedRsp/);
12592	switch (uAccessType)
12593	{
12594	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
12595	{
12596	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12597	AssertRCReturn(rc, rc);
12598
12599	rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, pVmxTransient->cbInstr,
12600	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
12601	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
12602	AssertMsg( rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_IEM_RAISED_XCPT \|\| rcStrict == VINF_PGM_CHANGE_MODE
12603	\|\| rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12604	switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
12605	{
12606	case 0: /* CR0 */
12607	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
12608	Log4(("CRX CR0 write rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr0));
12609	break;
12610	case 2: /* CR2 */
12611	/* Nothing to do here, CR2 it's not part of the VMCS. */
12612	break;
12613	case 3: /* CR3 */
12614	Assert(!pVM->hm.s.fNestedPaging \|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx) \|\| pVCpu->hm.s.fUsingDebugLoop);
12615	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR3);
12616	Log4(("CRX CR3 write rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr3));
12617	break;
12618	case 4: /* CR4 */
12619	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
12620	Log4(("CRX CR4 write rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n",
12621	VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
12622	break;
12623	case 8: /* CR8 */
12624	Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
12625	/* CR8 contains the APIC TPR. Was updated by IEMExecDecodedMovCRxWrite(). */
12626	HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
12627	break;
12628	default:
12629	AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
12630	break;
12631	}
12632
12633	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
12634	break;
12635	}
12636
12637	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
12638	{
12639	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12640	AssertRCReturn(rc, rc);
12641
12642	Assert( !pVM->hm.s.fNestedPaging
12643	\|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx)
12644	\|\| pVCpu->hm.s.fUsingDebugLoop
12645	\|\| VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
12646
12647	/* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
12648	Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
12649	\|\| !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
12650
12651	rcStrict = IEMExecDecodedMovCRxRead(pVCpu, pVmxTransient->cbInstr,
12652	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
12653	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
12654	AssertMsg(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12655	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
12656	Log4(("CRX CR%d Read access rcStrict=%Rrc\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
12657	VBOXSTRICTRC_VAL(rcStrict)));
12658	if (VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification) == X86_GREG_xSP)
12659	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RSP);
12660	break;
12661	}
12662
12663	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
12664	{
12665	AssertRCReturn(rc, rc);
12666	rcStrict = IEMExecDecodedClts(pVCpu, pVmxTransient->cbInstr);
12667	AssertMsg(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12668	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
12669	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
12670	Log4(("CRX CLTS rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12671	break;
12672	}
12673
12674	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
12675	{
12676	AssertRCReturn(rc, rc);
12677	rcStrict = IEMExecDecodedLmsw(pVCpu, pVmxTransient->cbInstr,
12678	VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
12679	AssertMsg(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_IEM_RAISED_XCPT \|\| rcStrict == VINF_PGM_CHANGE_MODE,
12680	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12681	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
12682	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
12683	Log4(("CRX LMSW rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12684	break;
12685	}
12686
12687	default:
12688	AssertMsgFailedReturn(("Invalid access-type in Mov CRx VM-exit qualification %#x\n", uAccessType),
12689	VERR_VMX_UNEXPECTED_EXCEPTION);
12690	}
12691
12692	HMCPU_CF_SET(pVCpu, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS : HM_CHANGED_ALL_GUEST);
12693	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
12694	NOREF(pVM);
12695	return rcStrict;
12696	}
12697
12698
12699	/**
12700	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
12701	* VM-exit.
12702	*/
12703	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12704	{
12705	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12706	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
12707
12708	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12709	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12710	rc \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
12711	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
12712	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
12713	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
12714	/* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
12715	AssertRCReturn(rc, rc);
12716
12717	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
12718	uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
12719	uint8_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
12720	bool fIOWrite = ( VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
12721	== VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
12722	bool fIOString = VMX_EXIT_QUALIFICATION_IO_IS_STRING(pVmxTransient->uExitQualification);
12723	bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
12724	bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
12725	AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_VMX_IPE_1);
12726
12727	/* I/O operation lookup arrays. */
12728	static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
12729	static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
12730
12731	VBOXSTRICTRC rcStrict;
12732	uint32_t const cbValue = s_aIOSizes[uIOWidth];
12733	uint32_t const cbInstr = pVmxTransient->cbInstr;
12734	bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
12735	PVM pVM = pVCpu->CTX_SUFF(pVM);
12736	if (fIOString)
12737	{
12738	#ifdef VBOX_WITH_2ND_IEM_STEP /* This used to gurus with debian 32-bit guest without NP (on ATA reads).
12739	See @bugref{5752#c158}. Should work now. */
12740	/*
12741	* INS/OUTS - I/O String instruction.
12742	*
12743	* Use instruction-information if available, otherwise fall back on
12744	* interpreting the instruction.
12745	*/
12746	Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue,
12747	fIOWrite ? 'w' : 'r'));
12748	AssertReturn(pMixedCtx->dx == uIOPort, VERR_VMX_IPE_2);
12749	if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
12750	{
12751	int rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
12752	/** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
12753	rc2 \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12754	AssertRCReturn(rc2, rc2);
12755	AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
12756	AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
12757	IEMMODE enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
12758	bool fRep = VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification);
12759	if (fIOWrite)
12760	{
12761	rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
12762	pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /fIoChecked/);
12763	}
12764	else
12765	{
12766	/*
12767	* The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
12768	* Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
12769	* See Intel Instruction spec. for "INS".
12770	* See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
12771	*/
12772	rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /fIoChecked/);
12773	}
12774	}
12775	else
12776	{
12777	/** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
12778	int rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12779	AssertRCReturn(rc2, rc2);
12780	rcStrict = IEMExecOne(pVCpu);
12781	}
12782	/** @todo IEM needs to be setting these flags somehow. */
12783	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
12784	fUpdateRipAlready = true;
12785	#else
12786	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
12787	rcStrict = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL /* pcbInstr */);
12788	if (RT_SUCCESS(rcStrict))
12789	{
12790	if (fIOWrite)
12791	{
12792	rcStrict = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
12793	(DISCPUMODE)pDis->uAddrMode, cbValue);
12794	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
12795	}
12796	else
12797	{
12798	rcStrict = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
12799	(DISCPUMODE)pDis->uAddrMode, cbValue);
12800	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
12801	}
12802	}
12803	else
12804	{
12805	AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("rcStrict=%Rrc RIP=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict),
12806	pMixedCtx->rip));
12807	rcStrict = VINF_EM_RAW_EMULATE_INSTR;
12808	}
12809	#endif
12810	}
12811	else
12812	{
12813	/*
12814	* IN/OUT - I/O instruction.
12815	*/
12816	Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
12817	uint32_t const uAndVal = s_aIOOpAnd[uIOWidth];
12818	Assert(!VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification));
12819	if (fIOWrite)
12820	{
12821	rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
12822	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
12823	}
12824	else
12825	{
12826	uint32_t u32Result = 0;
12827	rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
12828	if (IOM_SUCCESS(rcStrict))
12829	{
12830	/* Save result of I/O IN instr. in AL/AX/EAX. */
12831	pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
12832	}
12833	else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
12834	HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
12835	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
12836	}
12837	}
12838
12839	if (IOM_SUCCESS(rcStrict))
12840	{
12841	if (!fUpdateRipAlready)
12842	{
12843	hmR0VmxAdvanceGuestRipBy(pVCpu, pMixedCtx, cbInstr);
12844	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
12845	}
12846
12847	/*
12848	* INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru while booting Fedora 17 64-bit guest.
12849	* See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
12850	*/
12851	if (fIOString)
12852	{
12853	/** @todo Single-step for INS/OUTS with REP prefix? */
12854	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
12855	}
12856	else if ( !fDbgStepping
12857	&& fGstStepping)
12858	{
12859	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
12860	}
12861
12862	/*
12863	* If any I/O breakpoints are armed, we need to check if one triggered
12864	* and take appropriate action.
12865	* Note that the I/O breakpoint type is undefined if CR4.DE is 0.
12866	*/
12867	int rc2 = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
12868	AssertRCReturn(rc2, rc2);
12869
12870	/** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
12871	* execution engines about whether hyper BPs and such are pending. */
12872	uint32_t const uDr7 = pMixedCtx->dr[7];
12873	if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
12874	&& X86_DR7_ANY_RW_IO(uDr7)
12875	&& (pMixedCtx->cr4 & X86_CR4_DE))
12876	\|\| DBGFBpIsHwIoArmed(pVM)))
12877	{
12878	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
12879
12880	/* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
12881	VMMRZCallRing3Disable(pVCpu);
12882	HM_DISABLE_PREEMPT();
12883
12884	bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
12885
12886	VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
12887	if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
12888	{
12889	/* Raise #DB. */
12890	if (fIsGuestDbgActive)
12891	ASMSetDR6(pMixedCtx->dr[6]);
12892	if (pMixedCtx->dr[7] != uDr7)
12893	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
12894
12895	hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
12896	}
12897	/* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
12898	however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
12899	else if ( rcStrict2 != VINF_SUCCESS
12900	&& (rcStrict == VINF_SUCCESS \|\| rcStrict2 < rcStrict))
12901	rcStrict = rcStrict2;
12902	AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
12903
12904	HM_RESTORE_PREEMPT();
12905	VMMRZCallRing3Enable(pVCpu);
12906	}
12907	}
12908
12909	#ifdef VBOX_STRICT
12910	if (rcStrict == VINF_IOM_R3_IOPORT_READ)
12911	Assert(!fIOWrite);
12912	else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE \|\| rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE)
12913	Assert(fIOWrite);
12914	else
12915	{
12916	#if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
12917	* statuses, that the VMM device and some others may return. See
12918	* IOM_SUCCESS() for guidance. */
12919	AssertMsg( RT_FAILURE(rcStrict)
12920	\|\| rcStrict == VINF_SUCCESS
12921	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
12922	\|\| rcStrict == VINF_EM_DBG_BREAKPOINT
12923	\|\| rcStrict == VINF_EM_RAW_GUEST_TRAP
12924	\|\| rcStrict == VINF_EM_RAW_TO_R3
12925	\|\| rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12926	#endif
12927	}
12928	#endif
12929
12930	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
12931	return rcStrict;
12932	}
12933
12934
12935	/**
12936	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
12937	* VM-exit.
12938	*/
12939	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12940	{
12941	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12942
12943	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
12944	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12945	AssertRCReturn(rc, rc);
12946	if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
12947	{
12948	rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
12949	AssertRCReturn(rc, rc);
12950	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
12951	{
12952	uint32_t uErrCode;
12953	RTGCUINTPTR GCPtrFaultAddress;
12954	uint32_t const uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
12955	uint32_t const uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
12956	bool const fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
12957	if (fErrorCodeValid)
12958	{
12959	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
12960	AssertRCReturn(rc, rc);
12961	uErrCode = pVmxTransient->uIdtVectoringErrorCode;
12962	}
12963	else
12964	uErrCode = 0;
12965
12966	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
12967	&& uVector == X86_XCPT_PF)
12968	GCPtrFaultAddress = pMixedCtx->cr2;
12969	else
12970	GCPtrFaultAddress = 0;
12971
12972	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
12973	0 /* cbInstr */, uErrCode, GCPtrFaultAddress);
12974
12975	Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
12976	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12977	return VINF_EM_RAW_INJECT_TRPM_EVENT;
12978	}
12979	}
12980
12981	/* Fall back to the interpreter to emulate the task-switch. */
12982	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12983	return VERR_EM_INTERPRETER;
12984	}
12985
12986
12987	/**
12988	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
12989	*/
12990	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12991	{
12992	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12993	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
12994	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
12995	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12996	AssertRCReturn(rc, rc);
12997	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
12998	return VINF_EM_DBG_STEPPED;
12999	}
13000
13001
13002	/**
13003	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
13004	*/
13005	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13006	{
13007	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
13008
13009	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
13010
13011	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
13012	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
13013	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
13014	{
13015	/* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
13016	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
13017	{
13018	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
13019	return VINF_EM_RAW_INJECT_TRPM_EVENT;
13020	}
13021	}
13022	else
13023	{
13024	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
13025	rcStrict1 = VINF_SUCCESS;
13026	return rcStrict1;
13027	}
13028
13029	#if 0
13030	/** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
13031	* just sync the whole thing. */
13032	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13033	#else
13034	/* Aggressive state sync. for now. */
13035	int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
13036	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
13037	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
13038	#endif
13039	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13040	AssertRCReturn(rc, rc);
13041
13042	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
13043	uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
13044	VBOXSTRICTRC rcStrict2;
13045	switch (uAccessType)
13046	{
13047	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
13048	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
13049	{
13050	AssertMsg( !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
13051	\|\| VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) != XAPIC_OFF_TPR,
13052	("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
13053
13054	RTGCPHYS GCPhys = pVCpu->hm.s.vmx.u64MsrApicBase; /* Always up-to-date, u64MsrApicBase is not part of the VMCS. */
13055	GCPhys &= PAGE_BASE_GC_MASK;
13056	GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
13057	PVM pVM = pVCpu->CTX_SUFF(pVM);
13058	Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
13059	VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
13060
13061	rcStrict2 = IOMMMIOPhysHandler(pVM, pVCpu,
13062	uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
13063	CPUMCTX2CORE(pMixedCtx), GCPhys);
13064	Log4(("ApicAccess rcStrict2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
13065	if ( rcStrict2 == VINF_SUCCESS
13066	\|\| rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
13067	\|\| rcStrict2 == VERR_PAGE_NOT_PRESENT)
13068	{
13069	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13070	\| HM_CHANGED_GUEST_RSP
13071	\| HM_CHANGED_GUEST_RFLAGS
13072	\| HM_CHANGED_VMX_GUEST_APIC_STATE);
13073	rcStrict2 = VINF_SUCCESS;
13074	}
13075	break;
13076	}
13077
13078	default:
13079	Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
13080	rcStrict2 = VINF_EM_RAW_EMULATE_INSTR;
13081	break;
13082	}
13083
13084	if (rcStrict2 != VINF_SUCCESS)
13085	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
13086	return rcStrict2;
13087	}
13088
13089
13090	/**
13091	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
13092	* VM-exit.
13093	*/
13094	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13095	{
13096	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
13097
13098	/* We should -not- get this VM-exit if the guest's debug registers were active. */
13099	if (pVmxTransient->fWasGuestDebugStateActive)
13100	{
13101	AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
13102	HMVMX_RETURN_UNEXPECTED_EXIT();
13103	}
13104
13105	if ( !pVCpu->hm.s.fSingleInstruction
13106	&& !pVmxTransient->fWasHyperDebugStateActive)
13107	{
13108	Assert(!DBGFIsStepping(pVCpu));
13109	Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
13110
13111	/* Don't intercept MOV DRx any more. */
13112	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
13113	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
13114	AssertRCReturn(rc, rc);
13115
13116	/* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
13117	VMMRZCallRing3Disable(pVCpu);
13118	HM_DISABLE_PREEMPT();
13119
13120	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
13121	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
13122	Assert(CPUMIsGuestDebugStateActive(pVCpu) \|\| HC_ARCH_BITS == 32);
13123
13124	HM_RESTORE_PREEMPT();
13125	VMMRZCallRing3Enable(pVCpu);
13126
13127	#ifdef VBOX_WITH_STATISTICS
13128	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13129	AssertRCReturn(rc, rc);
13130	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
13131	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
13132	else
13133	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
13134	#endif
13135	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
13136	return VINF_SUCCESS;
13137	}
13138
13139	/*
13140	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
13141	* Update the segment registers and DR7 from the CPU.
13142	*/
13143	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13144	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
13145	rc \|= hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
13146	AssertRCReturn(rc, rc);
13147	Log4(("CS:RIP=%04x:%08RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
13148
13149	PVM pVM = pVCpu->CTX_SUFF(pVM);
13150	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
13151	{
13152	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
13153	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
13154	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
13155	if (RT_SUCCESS(rc))
13156	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
13157	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
13158	}
13159	else
13160	{
13161	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
13162	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
13163	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
13164	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
13165	}
13166
13167	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
13168	if (RT_SUCCESS(rc))
13169	{
13170	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
13171	AssertRCReturn(rc2, rc2);
13172	return VINF_SUCCESS;
13173	}
13174	return rc;
13175	}
13176
13177
13178	/**
13179	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
13180	* Conditional VM-exit.
13181	*/
13182	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13183	{
13184	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
13185	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
13186
13187	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
13188	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
13189	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
13190	{
13191	/* If event delivery causes an EPT misconfig (MMIO), go back to instruction emulation as otherwise
13192	injecting the original pending event would most likely cause the same EPT misconfig VM-exit. */
13193	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
13194	{
13195	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
13196	return VINF_EM_RAW_INJECT_TRPM_EVENT;
13197	}
13198	}
13199	else
13200	{
13201	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
13202	rcStrict1 = VINF_SUCCESS;
13203	return rcStrict1;
13204	}
13205
13206	RTGCPHYS GCPhys = 0;
13207	int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
13208
13209	#if 0
13210	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
13211	#else
13212	/* Aggressive state sync. for now. */
13213	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
13214	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
13215	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
13216	#endif
13217	AssertRCReturn(rc, rc);
13218
13219	/*
13220	* If we succeed, resume guest execution.
13221	* If we fail in interpreting the instruction because we couldn't get the guest physical address
13222	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
13223	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
13224	* weird case. See @bugref{6043}.
13225	*/
13226	PVM pVM = pVCpu->CTX_SUFF(pVM);
13227	VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
13228	Log4(("EPT misconfig at %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pMixedCtx->rip, VBOXSTRICTRC_VAL(rcStrict2)));
13229	if ( rcStrict2 == VINF_SUCCESS
13230	\|\| rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
13231	\|\| rcStrict2 == VERR_PAGE_NOT_PRESENT)
13232	{
13233	/* Successfully handled MMIO operation. */
13234	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13235	\| HM_CHANGED_GUEST_RSP
13236	\| HM_CHANGED_GUEST_RFLAGS
13237	\| HM_CHANGED_VMX_GUEST_APIC_STATE);
13238	return VINF_SUCCESS;
13239	}
13240	return rcStrict2;
13241	}
13242
13243
13244	/**
13245	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
13246	* VM-exit.
13247	*/
13248	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13249	{
13250	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
13251	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
13252
13253	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
13254	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
13255	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
13256	{
13257	/* In the unlikely case that the EPT violation happened as a result of delivering an event, log it. */
13258	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
13259	Log4(("EPT violation with an event pending u64IntInfo=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo));
13260	}
13261	else
13262	{
13263	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
13264	rcStrict1 = VINF_SUCCESS;
13265	return rcStrict1;
13266	}
13267
13268	RTGCPHYS GCPhys = 0;
13269	int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
13270	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13271	#if 0
13272	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
13273	#else
13274	/* Aggressive state sync. for now. */
13275	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
13276	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
13277	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
13278	#endif
13279	AssertRCReturn(rc, rc);
13280
13281	/* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
13282	AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
13283
13284	RTGCUINT uErrorCode = 0;
13285	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
13286	uErrorCode \|= X86_TRAP_PF_ID;
13287	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
13288	uErrorCode \|= X86_TRAP_PF_RW;
13289	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
13290	uErrorCode \|= X86_TRAP_PF_P;
13291
13292	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
13293
13294	Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:RIP=%04x:%08RX64\n", pVmxTransient->uExitQualification, GCPhys,
13295	uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
13296
13297	/* Handle the pagefault trap for the nested shadow table. */
13298	PVM pVM = pVCpu->CTX_SUFF(pVM);
13299	VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
13300	TRPMResetTrap(pVCpu);
13301
13302	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
13303	if ( rcStrict2 == VINF_SUCCESS
13304	\|\| rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
13305	\|\| rcStrict2 == VERR_PAGE_NOT_PRESENT)
13306	{
13307	/* Successfully synced our nested page tables. */
13308	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
13309	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13310	\| HM_CHANGED_GUEST_RSP
13311	\| HM_CHANGED_GUEST_RFLAGS);
13312	return VINF_SUCCESS;
13313	}
13314
13315	Log4(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
13316	return rcStrict2;
13317	}
13318
13319	/** @} */
13320
13321	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13322	/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
13323	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13324
13325	/** @name VM-exit exception handlers.
13326	* @{
13327	*/
13328
13329	/**
13330	* VM-exit exception handler for \#MF (Math Fault: floating point exception).
13331	*/
13332	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13333	{
13334	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13335	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
13336
13337	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
13338	AssertRCReturn(rc, rc);
13339
13340	if (!(pMixedCtx->cr0 & X86_CR0_NE))
13341	{
13342	/* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
13343	rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
13344
13345	/** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
13346	* provides VM-exit instruction length. If this causes problem later,
13347	* disassemble the instruction like it's done on AMD-V. */
13348	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
13349	AssertRCReturn(rc2, rc2);
13350	return rc;
13351	}
13352
13353	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13354	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13355	return rc;
13356	}
13357
13358
13359	/**
13360	* VM-exit exception handler for \#BP (Breakpoint exception).
13361	*/
13362	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13363	{
13364	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13365	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
13366
13367	/** @todo Try optimize this by not saving the entire guest state unless
13368	* really needed. */
13369	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13370	AssertRCReturn(rc, rc);
13371
13372	PVM pVM = pVCpu->CTX_SUFF(pVM);
13373	rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
13374	if (rc == VINF_EM_RAW_GUEST_TRAP)
13375	{
13376	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13377	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13378	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13379	AssertRCReturn(rc, rc);
13380
13381	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13382	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13383	}
13384
13385	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_EM_DBG_BREAKPOINT);
13386	return rc;
13387	}
13388
13389
13390	/**
13391	* VM-exit exception handler for \#AC (alignment check exception).
13392	*/
13393	static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13394	{
13395	RT_NOREF_PV(pMixedCtx);
13396	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13397
13398	/*
13399	* Re-inject it. We'll detect any nesting before getting here.
13400	*/
13401	int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13402	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13403	AssertRCReturn(rc, rc);
13404	Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
13405
13406	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13407	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13408	return VINF_SUCCESS;
13409	}
13410
13411
13412	/**
13413	* VM-exit exception handler for \#DB (Debug exception).
13414	*/
13415	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13416	{
13417	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13418	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
13419	Log6(("XcptDB\n"));
13420
13421	/*
13422	* Get the DR6-like values from the VM-exit qualification and pass it to DBGF
13423	* for processing.
13424	*/
13425	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13426	AssertRCReturn(rc, rc);
13427
13428	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
13429	uint64_t uDR6 = X86_DR6_INIT_VAL;
13430	uDR6 \|= ( pVmxTransient->uExitQualification
13431	& (X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3 \| X86_DR6_BD \| X86_DR6_BS));
13432
13433	rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
13434	if (rc == VINF_EM_RAW_GUEST_TRAP)
13435	{
13436	/*
13437	* The exception was for the guest. Update DR6, DR7.GD and
13438	* IA32_DEBUGCTL.LBR before forwarding it.
13439	* (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
13440	*/
13441	VMMRZCallRing3Disable(pVCpu);
13442	HM_DISABLE_PREEMPT();
13443
13444	pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
13445	pMixedCtx->dr[6] \|= uDR6;
13446	if (CPUMIsGuestDebugStateActive(pVCpu))
13447	ASMSetDR6(pMixedCtx->dr[6]);
13448
13449	HM_RESTORE_PREEMPT();
13450	VMMRZCallRing3Enable(pVCpu);
13451
13452	rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
13453	AssertRCReturn(rc, rc);
13454
13455	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
13456	pMixedCtx->dr[7] &= ~X86_DR7_GD;
13457
13458	/* Paranoia. */
13459	pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
13460	pMixedCtx->dr[7] \|= X86_DR7_RA1_MASK;
13461
13462	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
13463	AssertRCReturn(rc, rc);
13464
13465	/*
13466	* Raise #DB in the guest.
13467	*
13468	* It is important to reflect what the VM-exit gave us (preserving the interruption-type) rather than use
13469	* hmR0VmxSetPendingXcptDB() as the #DB could've been raised while executing ICEBP and not the 'normal' #DB.
13470	* Thus it -may- trigger different handling in the CPU (like skipped DPL checks). See @bugref{6398}.
13471	*
13472	* Since ICEBP isn't documented on Intel, see AMD spec. 15.20 "Event Injection".
13473	*/
13474	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13475	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13476	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13477	AssertRCReturn(rc, rc);
13478	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13479	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13480	return VINF_SUCCESS;
13481	}
13482
13483	/*
13484	* Not a guest trap, must be a hypervisor related debug event then.
13485	* Update DR6 in case someone is interested in it.
13486	*/
13487	AssertMsg(rc == VINF_EM_DBG_STEPPED \|\| rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
13488	AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
13489	CPUMSetHyperDR6(pVCpu, uDR6);
13490
13491	return rc;
13492	}
13493
13494
13495	/**
13496	* VM-exit exception handler for \#NM (Device-not-available exception: floating
13497	* point exception).
13498	*/
13499	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13500	{
13501	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13502
13503	/* We require CR0 and EFER. EFER is always up-to-date. */
13504	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
13505	AssertRCReturn(rc, rc);
13506
13507	/* We're playing with the host CPU state here, have to disable preemption or longjmp. */
13508	VMMRZCallRing3Disable(pVCpu);
13509	HM_DISABLE_PREEMPT();
13510
13511	/* If the guest FPU was active at the time of the #NM VM-exit, then it's a guest fault. */
13512	if (pVmxTransient->fWasGuestFPUStateActive)
13513	{
13514	rc = VINF_EM_RAW_GUEST_TRAP;
13515	Assert(CPUMIsGuestFPUStateActive(pVCpu) \|\| HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0));
13516	}
13517	else
13518	{
13519	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13520	Assert(!pVmxTransient->fWasGuestFPUStateActive \|\| pVCpu->hm.s.fUsingDebugLoop);
13521	#endif
13522	rc = CPUMR0Trap07Handler(pVCpu->CTX_SUFF(pVM), pVCpu);
13523	Assert( rc == VINF_EM_RAW_GUEST_TRAP
13524	\|\| ((rc == VINF_SUCCESS \|\| rc == VINF_CPUM_HOST_CR0_MODIFIED) && CPUMIsGuestFPUStateActive(pVCpu)));
13525	if (rc == VINF_CPUM_HOST_CR0_MODIFIED)
13526	HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT);
13527	}
13528
13529	HM_RESTORE_PREEMPT();
13530	VMMRZCallRing3Enable(pVCpu);
13531
13532	if (rc == VINF_SUCCESS \|\| rc == VINF_CPUM_HOST_CR0_MODIFIED)
13533	{
13534	/* Guest FPU state was activated, we'll want to change CR0 FPU intercepts before the next VM-reentry. */
13535	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
13536	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
13537	pVCpu->hm.s.fPreloadGuestFpu = true;
13538	}
13539	else
13540	{
13541	/* Forward #NM to the guest. */
13542	Assert(rc == VINF_EM_RAW_GUEST_TRAP);
13543	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13544	AssertRCReturn(rc, rc);
13545	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13546	pVmxTransient->cbInstr, 0 /* error code /, 0 / GCPtrFaultAddress */);
13547	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
13548	}
13549
13550	return VINF_SUCCESS;
13551	}
13552
13553
13554	/**
13555	* VM-exit exception handler for \#GP (General-protection exception).
13556	*
13557	* @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
13558	*/
13559	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13560	{
13561	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13562	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
13563
13564	int rc;
13565	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
13566	{ /* likely */ }
13567	else
13568	{
13569	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13570	Assert(pVCpu->hm.s.fUsingDebugLoop);
13571	#endif
13572	/* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
13573	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13574	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13575	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13576	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13577	AssertRCReturn(rc, rc);
13578	Log4(("#GP Gst: CS:RIP %04x:%08RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u TR=%#04x\n", pMixedCtx->cs.Sel, pMixedCtx->rip,
13579	pVmxTransient->uExitIntErrorCode, pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu), pMixedCtx->tr.Sel));
13580	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13581	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13582	return rc;
13583	}
13584
13585	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
13586	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
13587
13588	/* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
13589	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13590	AssertRCReturn(rc, rc);
13591
13592	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
13593	uint32_t cbOp = 0;
13594	PVM pVM = pVCpu->CTX_SUFF(pVM);
13595	bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
13596	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
13597	if (RT_SUCCESS(rc))
13598	{
13599	rc = VINF_SUCCESS;
13600	Assert(cbOp == pDis->cbInstr);
13601	Log4(("#GP Disas OpCode=%u CS:EIP %04x:%04RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
13602	switch (pDis->pCurInstr->uOpcode)
13603	{
13604	case OP_CLI:
13605	{
13606	pMixedCtx->eflags.Bits.u1IF = 0;
13607	pMixedCtx->eflags.Bits.u1RF = 0;
13608	pMixedCtx->rip += pDis->cbInstr;
13609	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13610	if ( !fDbgStepping
13611	&& pMixedCtx->eflags.Bits.u1TF)
13612	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13613	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
13614	break;
13615	}
13616
13617	case OP_STI:
13618	{
13619	bool fOldIF = pMixedCtx->eflags.Bits.u1IF;
13620	pMixedCtx->eflags.Bits.u1IF = 1;
13621	pMixedCtx->eflags.Bits.u1RF = 0;
13622	pMixedCtx->rip += pDis->cbInstr;
13623	if (!fOldIF)
13624	{
13625	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
13626	Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
13627	}
13628	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13629	if ( !fDbgStepping
13630	&& pMixedCtx->eflags.Bits.u1TF)
13631	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13632	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
13633	break;
13634	}
13635
13636	case OP_HLT:
13637	{
13638	rc = VINF_EM_HALT;
13639	pMixedCtx->rip += pDis->cbInstr;
13640	pMixedCtx->eflags.Bits.u1RF = 0;
13641	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13642	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
13643	break;
13644	}
13645
13646	case OP_POPF:
13647	{
13648	Log4(("POPF CS:EIP %04x:%04RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
13649	uint32_t cbParm;
13650	uint32_t uMask;
13651	bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
13652	if (pDis->fPrefix & DISPREFIX_OPSIZE)
13653	{
13654	cbParm = 4;
13655	uMask = 0xffffffff;
13656	}
13657	else
13658	{
13659	cbParm = 2;
13660	uMask = 0xffff;
13661	}
13662
13663	/* Get the stack pointer & pop the contents of the stack onto Eflags. */
13664	RTGCPTR GCPtrStack = 0;
13665	X86EFLAGS Eflags;
13666	Eflags.u32 = 0;
13667	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
13668	&GCPtrStack);
13669	if (RT_SUCCESS(rc))
13670	{
13671	Assert(sizeof(Eflags.u32) >= cbParm);
13672	rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm, PGMACCESSORIGIN_HM));
13673	AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13674	}
13675	if (RT_FAILURE(rc))
13676	{
13677	rc = VERR_EM_INTERPRETER;
13678	break;
13679	}
13680	Log4(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
13681	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~((X86_EFL_POPF_BITS & uMask) \| X86_EFL_RF))
13682	\| (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
13683	pMixedCtx->esp += cbParm;
13684	pMixedCtx->esp &= uMask;
13685	pMixedCtx->rip += pDis->cbInstr;
13686	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13687	\| HM_CHANGED_GUEST_RSP
13688	\| HM_CHANGED_GUEST_RFLAGS);
13689	/* Generate a pending-debug exception when the guest stepping over POPF regardless of how
13690	POPF restores EFLAGS.TF. */
13691	if ( !fDbgStepping
13692	&& fGstStepping)
13693	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13694	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
13695	break;
13696	}
13697
13698	case OP_PUSHF:
13699	{
13700	uint32_t cbParm;
13701	uint32_t uMask;
13702	if (pDis->fPrefix & DISPREFIX_OPSIZE)
13703	{
13704	cbParm = 4;
13705	uMask = 0xffffffff;
13706	}
13707	else
13708	{
13709	cbParm = 2;
13710	uMask = 0xffff;
13711	}
13712
13713	/* Get the stack pointer & push the contents of eflags onto the stack. */
13714	RTGCPTR GCPtrStack = 0;
13715	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
13716	SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
13717	if (RT_FAILURE(rc))
13718	{
13719	rc = VERR_EM_INTERPRETER;
13720	break;
13721	}
13722	X86EFLAGS Eflags = pMixedCtx->eflags;
13723	/* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
13724	Eflags.Bits.u1RF = 0;
13725	Eflags.Bits.u1VM = 0;
13726
13727	rc = VBOXSTRICTRC_TODO(PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm, PGMACCESSORIGIN_HM));
13728	if (RT_UNLIKELY(rc != VINF_SUCCESS))
13729	{
13730	AssertMsgFailed(("%Rrc\n", rc)); /** @todo allow strict return codes here */
13731	rc = VERR_EM_INTERPRETER;
13732	break;
13733	}
13734	Log4(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
13735	pMixedCtx->esp -= cbParm;
13736	pMixedCtx->esp &= uMask;
13737	pMixedCtx->rip += pDis->cbInstr;
13738	pMixedCtx->eflags.Bits.u1RF = 0;
13739	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13740	\| HM_CHANGED_GUEST_RSP
13741	\| HM_CHANGED_GUEST_RFLAGS);
13742	if ( !fDbgStepping
13743	&& pMixedCtx->eflags.Bits.u1TF)
13744	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13745	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
13746	break;
13747	}
13748
13749	case OP_IRET:
13750	{
13751	/** @todo Handle 32-bit operand sizes and check stack limits. See Intel
13752	* instruction reference. */
13753	RTGCPTR GCPtrStack = 0;
13754	uint32_t uMask = 0xffff;
13755	bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
13756	uint16_t aIretFrame[3];
13757	if (pDis->fPrefix & (DISPREFIX_OPSIZE \| DISPREFIX_ADDRSIZE))
13758	{
13759	rc = VERR_EM_INTERPRETER;
13760	break;
13761	}
13762	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
13763	&GCPtrStack);
13764	if (RT_SUCCESS(rc))
13765	{
13766	rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame),
13767	PGMACCESSORIGIN_HM));
13768	AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13769	}
13770	if (RT_FAILURE(rc))
13771	{
13772	rc = VERR_EM_INTERPRETER;
13773	break;
13774	}
13775	pMixedCtx->eip = 0;
13776	pMixedCtx->ip = aIretFrame[0];
13777	pMixedCtx->cs.Sel = aIretFrame[1];
13778	pMixedCtx->cs.ValidSel = aIretFrame[1];
13779	pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
13780	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ((UINT32_C(0xffff0000) \| X86_EFL_1) & ~X86_EFL_RF))
13781	\| (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
13782	pMixedCtx->sp += sizeof(aIretFrame);
13783	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13784	\| HM_CHANGED_GUEST_SEGMENT_REGS
13785	\| HM_CHANGED_GUEST_RSP
13786	\| HM_CHANGED_GUEST_RFLAGS);
13787	/* Generate a pending-debug exception when stepping over IRET regardless of how IRET modifies EFLAGS.TF. */
13788	if ( !fDbgStepping
13789	&& fGstStepping)
13790	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13791	Log4(("IRET %#RX32 to %04x:%04x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
13792	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
13793	break;
13794	}
13795
13796	case OP_INT:
13797	{
13798	uint16_t uVector = pDis->Param1.uValue & 0xff;
13799	hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
13800	/* INT clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13801	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13802	break;
13803	}
13804
13805	case OP_INTO:
13806	{
13807	if (pMixedCtx->eflags.Bits.u1OF)
13808	{
13809	hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
13810	/* INTO clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13811	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13812	}
13813	else
13814	{
13815	pMixedCtx->eflags.Bits.u1RF = 0;
13816	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
13817	}
13818	break;
13819	}
13820
13821	default:
13822	{
13823	pMixedCtx->eflags.Bits.u1RF = 0; /* This is correct most of the time... */
13824	VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
13825	EMCODETYPE_SUPERVISOR);
13826	rc = VBOXSTRICTRC_VAL(rc2);
13827	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
13828	/** @todo We have to set pending-debug exceptions here when the guest is
13829	* single-stepping depending on the instruction that was interpreted. */
13830	Log4(("#GP rc=%Rrc\n", rc));
13831	break;
13832	}
13833	}
13834	}
13835	else
13836	rc = VERR_EM_INTERPRETER;
13837
13838	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_EM_HALT,
13839	("#GP Unexpected rc=%Rrc\n", rc));
13840	return rc;
13841	}
13842
13843
13844	/**
13845	* VM-exit exception handler wrapper for generic exceptions. Simply re-injects
13846	* the exception reported in the VMX transient structure back into the VM.
13847	*
13848	* @remarks Requires uExitIntInfo in the VMX transient structure to be
13849	* up-to-date.
13850	*/
13851	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13852	{
13853	RT_NOREF_PV(pMixedCtx);
13854	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13855	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13856	AssertMsg(pVCpu->hm.s.fUsingDebugLoop \|\| pVCpu->hm.s.vmx.RealMode.fRealOnV86Active,
13857	("uVector=%#04x u32XcptBitmap=%#010RX32\n",
13858	VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo), pVCpu->hm.s.vmx.u32XcptBitmap));
13859	#endif
13860
13861	/* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
13862	hmR0VmxCheckExitDueToEventDelivery(). */
13863	int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13864	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13865	AssertRCReturn(rc, rc);
13866	Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
13867
13868	#ifdef DEBUG_ramshankar
13869	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
13870	uint8_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
13871	Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
13872	#endif
13873
13874	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13875	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13876	return VINF_SUCCESS;
13877	}
13878
13879
13880	/**
13881	* VM-exit exception handler for \#PF (Page-fault exception).
13882	*/
13883	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13884	{
13885	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13886	PVM pVM = pVCpu->CTX_SUFF(pVM);
13887	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13888	rc \|= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13889	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13890	AssertRCReturn(rc, rc);
13891
13892	if (!pVM->hm.s.fNestedPaging)
13893	{ /* likely */ }
13894	else
13895	{
13896	#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF)
13897	Assert(pVCpu->hm.s.fUsingDebugLoop);
13898	#endif
13899	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
13900	if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
13901	{
13902	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13903	0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
13904	}
13905	else
13906	{
13907	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13908	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
13909	Log4(("Pending #DF due to vectoring #PF. NP\n"));
13910	}
13911	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13912	return rc;
13913	}
13914
13915	/* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
13916	of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
13917	if (pVmxTransient->fVectoringPF)
13918	{
13919	Assert(pVCpu->hm.s.Event.fPending);
13920	return VINF_EM_RAW_INJECT_TRPM_EVENT;
13921	}
13922
13923	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13924	AssertRCReturn(rc, rc);
13925
13926	Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
13927	pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntErrorCode, pMixedCtx->cr3));
13928
13929	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
13930	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pMixedCtx),
13931	(RTGCPTR)pVmxTransient->uExitQualification);
13932
13933	Log4(("#PF: rc=%Rrc\n", rc));
13934	if (rc == VINF_SUCCESS)
13935	{
13936	#if 0
13937	/* Successfully synced shadow pages tables or emulated an MMIO instruction. */
13938	/** @todo this isn't quite right, what if guest does lgdt with some MMIO
13939	* memory? We don't update the whole state here... */
13940	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13941	\| HM_CHANGED_GUEST_RSP
13942	\| HM_CHANGED_GUEST_RFLAGS
13943	\| HM_CHANGED_VMX_GUEST_APIC_STATE);
13944	#else
13945	/*
13946	* This is typically a shadow page table sync or a MMIO instruction. But we may have
13947	* emulated something like LTR or a far jump. Any part of the CPU context may have changed.
13948	*/
13949	/** @todo take advantage of CPUM changed flags instead of brute forcing. */
13950	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
13951	#endif
13952	TRPMResetTrap(pVCpu);
13953	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
13954	return rc;
13955	}
13956
13957	if (rc == VINF_EM_RAW_GUEST_TRAP)
13958	{
13959	if (!pVmxTransient->fVectoringDoublePF)
13960	{
13961	/* It's a guest page fault and needs to be reflected to the guest. */
13962	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
13963	TRPMResetTrap(pVCpu);
13964	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
13965	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13966	0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
13967	}
13968	else
13969	{
13970	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13971	TRPMResetTrap(pVCpu);
13972	pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
13973	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
13974	Log4(("#PF: Pending #DF due to vectoring #PF\n"));
13975	}
13976
13977	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13978	return VINF_SUCCESS;
13979	}
13980
13981	TRPMResetTrap(pVCpu);
13982	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
13983	return rc;
13984	}
13985
13986	/** @} */
13987

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 70606

Download in other formats: