HMVMXR0.cpp@ 70687

Last change on this file since 70687 was 70612, checked in by vboxsync, 7 years ago
VMM: Expose PCID, INVPCID, FSGSBASE features to guests. Implemented the relevant instructions in IEM.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 598.8 KB

Line
1	/* $Id: HMVMXR0.cpp 70612 2018-01-17 18:12:23Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012-2017 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*********************************************************************************************************************************
20	* Header Files *
21	*********************************************************************************************************************************/
22	#define LOG_GROUP LOG_GROUP_HM
23	#include <iprt/x86.h>
24	#include <iprt/asm-amd64-x86.h>
25	#include <iprt/thread.h>
26
27	#include <VBox/vmm/pdmapi.h>
28	#include <VBox/vmm/dbgf.h>
29	#include <VBox/vmm/iem.h>
30	#include <VBox/vmm/iom.h>
31	#include <VBox/vmm/selm.h>
32	#include <VBox/vmm/tm.h>
33	#include <VBox/vmm/gim.h>
34	#include <VBox/vmm/apic.h>
35	#ifdef VBOX_WITH_REM
36	# include <VBox/vmm/rem.h>
37	#endif
38	#include "HMInternal.h"
39	#include <VBox/vmm/vm.h>
40	#include "HMVMXR0.h"
41	#include "dtrace/VBoxVMM.h"
42
43	#define HMVMX_USE_IEM_EVENT_REFLECTION
44	#ifdef DEBUG_ramshankar
45	# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
46	# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
47	# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
48	# define HMVMX_ALWAYS_CHECK_GUEST_STATE
49	# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
50	# define HMVMX_ALWAYS_TRAP_PF
51	# define HMVMX_ALWAYS_SWAP_FPU_STATE
52	# define HMVMX_ALWAYS_FLUSH_TLB
53	# define HMVMX_ALWAYS_SWAP_EFER
54	#endif
55
56
57	/*********************************************************************************************************************************
58	* Defined Constants And Macros *
59	*********************************************************************************************************************************/
60	/** Use the function table. */
61	#define HMVMX_USE_FUNCTION_TABLE
62
63	/** Determine which tagged-TLB flush handler to use. */
64	#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
65	#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
66	#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
67	#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
68
69	/** @name Updated-guest-state flags.
70	* @{ */
71	#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
72	#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
73	#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
74	#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
75	#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
76	#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
77	#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
78	#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
79	#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
80	#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
81	#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
82	#define HMVMX_UPDATED_GUEST_DR7 RT_BIT(11)
83	#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(12)
84	#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(13)
85	#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(14)
86	#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(15)
87	#define HMVMX_UPDATED_GUEST_LAZY_MSRS RT_BIT(16)
88	#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(17)
89	#define HMVMX_UPDATED_GUEST_INTR_STATE RT_BIT(18)
90	#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(19)
91	#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
92	\| HMVMX_UPDATED_GUEST_RSP \
93	\| HMVMX_UPDATED_GUEST_RFLAGS \
94	\| HMVMX_UPDATED_GUEST_CR0 \
95	\| HMVMX_UPDATED_GUEST_CR3 \
96	\| HMVMX_UPDATED_GUEST_CR4 \
97	\| HMVMX_UPDATED_GUEST_GDTR \
98	\| HMVMX_UPDATED_GUEST_IDTR \
99	\| HMVMX_UPDATED_GUEST_LDTR \
100	\| HMVMX_UPDATED_GUEST_TR \
101	\| HMVMX_UPDATED_GUEST_SEGMENT_REGS \
102	\| HMVMX_UPDATED_GUEST_DR7 \
103	\| HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
104	\| HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
105	\| HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
106	\| HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
107	\| HMVMX_UPDATED_GUEST_LAZY_MSRS \
108	\| HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
109	\| HMVMX_UPDATED_GUEST_INTR_STATE \
110	\| HMVMX_UPDATED_GUEST_APIC_STATE)
111	/** @} */
112
113	/** @name
114	* Flags to skip redundant reads of some common VMCS fields that are not part of
115	* the guest-CPU state but are in the transient structure.
116	*/
117	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
118	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
119	#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
120	#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
121	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
122	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
123	#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO RT_BIT(6)
124	/** @} */
125
126	/** @name
127	* States of the VMCS.
128	*
129	* This does not reflect all possible VMCS states but currently only those
130	* needed for maintaining the VMCS consistently even when thread-context hooks
131	* are used. Maybe later this can be extended (i.e. Nested Virtualization).
132	*/
133	#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
134	#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
135	#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
136	/** @} */
137
138	/**
139	* Exception bitmap mask for real-mode guests (real-on-v86).
140	*
141	* We need to intercept all exceptions manually except:
142	* - \#NM, \#MF handled in hmR0VmxLoadSharedCR0().
143	* - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
144	* due to bugs in Intel CPUs.
145	* - \#PF need not be intercepted even in real-mode if we have Nested Paging
146	* support.
147	*/
148	#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: \| RT_BIT(X86_XCPT_DB) */ \| RT_BIT(X86_XCPT_NMI) \
149	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
150	\| RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ \| RT_BIT(X86_XCPT_DF) \
151	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
152	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
153	/* RT_BIT(X86_XCPT_MF) always: \| RT_BIT(X86_XCPT_AC) */ \| RT_BIT(X86_XCPT_MC) \
154	\| RT_BIT(X86_XCPT_XF))
155
156	/**
157	* Exception bitmap mask for all contributory exceptions.
158	*
159	* Page fault is deliberately excluded here as it's conditional as to whether
160	* it's contributory or benign. Page faults are handled separately.
161	*/
162	#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) \| RT_BIT(X86_XCPT_NP) \| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_TS) \
163	\| RT_BIT(X86_XCPT_DE))
164
165	/** Maximum VM-instruction error number. */
166	#define HMVMX_INSTR_ERROR_MAX 28
167
168	/** Profiling macro. */
169	#ifdef HM_PROFILE_EXIT_DISPATCH
170	# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
171	# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
172	#else
173	# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
174	# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
175	#endif
176
177	/** Assert that preemption is disabled or covered by thread-context hooks. */
178	#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
179	\|\| !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
180
181	/** Assert that we haven't migrated CPUs when thread-context hooks are not
182	* used. */
183	#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
184	\|\| pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
185	("Illegal migration! Entered on CPU %u Current %u\n", \
186	pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
187
188	/** Helper macro for VM-exit handlers called unexpectedly. */
189	#define HMVMX_RETURN_UNEXPECTED_EXIT() \
190	do { \
191	pVCpu->hm.s.u32HMError = pVmxTransient->uExitReason; \
192	return VERR_VMX_UNEXPECTED_EXIT; \
193	} while (0)
194
195
196	/*********************************************************************************************************************************
197	* Structures and Typedefs *
198	*********************************************************************************************************************************/
199	/**
200	* VMX transient state.
201	*
202	* A state structure for holding miscellaneous information across
203	* VMX non-root operation and restored after the transition.
204	*/
205	typedef struct VMXTRANSIENT
206	{
207	/** The host's rflags/eflags. */
208	RTCCUINTREG fEFlags;
209	#if HC_ARCH_BITS == 32
210	uint32_t u32Alignment0;
211	#endif
212	/** The guest's TPR value used for TPR shadowing. */
213	uint8_t u8GuestTpr;
214	/** Alignment. */
215	uint8_t abAlignment0[7];
216
217	/** The basic VM-exit reason. */
218	uint16_t uExitReason;
219	/** Alignment. */
220	uint16_t u16Alignment0;
221	/** The VM-exit interruption error code. */
222	uint32_t uExitIntErrorCode;
223	/** The VM-exit exit code qualification. */
224	uint64_t uExitQualification;
225
226	/** The VM-exit interruption-information field. */
227	uint32_t uExitIntInfo;
228	/** The VM-exit instruction-length field. */
229	uint32_t cbInstr;
230	/** The VM-exit instruction-information field. */
231	union
232	{
233	/** Plain unsigned int representation. */
234	uint32_t u;
235	/** INS and OUTS information. */
236	struct
237	{
238	uint32_t u7Reserved0 : 7;
239	/** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
240	uint32_t u3AddrSize : 3;
241	uint32_t u5Reserved1 : 5;
242	/** The segment register (X86_SREG_XXX). */
243	uint32_t iSegReg : 3;
244	uint32_t uReserved2 : 14;
245	} StrIo;
246	/** INVEPT, INVVPID, INVPCID information. */
247	struct
248	{
249	/** Scaling; 0=no scaling, 1=scale-by-2, 2=scale-by-4, 3=scale-by-8. */
250	uint32_t u2Scaling : 2;
251	uint32_t u5Reserved0 : 5;
252	/** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
253	uint32_t u3AddrSize : 3;
254	uint32_t u1Reserved0 : 1;
255	uint32_t u4Reserved0 : 4;
256	/** The segment register (X86_SREG_XXX). */
257	uint32_t iSegReg : 3;
258	/** The index register (X86_GREG_XXX). */
259	uint32_t iIdxReg : 4;
260	/** Set if index register is invalid. */
261	uint32_t fIdxRegValid : 1;
262	/** The base register (X86_GREG_XXX). */
263	uint32_t iBaseReg : 4;
264	/** Set if base register is invalid. */
265	uint32_t fBaseRegValid : 1;
266	/** Register 2 (X86_GREG_XXX). */
267	uint32_t iReg2 : 4;
268	} Inv;
269	} ExitInstrInfo;
270	/** Whether the VM-entry failed or not. */
271	bool fVMEntryFailed;
272	/** Alignment. */
273	uint8_t abAlignment1[3];
274
275	/** The VM-entry interruption-information field. */
276	uint32_t uEntryIntInfo;
277	/** The VM-entry exception error code field. */
278	uint32_t uEntryXcptErrorCode;
279	/** The VM-entry instruction length field. */
280	uint32_t cbEntryInstr;
281
282	/** IDT-vectoring information field. */
283	uint32_t uIdtVectoringInfo;
284	/** IDT-vectoring error code. */
285	uint32_t uIdtVectoringErrorCode;
286
287	/** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_. /
288	uint32_t fVmcsFieldsRead;
289
290	/** Whether the guest FPU was active at the time of VM-exit. */
291	bool fWasGuestFPUStateActive;
292	/** Whether the guest debug state was active at the time of VM-exit. */
293	bool fWasGuestDebugStateActive;
294	/** Whether the hyper debug state was active at the time of VM-exit. */
295	bool fWasHyperDebugStateActive;
296	/** Whether TSC-offsetting should be setup before VM-entry. */
297	bool fUpdateTscOffsettingAndPreemptTimer;
298	/** Whether the VM-exit was caused by a page-fault during delivery of a
299	* contributory exception or a page-fault. */
300	bool fVectoringDoublePF;
301	/** Whether the VM-exit was caused by a page-fault during delivery of an
302	* external interrupt or NMI. */
303	bool fVectoringPF;
304	} VMXTRANSIENT;
305	AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
306	AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
307	AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
308	AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestFPUStateActive, sizeof(uint64_t));
309	AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
310	/** Pointer to VMX transient state. */
311	typedef VMXTRANSIENT *PVMXTRANSIENT;
312
313
314	/**
315	* MSR-bitmap read permissions.
316	*/
317	typedef enum VMXMSREXITREAD
318	{
319	/** Reading this MSR causes a VM-exit. */
320	VMXMSREXIT_INTERCEPT_READ = 0xb,
321	/** Reading this MSR does not cause a VM-exit. */
322	VMXMSREXIT_PASSTHRU_READ
323	} VMXMSREXITREAD;
324	/** Pointer to MSR-bitmap read permissions. */
325	typedef VMXMSREXITREAD* PVMXMSREXITREAD;
326
327	/**
328	* MSR-bitmap write permissions.
329	*/
330	typedef enum VMXMSREXITWRITE
331	{
332	/** Writing to this MSR causes a VM-exit. */
333	VMXMSREXIT_INTERCEPT_WRITE = 0xd,
334	/** Writing to this MSR does not cause a VM-exit. */
335	VMXMSREXIT_PASSTHRU_WRITE
336	} VMXMSREXITWRITE;
337	/** Pointer to MSR-bitmap write permissions. */
338	typedef VMXMSREXITWRITE* PVMXMSREXITWRITE;
339
340
341	/**
342	* VMX VM-exit handler.
343	*
344	* @returns Strict VBox status code (i.e. informational status codes too).
345	* @param pVCpu The cross context virtual CPU structure.
346	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
347	* out-of-sync. Make sure to update the required
348	* fields before using them.
349	* @param pVmxTransient Pointer to the VMX-transient structure.
350	*/
351	#ifndef HMVMX_USE_FUNCTION_TABLE
352	typedef VBOXSTRICTRC FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
353	#else
354	typedef DECLCALLBACK(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
355	/** Pointer to VM-exit handler. */
356	typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
357	#endif
358
359	/**
360	* VMX VM-exit handler, non-strict status code.
361	*
362	* This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
363	*
364	* @returns VBox status code, no informational status code returned.
365	* @param pVCpu The cross context virtual CPU structure.
366	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
367	* out-of-sync. Make sure to update the required
368	* fields before using them.
369	* @param pVmxTransient Pointer to the VMX-transient structure.
370	*
371	* @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
372	* use of that status code will be replaced with VINF_EM_SOMETHING
373	* later when switching over to IEM.
374	*/
375	#ifndef HMVMX_USE_FUNCTION_TABLE
376	typedef int FNVMXEXITHANDLERNSRC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
377	#else
378	typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
379	#endif
380
381
382	/*********************************************************************************************************************************
383	* Internal Functions *
384	*********************************************************************************************************************************/
385	static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush);
386	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr);
387	static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu);
388	static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
389	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress,
390	bool fStepping, uint32_t *puIntState);
391	#if HC_ARCH_BITS == 32
392	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
393	#endif
394	#ifndef HMVMX_USE_FUNCTION_TABLE
395	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
396	# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
397	# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
398	#else
399	# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
400	# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
401	#endif
402
403
404	/** @name VM-exit handlers.
405	* @{
406	*/
407	static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
408	static FNVMXEXITHANDLER hmR0VmxExitExtInt;
409	static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
410	static FNVMXEXITHANDLERNSRC hmR0VmxExitInitSignal;
411	static FNVMXEXITHANDLERNSRC hmR0VmxExitSipi;
412	static FNVMXEXITHANDLERNSRC hmR0VmxExitIoSmi;
413	static FNVMXEXITHANDLERNSRC hmR0VmxExitSmi;
414	static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindow;
415	static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindow;
416	static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
417	static FNVMXEXITHANDLER hmR0VmxExitCpuid;
418	static FNVMXEXITHANDLER hmR0VmxExitGetsec;
419	static FNVMXEXITHANDLER hmR0VmxExitHlt;
420	static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
421	static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
422	static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
423	static FNVMXEXITHANDLER hmR0VmxExitVmcall;
424	static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
425	static FNVMXEXITHANDLERNSRC hmR0VmxExitRsm;
426	static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
427	static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
428	static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
429	static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
430	static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
431	static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
432	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
433	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMsrLoad;
434	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUndefined;
435	static FNVMXEXITHANDLER hmR0VmxExitMwait;
436	static FNVMXEXITHANDLER hmR0VmxExitMtf;
437	static FNVMXEXITHANDLER hmR0VmxExitMonitor;
438	static FNVMXEXITHANDLER hmR0VmxExitPause;
439	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMachineCheck;
440	static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThreshold;
441	static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
442	static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
443	static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
444	static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
445	static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
446	static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
447	static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
448	static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvd;
449	static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
450	static FNVMXEXITHANDLER hmR0VmxExitRdrand;
451	static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
452	/** @} */
453
454	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
455	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
456	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
457	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
458	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
459	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
460	static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
461	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
462	static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
463
464
465	/*********************************************************************************************************************************
466	* Global Variables *
467	*********************************************************************************************************************************/
468	#ifdef HMVMX_USE_FUNCTION_TABLE
469
470	/**
471	* VMX_EXIT dispatch table.
472	*/
473	static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
474	{
475	/* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
476	/* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
477	/* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
478	/* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
479	/* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
480	/* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
481	/* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
482	/* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
483	/* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
484	/* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
485	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
486	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
487	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
488	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
489	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
490	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
491	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
492	/* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
493	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
494	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
495	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
496	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
497	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
498	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
499	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
500	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
501	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
502	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
503	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
504	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
505	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
506	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
507	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
508	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
509	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
510	/* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
511	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
512	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
513	/* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
514	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
515	/* 40 UNDEFINED */ hmR0VmxExitPause,
516	/* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
517	/* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
518	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
519	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
520	/* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
521	/* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
522	/* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
523	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
524	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
525	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
526	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
527	/* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
528	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
529	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
530	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
531	/* 56 VMX_EXIT_APIC_WRITE */ hmR0VmxExitErrUndefined,
532	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
533	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
534	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD,
535	/* 60 VMX_EXIT_ENCLS */ hmR0VmxExitErrUndefined,
536	/* 61 VMX_EXIT_RDSEED / hmR0VmxExitErrUndefined, / only spurious exits, so undefined */
537	/* 62 VMX_EXIT_PML_FULL */ hmR0VmxExitErrUndefined,
538	/* 63 VMX_EXIT_XSAVES */ hmR0VmxExitSetPendingXcptUD,
539	/* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitSetPendingXcptUD,
540	};
541	#endif /* HMVMX_USE_FUNCTION_TABLE */
542
543	#ifdef VBOX_STRICT
544	static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
545	{
546	/* 0 */ "(Not Used)",
547	/* 1 */ "VMCALL executed in VMX root operation.",
548	/* 2 */ "VMCLEAR with invalid physical address.",
549	/* 3 */ "VMCLEAR with VMXON pointer.",
550	/* 4 */ "VMLAUNCH with non-clear VMCS.",
551	/* 5 */ "VMRESUME with non-launched VMCS.",
552	/* 6 */ "VMRESUME after VMXOFF",
553	/* 7 */ "VM-entry with invalid control fields.",
554	/* 8 */ "VM-entry with invalid host state fields.",
555	/* 9 */ "VMPTRLD with invalid physical address.",
556	/* 10 */ "VMPTRLD with VMXON pointer.",
557	/* 11 */ "VMPTRLD with incorrect revision identifier.",
558	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
559	/* 13 */ "VMWRITE to read-only VMCS component.",
560	/* 14 */ "(Not Used)",
561	/* 15 */ "VMXON executed in VMX root operation.",
562	/* 16 */ "VM-entry with invalid executive-VMCS pointer.",
563	/* 17 */ "VM-entry with non-launched executing VMCS.",
564	/* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
565	/* 19 */ "VMCALL with non-clear VMCS.",
566	/* 20 */ "VMCALL with invalid VM-exit control fields.",
567	/* 21 */ "(Not Used)",
568	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
569	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
570	/* 24 */ "VMCALL with invalid SMM-monitor features.",
571	/* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
572	/* 26 */ "VM-entry with events blocked by MOV SS.",
573	/* 27 */ "(Not Used)",
574	/* 28 */ "Invalid operand to INVEPT/INVVPID."
575	};
576	#endif /* VBOX_STRICT */
577
578
579
580	/**
581	* Updates the VM's last error record.
582	*
583	* If there was a VMX instruction error, reads the error data from the VMCS and
584	* updates VCPU's last error record as well.
585	*
586	* @param pVM The cross context VM structure.
587	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
588	* Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
589	* VERR_VMX_INVALID_VMCS_FIELD.
590	* @param rc The error code.
591	*/
592	static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
593	{
594	AssertPtr(pVM);
595	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
596	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
597	{
598	AssertPtrReturnVoid(pVCpu);
599	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
600	}
601	pVM->hm.s.lLastError = rc;
602	}
603
604
605	/**
606	* Reads the VM-entry interruption-information field from the VMCS into the VMX
607	* transient structure.
608	*
609	* @returns VBox status code.
610	* @param pVmxTransient Pointer to the VMX transient structure.
611	*
612	* @remarks No-long-jump zone!!!
613	*/
614	DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
615	{
616	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
617	AssertRCReturn(rc, rc);
618	return VINF_SUCCESS;
619	}
620
621
622	#ifdef VBOX_STRICT
623	/**
624	* Reads the VM-entry exception error code field from the VMCS into
625	* the VMX transient structure.
626	*
627	* @returns VBox status code.
628	* @param pVmxTransient Pointer to the VMX transient structure.
629	*
630	* @remarks No-long-jump zone!!!
631	*/
632	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
633	{
634	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
635	AssertRCReturn(rc, rc);
636	return VINF_SUCCESS;
637	}
638	#endif /* VBOX_STRICT */
639
640
641	#ifdef VBOX_STRICT
642	/**
643	* Reads the VM-entry exception error code field from the VMCS into
644	* the VMX transient structure.
645	*
646	* @returns VBox status code.
647	* @param pVmxTransient Pointer to the VMX transient structure.
648	*
649	* @remarks No-long-jump zone!!!
650	*/
651	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
652	{
653	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
654	AssertRCReturn(rc, rc);
655	return VINF_SUCCESS;
656	}
657	#endif /* VBOX_STRICT */
658
659
660	/**
661	* Reads the VM-exit interruption-information field from the VMCS into the VMX
662	* transient structure.
663	*
664	* @returns VBox status code.
665	* @param pVmxTransient Pointer to the VMX transient structure.
666	*/
667	DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
668	{
669	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
670	{
671	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
672	AssertRCReturn(rc, rc);
673	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
674	}
675	return VINF_SUCCESS;
676	}
677
678
679	/**
680	* Reads the VM-exit interruption error code from the VMCS into the VMX
681	* transient structure.
682	*
683	* @returns VBox status code.
684	* @param pVmxTransient Pointer to the VMX transient structure.
685	*/
686	DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
687	{
688	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
689	{
690	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
691	AssertRCReturn(rc, rc);
692	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
693	}
694	return VINF_SUCCESS;
695	}
696
697
698	/**
699	* Reads the VM-exit instruction length field from the VMCS into the VMX
700	* transient structure.
701	*
702	* @returns VBox status code.
703	* @param pVmxTransient Pointer to the VMX transient structure.
704	*/
705	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
706	{
707	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
708	{
709	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
710	AssertRCReturn(rc, rc);
711	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
712	}
713	return VINF_SUCCESS;
714	}
715
716
717	/**
718	* Reads the VM-exit instruction-information field from the VMCS into
719	* the VMX transient structure.
720	*
721	* @returns VBox status code.
722	* @param pVmxTransient Pointer to the VMX transient structure.
723	*/
724	DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
725	{
726	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO))
727	{
728	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
729	AssertRCReturn(rc, rc);
730	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO;
731	}
732	return VINF_SUCCESS;
733	}
734
735
736	/**
737	* Reads the exit code qualification from the VMCS into the VMX transient
738	* structure.
739	*
740	* @returns VBox status code.
741	* @param pVCpu The cross context virtual CPU structure of the
742	* calling EMT. (Required for the VMCS cache case.)
743	* @param pVmxTransient Pointer to the VMX transient structure.
744	*/
745	DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
746	{
747	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
748	{
749	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification); NOREF(pVCpu);
750	AssertRCReturn(rc, rc);
751	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
752	}
753	return VINF_SUCCESS;
754	}
755
756
757	/**
758	* Reads the IDT-vectoring information field from the VMCS into the VMX
759	* transient structure.
760	*
761	* @returns VBox status code.
762	* @param pVmxTransient Pointer to the VMX transient structure.
763	*
764	* @remarks No-long-jump zone!!!
765	*/
766	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
767	{
768	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
769	{
770	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
771	AssertRCReturn(rc, rc);
772	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
773	}
774	return VINF_SUCCESS;
775	}
776
777
778	/**
779	* Reads the IDT-vectoring error code from the VMCS into the VMX
780	* transient structure.
781	*
782	* @returns VBox status code.
783	* @param pVmxTransient Pointer to the VMX transient structure.
784	*/
785	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
786	{
787	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
788	{
789	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
790	AssertRCReturn(rc, rc);
791	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
792	}
793	return VINF_SUCCESS;
794	}
795
796
797	/**
798	* Enters VMX root mode operation on the current CPU.
799	*
800	* @returns VBox status code.
801	* @param pVM The cross context VM structure. Can be
802	* NULL, after a resume.
803	* @param HCPhysCpuPage Physical address of the VMXON region.
804	* @param pvCpuPage Pointer to the VMXON region.
805	*/
806	static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
807	{
808	Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
809	Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
810	Assert(pvCpuPage);
811	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
812
813	if (pVM)
814	{
815	/* Write the VMCS revision dword to the VMXON region. */
816	(uint32_t )pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
817	}
818
819	/* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
820	RTCCUINTREG fEFlags = ASMIntDisableFlags();
821
822	/* Enable the VMX bit in CR4 if necessary. */
823	RTCCUINTREG uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
824
825	/* Enter VMX root mode. */
826	int rc = VMXEnable(HCPhysCpuPage);
827	if (RT_FAILURE(rc))
828	{
829	if (!(uOldCr4 & X86_CR4_VMXE))
830	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
831
832	if (pVM)
833	pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
834	}
835
836	/* Restore interrupts. */
837	ASMSetFlags(fEFlags);
838	return rc;
839	}
840
841
842	/**
843	* Exits VMX root mode operation on the current CPU.
844	*
845	* @returns VBox status code.
846	*/
847	static int hmR0VmxLeaveRootMode(void)
848	{
849	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
850
851	/* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
852	RTCCUINTREG fEFlags = ASMIntDisableFlags();
853
854	/* If we're for some reason not in VMX root mode, then don't leave it. */
855	RTCCUINTREG uHostCR4 = ASMGetCR4();
856
857	int rc;
858	if (uHostCR4 & X86_CR4_VMXE)
859	{
860	/* Exit VMX root mode and clear the VMX bit in CR4. */
861	VMXDisable();
862	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
863	rc = VINF_SUCCESS;
864	}
865	else
866	rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
867
868	/* Restore interrupts. */
869	ASMSetFlags(fEFlags);
870	return rc;
871	}
872
873
874	/**
875	* Allocates and maps one physically contiguous page. The allocated page is
876	* zero'd out. (Used by various VT-x structures).
877	*
878	* @returns IPRT status code.
879	* @param pMemObj Pointer to the ring-0 memory object.
880	* @param ppVirt Where to store the virtual address of the
881	* allocation.
882	* @param pHCPhys Where to store the physical address of the
883	* allocation.
884	*/
885	DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
886	{
887	AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
888	AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
889	AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
890
891	int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
892	if (RT_FAILURE(rc))
893	return rc;
894	ppVirt = RTR0MemObjAddress(pMemObj);
895	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
896	ASMMemZero32(*ppVirt, PAGE_SIZE);
897	return VINF_SUCCESS;
898	}
899
900
901	/**
902	* Frees and unmaps an allocated physical page.
903	*
904	* @param pMemObj Pointer to the ring-0 memory object.
905	* @param ppVirt Where to re-initialize the virtual address of
906	* allocation as 0.
907	* @param pHCPhys Where to re-initialize the physical address of the
908	* allocation as 0.
909	*/
910	DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
911	{
912	AssertPtr(pMemObj);
913	AssertPtr(ppVirt);
914	AssertPtr(pHCPhys);
915	if (*pMemObj != NIL_RTR0MEMOBJ)
916	{
917	int rc = RTR0MemObjFree(pMemObj, true / fFreeMappings */);
918	AssertRC(rc);
919	*pMemObj = NIL_RTR0MEMOBJ;
920	*ppVirt = 0;
921	*pHCPhys = 0;
922	}
923	}
924
925
926	/**
927	* Worker function to free VT-x related structures.
928	*
929	* @returns IPRT status code.
930	* @param pVM The cross context VM structure.
931	*/
932	static void hmR0VmxStructsFree(PVM pVM)
933	{
934	for (VMCPUID i = 0; i < pVM->cCpus; i++)
935	{
936	PVMCPU pVCpu = &pVM->aCpus[i];
937	AssertPtr(pVCpu);
938
939	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
940	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
941
942	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
943	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
944
945	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
946	}
947
948	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
949	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
950	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
951	#endif
952	}
953
954
955	/**
956	* Worker function to allocate VT-x related VM structures.
957	*
958	* @returns IPRT status code.
959	* @param pVM The cross context VM structure.
960	*/
961	static int hmR0VmxStructsAlloc(PVM pVM)
962	{
963	/*
964	* Initialize members up-front so we can cleanup properly on allocation failure.
965	*/
966	#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
967	pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
968	pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
969	pVM->hm.s.vmx.HCPhys##a_Name = 0;
970
971	#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
972	pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
973	pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
974	pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
975
976	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
977	VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
978	#endif
979	VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
980
981	AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
982	for (VMCPUID i = 0; i < pVM->cCpus; i++)
983	{
984	PVMCPU pVCpu = &pVM->aCpus[i];
985	VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
986	VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
987	VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
988	VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
989	}
990	#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
991	#undef VMXLOCAL_INIT_VM_MEMOBJ
992
993	/* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
994	AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
995	(&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
996	VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
997
998	/*
999	* Allocate all the VT-x structures.
1000	*/
1001	int rc = VINF_SUCCESS;
1002	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1003	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
1004	if (RT_FAILURE(rc))
1005	goto cleanup;
1006	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
1007	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
1008	#endif
1009
1010	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
1011	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
1012	{
1013	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
1014	&pVM->hm.s.vmx.HCPhysApicAccess);
1015	if (RT_FAILURE(rc))
1016	goto cleanup;
1017	}
1018
1019	/*
1020	* Initialize per-VCPU VT-x structures.
1021	*/
1022	for (VMCPUID i = 0; i < pVM->cCpus; i++)
1023	{
1024	PVMCPU pVCpu = &pVM->aCpus[i];
1025	AssertPtr(pVCpu);
1026
1027	/* Allocate the VM control structure (VMCS). */
1028	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
1029	if (RT_FAILURE(rc))
1030	goto cleanup;
1031
1032	/* Get the allocated virtual-APIC page from the APIC device for transparent TPR accesses. */
1033	if ( PDMHasApic(pVM)
1034	&& (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW))
1035	{
1036	rc = APICGetApicPageForCpu(pVCpu, &pVCpu->hm.s.vmx.HCPhysVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
1037	NULL /* pR3Ptr /, NULL / pRCPtr */);
1038	if (RT_FAILURE(rc))
1039	goto cleanup;
1040	}
1041
1042	/*
1043	* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
1044	* transparent accesses of specific MSRs.
1045	*
1046	* If the condition for enabling MSR bitmaps changes here, don't forget to
1047	* update HMAreMsrBitmapsAvailable().
1048	*/
1049	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1050	{
1051	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
1052	&pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1053	if (RT_FAILURE(rc))
1054	goto cleanup;
1055	ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
1056	}
1057
1058	/* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
1059	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
1060	if (RT_FAILURE(rc))
1061	goto cleanup;
1062
1063	/* Allocate the VM-exit MSR-load page for the host MSRs. */
1064	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
1065	if (RT_FAILURE(rc))
1066	goto cleanup;
1067	}
1068
1069	return VINF_SUCCESS;
1070
1071	cleanup:
1072	hmR0VmxStructsFree(pVM);
1073	return rc;
1074	}
1075
1076
1077	/**
1078	* Does global VT-x initialization (called during module initialization).
1079	*
1080	* @returns VBox status code.
1081	*/
1082	VMMR0DECL(int) VMXR0GlobalInit(void)
1083	{
1084	#ifdef HMVMX_USE_FUNCTION_TABLE
1085	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1086	# ifdef VBOX_STRICT
1087	for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1088	Assert(g_apfnVMExitHandlers[i]);
1089	# endif
1090	#endif
1091	return VINF_SUCCESS;
1092	}
1093
1094
1095	/**
1096	* Does global VT-x termination (called during module termination).
1097	*/
1098	VMMR0DECL(void) VMXR0GlobalTerm()
1099	{
1100	/* Nothing to do currently. */
1101	}
1102
1103
1104	/**
1105	* Sets up and activates VT-x on the current CPU.
1106	*
1107	* @returns VBox status code.
1108	* @param pCpu Pointer to the global CPU info struct.
1109	* @param pVM The cross context VM structure. Can be
1110	* NULL after a host resume operation.
1111	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1112	* fEnabledByHost is @c true).
1113	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1114	* @a fEnabledByHost is @c true).
1115	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1116	* enable VT-x on the host.
1117	* @param pvMsrs Opaque pointer to VMXMSRS struct.
1118	*/
1119	VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1120	void *pvMsrs)
1121	{
1122	Assert(pCpu);
1123	Assert(pvMsrs);
1124	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1125
1126	/* Enable VT-x if it's not already enabled by the host. */
1127	if (!fEnabledByHost)
1128	{
1129	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1130	if (RT_FAILURE(rc))
1131	return rc;
1132	}
1133
1134	/*
1135	* Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been using EPTPs) so
1136	* we don't retain any stale guest-physical mappings which won't get invalidated when flushing by VPID.
1137	*/
1138	PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1139	if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1140	{
1141	hmR0VmxFlushEpt(NULL /* pVCpu */, VMXFLUSHEPT_ALL_CONTEXTS);
1142	pCpu->fFlushAsidBeforeUse = false;
1143	}
1144	else
1145	pCpu->fFlushAsidBeforeUse = true;
1146
1147	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1148	++pCpu->cTlbFlushes;
1149
1150	return VINF_SUCCESS;
1151	}
1152
1153
1154	/**
1155	* Deactivates VT-x on the current CPU.
1156	*
1157	* @returns VBox status code.
1158	* @param pCpu Pointer to the global CPU info struct.
1159	* @param pvCpuPage Pointer to the VMXON region.
1160	* @param HCPhysCpuPage Physical address of the VMXON region.
1161	*
1162	* @remarks This function should never be called when SUPR0EnableVTx() or
1163	* similar was used to enable VT-x on the host.
1164	*/
1165	VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1166	{
1167	NOREF(pCpu);
1168	NOREF(pvCpuPage);
1169	NOREF(HCPhysCpuPage);
1170
1171	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1172	return hmR0VmxLeaveRootMode();
1173	}
1174
1175
1176	/**
1177	* Sets the permission bits for the specified MSR in the MSR bitmap.
1178	*
1179	* @param pVCpu The cross context virtual CPU structure.
1180	* @param uMsr The MSR value.
1181	* @param enmRead Whether reading this MSR causes a VM-exit.
1182	* @param enmWrite Whether writing this MSR causes a VM-exit.
1183	*/
1184	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1185	{
1186	int32_t iBit;
1187	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
1188
1189	/*
1190	* Layout:
1191	* 0x000 - 0x3ff - Low MSR read bits
1192	* 0x400 - 0x7ff - High MSR read bits
1193	* 0x800 - 0xbff - Low MSR write bits
1194	* 0xc00 - 0xfff - High MSR write bits
1195	*/
1196	if (uMsr <= 0x00001FFF)
1197	iBit = uMsr;
1198	else if (uMsr - UINT32_C(0xC0000000) <= UINT32_C(0x00001FFF))
1199	{
1200	iBit = uMsr - UINT32_C(0xC0000000);
1201	pbMsrBitmap += 0x400;
1202	}
1203	else
1204	AssertMsgFailedReturnVoid(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1205
1206	Assert(iBit <= 0x1fff);
1207	if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1208	ASMBitSet(pbMsrBitmap, iBit);
1209	else
1210	ASMBitClear(pbMsrBitmap, iBit);
1211
1212	if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1213	ASMBitSet(pbMsrBitmap + 0x800, iBit);
1214	else
1215	ASMBitClear(pbMsrBitmap + 0x800, iBit);
1216	}
1217
1218
1219	#ifdef VBOX_STRICT
1220	/**
1221	* Gets the permission bits for the specified MSR in the MSR bitmap.
1222	*
1223	* @returns VBox status code.
1224	* @retval VINF_SUCCESS if the specified MSR is found.
1225	* @retval VERR_NOT_FOUND if the specified MSR is not found.
1226	* @retval VERR_NOT_SUPPORTED if VT-x doesn't allow the MSR.
1227	*
1228	* @param pVCpu The cross context virtual CPU structure.
1229	* @param uMsr The MSR.
1230	* @param penmRead Where to store the read permissions.
1231	* @param penmWrite Where to store the write permissions.
1232	*/
1233	static int hmR0VmxGetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, PVMXMSREXITREAD penmRead, PVMXMSREXITWRITE penmWrite)
1234	{
1235	AssertPtrReturn(penmRead, VERR_INVALID_PARAMETER);
1236	AssertPtrReturn(penmWrite, VERR_INVALID_PARAMETER);
1237	int32_t iBit;
1238	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
1239
1240	/* See hmR0VmxSetMsrPermission() for the layout. */
1241	if (uMsr <= 0x00001FFF)
1242	iBit = uMsr;
1243	else if ( uMsr >= 0xC0000000
1244	&& uMsr <= 0xC0001FFF)
1245	{
1246	iBit = (uMsr - 0xC0000000);
1247	pbMsrBitmap += 0x400;
1248	}
1249	else
1250	AssertMsgFailedReturn(("hmR0VmxGetMsrPermission: Invalid MSR %#RX32\n", uMsr), VERR_NOT_SUPPORTED);
1251
1252	Assert(iBit <= 0x1fff);
1253	if (ASMBitTest(pbMsrBitmap, iBit))
1254	*penmRead = VMXMSREXIT_INTERCEPT_READ;
1255	else
1256	*penmRead = VMXMSREXIT_PASSTHRU_READ;
1257
1258	if (ASMBitTest(pbMsrBitmap + 0x800, iBit))
1259	*penmWrite = VMXMSREXIT_INTERCEPT_WRITE;
1260	else
1261	*penmWrite = VMXMSREXIT_PASSTHRU_WRITE;
1262	return VINF_SUCCESS;
1263	}
1264	#endif /* VBOX_STRICT */
1265
1266
1267	/**
1268	* Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1269	* area.
1270	*
1271	* @returns VBox status code.
1272	* @param pVCpu The cross context virtual CPU structure.
1273	* @param cMsrs The number of MSRs.
1274	*/
1275	DECLINLINE(int) hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1276	{
1277	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1278	uint32_t const cMaxSupportedMsrs = MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1279	if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1280	{
1281	LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1282	pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1283	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1284	}
1285
1286	/* Update number of guest MSRs to load/store across the world-switch. */
1287	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs);
1288	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs);
1289
1290	/* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1291	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs);
1292	AssertRCReturn(rc, rc);
1293
1294	/* Update the VCPU's copy of the MSR count. */
1295	pVCpu->hm.s.vmx.cMsrs = cMsrs;
1296
1297	return VINF_SUCCESS;
1298	}
1299
1300
1301	/**
1302	* Adds a new (or updates the value of an existing) guest/host MSR
1303	* pair to be swapped during the world-switch as part of the
1304	* auto-load/store MSR area in the VMCS.
1305	*
1306	* @returns VBox status code.
1307	* @param pVCpu The cross context virtual CPU structure.
1308	* @param uMsr The MSR.
1309	* @param uGuestMsrValue Value of the guest MSR.
1310	* @param fUpdateHostMsr Whether to update the value of the host MSR if
1311	* necessary.
1312	* @param pfAddedAndUpdated Where to store whether the MSR was added -and-
1313	* its value was updated. Optional, can be NULL.
1314	*/
1315	static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr,
1316	bool *pfAddedAndUpdated)
1317	{
1318	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1319	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1320	uint32_t i;
1321	for (i = 0; i < cMsrs; i++)
1322	{
1323	if (pGuestMsr->u32Msr == uMsr)
1324	break;
1325	pGuestMsr++;
1326	}
1327
1328	bool fAdded = false;
1329	if (i == cMsrs)
1330	{
1331	++cMsrs;
1332	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1333	AssertMsgRCReturn(rc, ("hmR0VmxAddAutoLoadStoreMsr: Insufficient space to add MSR %u\n", uMsr), rc);
1334
1335	/* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1336	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1337	hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1338
1339	fAdded = true;
1340	}
1341
1342	/* Update the MSR values in the auto-load/store MSR area. */
1343	pGuestMsr->u32Msr = uMsr;
1344	pGuestMsr->u64Value = uGuestMsrValue;
1345
1346	/* Create/update the MSR slot in the host MSR area. */
1347	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1348	pHostMsr += i;
1349	pHostMsr->u32Msr = uMsr;
1350
1351	/*
1352	* Update the host MSR only when requested by the caller AND when we're
1353	* adding it to the auto-load/store area. Otherwise, it would have been
1354	* updated by hmR0VmxSaveHostMsrs(). We do this for performance reasons.
1355	*/
1356	bool fUpdatedMsrValue = false;
1357	if ( fAdded
1358	&& fUpdateHostMsr)
1359	{
1360	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1361	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1362	pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1363	fUpdatedMsrValue = true;
1364	}
1365
1366	if (pfAddedAndUpdated)
1367	*pfAddedAndUpdated = fUpdatedMsrValue;
1368	return VINF_SUCCESS;
1369	}
1370
1371
1372	/**
1373	* Removes a guest/host MSR pair to be swapped during the world-switch from the
1374	* auto-load/store MSR area in the VMCS.
1375	*
1376	* @returns VBox status code.
1377	* @param pVCpu The cross context virtual CPU structure.
1378	* @param uMsr The MSR.
1379	*/
1380	static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1381	{
1382	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1383	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1384	for (uint32_t i = 0; i < cMsrs; i++)
1385	{
1386	/* Find the MSR. */
1387	if (pGuestMsr->u32Msr == uMsr)
1388	{
1389	/* If it's the last MSR, simply reduce the count. */
1390	if (i == cMsrs - 1)
1391	{
1392	--cMsrs;
1393	break;
1394	}
1395
1396	/* Remove it by swapping the last MSR in place of it, and reducing the count. */
1397	PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1398	pLastGuestMsr += cMsrs - 1;
1399	pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1400	pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1401
1402	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1403	PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1404	pLastHostMsr += cMsrs - 1;
1405	pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1406	pHostMsr->u64Value = pLastHostMsr->u64Value;
1407	--cMsrs;
1408	break;
1409	}
1410	pGuestMsr++;
1411	}
1412
1413	/* Update the VMCS if the count changed (meaning the MSR was found). */
1414	if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1415	{
1416	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1417	AssertRCReturn(rc, rc);
1418
1419	/* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1420	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1421	hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1422
1423	Log4(("Removed MSR %#RX32 new cMsrs=%u\n", uMsr, pVCpu->hm.s.vmx.cMsrs));
1424	return VINF_SUCCESS;
1425	}
1426
1427	return VERR_NOT_FOUND;
1428	}
1429
1430
1431	/**
1432	* Checks if the specified guest MSR is part of the auto-load/store area in
1433	* the VMCS.
1434	*
1435	* @returns true if found, false otherwise.
1436	* @param pVCpu The cross context virtual CPU structure.
1437	* @param uMsr The MSR to find.
1438	*/
1439	static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1440	{
1441	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1442	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1443
1444	for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1445	{
1446	if (pGuestMsr->u32Msr == uMsr)
1447	return true;
1448	}
1449	return false;
1450	}
1451
1452
1453	/**
1454	* Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1455	*
1456	* @param pVCpu The cross context virtual CPU structure.
1457	*
1458	* @remarks No-long-jump zone!!!
1459	*/
1460	static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1461	{
1462	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1463	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1464	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1465	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1466
1467	for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1468	{
1469	AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1470
1471	/*
1472	* Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1473	* Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1474	*/
1475	if (pHostMsr->u32Msr == MSR_K6_EFER)
1476	pHostMsr->u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostEfer;
1477	else
1478	pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1479	}
1480
1481	pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1482	}
1483
1484
1485	/**
1486	* Saves a set of host MSRs to allow read/write passthru access to the guest and
1487	* perform lazy restoration of the host MSRs while leaving VT-x.
1488	*
1489	* @param pVCpu The cross context virtual CPU structure.
1490	*
1491	* @remarks No-long-jump zone!!!
1492	*/
1493	static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1494	{
1495	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1496
1497	/*
1498	* Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1499	*/
1500	if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST))
1501	{
1502	Assert(!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)); /* Guest MSRs better not be loaded now. */
1503	#if HC_ARCH_BITS == 64
1504	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1505	{
1506	pVCpu->hm.s.vmx.u64HostLStarMsr = ASMRdMsr(MSR_K8_LSTAR);
1507	pVCpu->hm.s.vmx.u64HostStarMsr = ASMRdMsr(MSR_K6_STAR);
1508	pVCpu->hm.s.vmx.u64HostSFMaskMsr = ASMRdMsr(MSR_K8_SF_MASK);
1509	pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1510	}
1511	#endif
1512	pVCpu->hm.s.vmx.fLazyMsrs \|= VMX_LAZY_MSRS_SAVED_HOST;
1513	}
1514	}
1515
1516
1517	/**
1518	* Checks whether the MSR belongs to the set of guest MSRs that we restore
1519	* lazily while leaving VT-x.
1520	*
1521	* @returns true if it does, false otherwise.
1522	* @param pVCpu The cross context virtual CPU structure.
1523	* @param uMsr The MSR to check.
1524	*/
1525	static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1526	{
1527	NOREF(pVCpu);
1528	#if HC_ARCH_BITS == 64
1529	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1530	{
1531	switch (uMsr)
1532	{
1533	case MSR_K8_LSTAR:
1534	case MSR_K6_STAR:
1535	case MSR_K8_SF_MASK:
1536	case MSR_K8_KERNEL_GS_BASE:
1537	return true;
1538	}
1539	}
1540	#else
1541	RT_NOREF(pVCpu, uMsr);
1542	#endif
1543	return false;
1544	}
1545
1546
1547	/**
1548	* Saves a set of guest MSRs back into the guest-CPU context.
1549	*
1550	* @param pVCpu The cross context virtual CPU structure.
1551	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
1552	* out-of-sync. Make sure to update the required fields
1553	* before using them.
1554	*
1555	* @remarks No-long-jump zone!!!
1556	*/
1557	static void hmR0VmxLazySaveGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1558	{
1559	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1560	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1561
1562	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1563	{
1564	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1565	#if HC_ARCH_BITS == 64
1566	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1567	{
1568	pMixedCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
1569	pMixedCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
1570	pMixedCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
1571	pMixedCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1572	}
1573	#else
1574	NOREF(pMixedCtx);
1575	#endif
1576	}
1577	}
1578
1579
1580	/**
1581	* Loads a set of guests MSRs to allow read/passthru to the guest.
1582	*
1583	* The name of this function is slightly confusing. This function does NOT
1584	* postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1585	* common prefix for functions dealing with "lazy restoration" of the shared
1586	* MSRs.
1587	*
1588	* @param pVCpu The cross context virtual CPU structure.
1589	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
1590	* out-of-sync. Make sure to update the required fields
1591	* before using them.
1592	*
1593	* @remarks No-long-jump zone!!!
1594	*/
1595	static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1596	{
1597	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1598	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1599
1600	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1601	#if HC_ARCH_BITS == 64
1602	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1603	{
1604	/*
1605	* If the guest MSRs are not loaded -and- if all the guest MSRs are identical
1606	* to the MSRs on the CPU (which are the saved host MSRs, see assertion above) then
1607	* we can skip a few MSR writes.
1608	*
1609	* Otherwise, it implies either 1. they're not loaded, or 2. they're loaded but the
1610	* guest MSR values in the guest-CPU context might be different to what's currently
1611	* loaded in the CPU. In either case, we need to write the new guest MSR values to the
1612	* CPU, see @bugref{8728}.
1613	*/
1614	if ( !(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1615	&& pMixedCtx->msrKERNELGSBASE == pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr
1616	&& pMixedCtx->msrLSTAR == pVCpu->hm.s.vmx.u64HostLStarMsr
1617	&& pMixedCtx->msrSTAR == pVCpu->hm.s.vmx.u64HostStarMsr
1618	&& pMixedCtx->msrSFMASK == pVCpu->hm.s.vmx.u64HostSFMaskMsr)
1619	{
1620	#ifdef VBOX_STRICT
1621	Assert(ASMRdMsr(MSR_K8_KERNEL_GS_BASE) == pMixedCtx->msrKERNELGSBASE);
1622	Assert(ASMRdMsr(MSR_K8_LSTAR) == pMixedCtx->msrLSTAR);
1623	Assert(ASMRdMsr(MSR_K6_STAR) == pMixedCtx->msrSTAR);
1624	Assert(ASMRdMsr(MSR_K8_SF_MASK) == pMixedCtx->msrSFMASK);
1625	#endif
1626	}
1627	else
1628	{
1629	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE);
1630	ASMWrMsr(MSR_K8_LSTAR, pMixedCtx->msrLSTAR);
1631	ASMWrMsr(MSR_K6_STAR, pMixedCtx->msrSTAR);
1632	ASMWrMsr(MSR_K8_SF_MASK, pMixedCtx->msrSFMASK);
1633	}
1634	}
1635	#else
1636	RT_NOREF(pMixedCtx);
1637	#endif
1638	pVCpu->hm.s.vmx.fLazyMsrs \|= VMX_LAZY_MSRS_LOADED_GUEST;
1639	}
1640
1641
1642	/**
1643	* Performs lazy restoration of the set of host MSRs if they were previously
1644	* loaded with guest MSR values.
1645	*
1646	* @param pVCpu The cross context virtual CPU structure.
1647	*
1648	* @remarks No-long-jump zone!!!
1649	* @remarks The guest MSRs should have been saved back into the guest-CPU
1650	* context by hmR0VmxSaveGuestLazyMsrs()!!!
1651	*/
1652	static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1653	{
1654	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1655	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1656
1657	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1658	{
1659	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1660	#if HC_ARCH_BITS == 64
1661	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1662	{
1663	ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostLStarMsr);
1664	ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostStarMsr);
1665	ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostSFMaskMsr);
1666	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr);
1667	}
1668	#endif
1669	}
1670	pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST \| VMX_LAZY_MSRS_SAVED_HOST);
1671	}
1672
1673
1674	/**
1675	* Verifies that our cached values of the VMCS controls are all
1676	* consistent with what's actually present in the VMCS.
1677	*
1678	* @returns VBox status code.
1679	* @param pVCpu The cross context virtual CPU structure.
1680	*/
1681	static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu)
1682	{
1683	uint32_t u32Val;
1684	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
1685	AssertRCReturn(rc, rc);
1686	AssertMsgReturn(pVCpu->hm.s.vmx.u32EntryCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32EntryCtls, u32Val),
1687	VERR_VMX_ENTRY_CTLS_CACHE_INVALID);
1688
1689	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
1690	AssertRCReturn(rc, rc);
1691	AssertMsgReturn(pVCpu->hm.s.vmx.u32ExitCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ExitCtls, u32Val),
1692	VERR_VMX_EXIT_CTLS_CACHE_INVALID);
1693
1694	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
1695	AssertRCReturn(rc, rc);
1696	AssertMsgReturn(pVCpu->hm.s.vmx.u32PinCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32PinCtls, u32Val),
1697	VERR_VMX_PIN_EXEC_CTLS_CACHE_INVALID);
1698
1699	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
1700	AssertRCReturn(rc, rc);
1701	AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls, u32Val),
1702	VERR_VMX_PROC_EXEC_CTLS_CACHE_INVALID);
1703
1704	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1705	{
1706	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
1707	AssertRCReturn(rc, rc);
1708	AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls2 == u32Val,
1709	("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls2, u32Val),
1710	VERR_VMX_PROC_EXEC2_CTLS_CACHE_INVALID);
1711	}
1712
1713	return VINF_SUCCESS;
1714	}
1715
1716
1717	#ifdef VBOX_STRICT
1718	/**
1719	* Verifies that our cached host EFER value has not changed
1720	* since we cached it.
1721	*
1722	* @param pVCpu The cross context virtual CPU structure.
1723	*/
1724	static void hmR0VmxCheckHostEferMsr(PVMCPU pVCpu)
1725	{
1726	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1727
1728	if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
1729	{
1730	uint64_t u64Val;
1731	int rc = VMXReadVmcs64(VMX_VMCS64_HOST_EFER_FULL, &u64Val);
1732	AssertRC(rc);
1733
1734	uint64_t u64HostEferMsr = ASMRdMsr(MSR_K6_EFER);
1735	AssertMsgReturnVoid(u64HostEferMsr == u64Val, ("u64HostEferMsr=%#RX64 u64Val=%#RX64\n", u64HostEferMsr, u64Val));
1736	}
1737	}
1738
1739
1740	/**
1741	* Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1742	* VMCS are correct.
1743	*
1744	* @param pVCpu The cross context virtual CPU structure.
1745	*/
1746	static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1747	{
1748	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1749
1750	/* Verify MSR counts in the VMCS are what we think it should be. */
1751	uint32_t cMsrs;
1752	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1753	Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1754
1755	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1756	Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1757
1758	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1759	Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1760
1761	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1762	PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1763	for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1764	{
1765	/* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1766	AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32 cMsrs=%u\n", pHostMsr->u32Msr,
1767	pGuestMsr->u32Msr, cMsrs));
1768
1769	uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1770	AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
1771	pHostMsr->u32Msr, pHostMsr->u64Value, u64Msr, cMsrs));
1772
1773	/* Verify that the permissions are as expected in the MSR bitmap. */
1774	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1775	{
1776	VMXMSREXITREAD enmRead;
1777	VMXMSREXITWRITE enmWrite;
1778	rc = hmR0VmxGetMsrPermission(pVCpu, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1779	AssertMsgReturnVoid(rc == VINF_SUCCESS, ("hmR0VmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1780	if (pGuestMsr->u32Msr == MSR_K6_EFER)
1781	{
1782	AssertMsgReturnVoid(enmRead == VMXMSREXIT_INTERCEPT_READ, ("Passthru read for EFER!?\n"));
1783	AssertMsgReturnVoid(enmWrite == VMXMSREXIT_INTERCEPT_WRITE, ("Passthru write for EFER!?\n"));
1784	}
1785	else
1786	{
1787	AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 cMsrs=%u No passthru read!\n",
1788	pGuestMsr->u32Msr, cMsrs));
1789	AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 cMsrs=%u No passthru write!\n",
1790	pGuestMsr->u32Msr, cMsrs));
1791	}
1792	}
1793	}
1794	}
1795	#endif /* VBOX_STRICT */
1796
1797
1798	/**
1799	* Flushes the TLB using EPT.
1800	*
1801	* @returns VBox status code.
1802	* @param pVCpu The cross context virtual CPU structure of the calling
1803	* EMT. Can be NULL depending on @a enmFlush.
1804	* @param enmFlush Type of flush.
1805	*
1806	* @remarks Caller is responsible for making sure this function is called only
1807	* when NestedPaging is supported and providing @a enmFlush that is
1808	* supported by the CPU.
1809	* @remarks Can be called with interrupts disabled.
1810	*/
1811	static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush)
1812	{
1813	uint64_t au64Descriptor[2];
1814	if (enmFlush == VMXFLUSHEPT_ALL_CONTEXTS)
1815	au64Descriptor[0] = 0;
1816	else
1817	{
1818	Assert(pVCpu);
1819	au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1820	}
1821	au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1822
1823	int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1824	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1825	rc));
1826	if ( RT_SUCCESS(rc)
1827	&& pVCpu)
1828	{
1829	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1830	}
1831	}
1832
1833
1834	/**
1835	* Flushes the TLB using VPID.
1836	*
1837	* @returns VBox status code.
1838	* @param pVM The cross context VM structure.
1839	* @param pVCpu The cross context virtual CPU structure of the calling
1840	* EMT. Can be NULL depending on @a enmFlush.
1841	* @param enmFlush Type of flush.
1842	* @param GCPtr Virtual address of the page to flush (can be 0 depending
1843	* on @a enmFlush).
1844	*
1845	* @remarks Can be called with interrupts disabled.
1846	*/
1847	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr)
1848	{
1849	NOREF(pVM);
1850	AssertPtr(pVM);
1851	Assert(pVM->hm.s.vmx.fVpid);
1852
1853	uint64_t au64Descriptor[2];
1854	if (enmFlush == VMXFLUSHVPID_ALL_CONTEXTS)
1855	{
1856	au64Descriptor[0] = 0;
1857	au64Descriptor[1] = 0;
1858	}
1859	else
1860	{
1861	AssertPtr(pVCpu);
1862	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1863	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1864	au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1865	au64Descriptor[1] = GCPtr;
1866	}
1867
1868	int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]); NOREF(rc);
1869	AssertMsg(rc == VINF_SUCCESS,
1870	("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1871	if ( RT_SUCCESS(rc)
1872	&& pVCpu)
1873	{
1874	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1875	}
1876	}
1877
1878
1879	/**
1880	* Invalidates a guest page by guest virtual address. Only relevant for
1881	* EPT/VPID, otherwise there is nothing really to invalidate.
1882	*
1883	* @returns VBox status code.
1884	* @param pVM The cross context VM structure.
1885	* @param pVCpu The cross context virtual CPU structure.
1886	* @param GCVirt Guest virtual address of the page to invalidate.
1887	*/
1888	VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1889	{
1890	AssertPtr(pVM);
1891	AssertPtr(pVCpu);
1892	LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1893
1894	bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1895	if (!fFlushPending)
1896	{
1897	/*
1898	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1899	* See @bugref{6043} and @bugref{6177}.
1900	*
1901	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1902	* function maybe called in a loop with individual addresses.
1903	*/
1904	if (pVM->hm.s.vmx.fVpid)
1905	{
1906	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1907	{
1908	hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_INDIV_ADDR, GCVirt);
1909	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1910	}
1911	else
1912	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1913	}
1914	else if (pVM->hm.s.fNestedPaging)
1915	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1916	}
1917
1918	return VINF_SUCCESS;
1919	}
1920
1921
1922	/**
1923	* Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1924	* otherwise there is nothing really to invalidate.
1925	*
1926	* @returns VBox status code.
1927	* @param pVM The cross context VM structure.
1928	* @param pVCpu The cross context virtual CPU structure.
1929	* @param GCPhys Guest physical address of the page to invalidate.
1930	*/
1931	VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1932	{
1933	NOREF(pVM); NOREF(GCPhys);
1934	LogFlowFunc(("%RGp\n", GCPhys));
1935
1936	/*
1937	* We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1938	* by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1939	* This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1940	*/
1941	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1942	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1943	return VINF_SUCCESS;
1944	}
1945
1946
1947	/**
1948	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1949	* case where neither EPT nor VPID is supported by the CPU.
1950	*
1951	* @param pVM The cross context VM structure.
1952	* @param pVCpu The cross context virtual CPU structure.
1953	* @param pCpu Pointer to the global HM struct.
1954	*
1955	* @remarks Called with interrupts disabled.
1956	*/
1957	static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1958	{
1959	AssertPtr(pVCpu);
1960	AssertPtr(pCpu);
1961	NOREF(pVM);
1962
1963	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1964
1965	Assert(pCpu->idCpu != NIL_RTCPUID);
1966	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1967	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1968	pVCpu->hm.s.fForceTLBFlush = false;
1969	return;
1970	}
1971
1972
1973	/**
1974	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1975	*
1976	* @param pVM The cross context VM structure.
1977	* @param pVCpu The cross context virtual CPU structure.
1978	* @param pCpu Pointer to the global HM CPU struct.
1979	* @remarks All references to "ASID" in this function pertains to "VPID" in
1980	* Intel's nomenclature. The reason is, to avoid confusion in compare
1981	* statements since the host-CPU copies are named "ASID".
1982	*
1983	* @remarks Called with interrupts disabled.
1984	*/
1985	static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1986	{
1987	#ifdef VBOX_WITH_STATISTICS
1988	bool fTlbFlushed = false;
1989	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1990	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1991	if (!fTlbFlushed) \
1992	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1993	} while (0)
1994	#else
1995	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1996	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1997	#endif
1998
1999	AssertPtr(pVM);
2000	AssertPtr(pCpu);
2001	AssertPtr(pVCpu);
2002	Assert(pCpu->idCpu != NIL_RTCPUID);
2003
2004	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
2005	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
2006	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
2007
2008	/*
2009	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2010	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
2011	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
2012	*/
2013	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2014	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2015	{
2016	++pCpu->uCurrentAsid;
2017	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2018	{
2019	pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
2020	pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2021	pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2022	}
2023
2024	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2025	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2026	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2027
2028	/*
2029	* Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
2030	* invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
2031	*/
2032	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2033	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2034	HMVMX_SET_TAGGED_TLB_FLUSHED();
2035	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
2036	}
2037
2038	/* Check for explicit TLB flushes. */
2039	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2040	{
2041	/*
2042	* Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
2043	* guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
2044	* Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
2045	* but not guest-physical mappings.
2046	* See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
2047	*/
2048	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2049	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2050	HMVMX_SET_TAGGED_TLB_FLUSHED();
2051	}
2052
2053	pVCpu->hm.s.fForceTLBFlush = false;
2054	HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
2055
2056	Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
2057	Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
2058	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2059	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2060	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2061	("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2062	pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2063	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2064	("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2065
2066	/* Update VMCS with the VPID. */
2067	int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2068	AssertRC(rc);
2069
2070	#undef HMVMX_SET_TAGGED_TLB_FLUSHED
2071	}
2072
2073
2074	/**
2075	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
2076	*
2077	* @returns VBox status code.
2078	* @param pVM The cross context VM structure.
2079	* @param pVCpu The cross context virtual CPU structure.
2080	* @param pCpu Pointer to the global HM CPU struct.
2081	*
2082	* @remarks Called with interrupts disabled.
2083	*/
2084	static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2085	{
2086	AssertPtr(pVM);
2087	AssertPtr(pVCpu);
2088	AssertPtr(pCpu);
2089	Assert(pCpu->idCpu != NIL_RTCPUID);
2090	AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
2091	AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
2092
2093	/*
2094	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2095	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2096	*/
2097	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2098	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2099	{
2100	pVCpu->hm.s.fForceTLBFlush = true;
2101	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2102	}
2103
2104	/* Check for explicit TLB flushes. */
2105	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2106	{
2107	pVCpu->hm.s.fForceTLBFlush = true;
2108	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2109	}
2110
2111	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2112	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2113
2114	if (pVCpu->hm.s.fForceTLBFlush)
2115	{
2116	hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2117	pVCpu->hm.s.fForceTLBFlush = false;
2118	}
2119	}
2120
2121
2122	/**
2123	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
2124	*
2125	* @returns VBox status code.
2126	* @param pVM The cross context VM structure.
2127	* @param pVCpu The cross context virtual CPU structure.
2128	* @param pCpu Pointer to the global HM CPU struct.
2129	*
2130	* @remarks Called with interrupts disabled.
2131	*/
2132	static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2133	{
2134	AssertPtr(pVM);
2135	AssertPtr(pVCpu);
2136	AssertPtr(pCpu);
2137	Assert(pCpu->idCpu != NIL_RTCPUID);
2138	AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
2139	AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
2140
2141	/*
2142	* Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
2143	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
2144	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
2145	*/
2146	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2147	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2148	{
2149	pVCpu->hm.s.fForceTLBFlush = true;
2150	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2151	}
2152
2153	/* Check for explicit TLB flushes. */
2154	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2155	{
2156	/*
2157	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
2158	* we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
2159	* pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
2160	*/
2161	pVCpu->hm.s.fForceTLBFlush = true;
2162	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2163	}
2164
2165	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2166	if (pVCpu->hm.s.fForceTLBFlush)
2167	{
2168	++pCpu->uCurrentAsid;
2169	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2170	{
2171	pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2172	pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2173	pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2174	}
2175
2176	pVCpu->hm.s.fForceTLBFlush = false;
2177	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2178	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2179	if (pCpu->fFlushAsidBeforeUse)
2180	{
2181	if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_SINGLE_CONTEXT)
2182	hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2183	else if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_ALL_CONTEXTS)
2184	{
2185	hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2186	pCpu->fFlushAsidBeforeUse = false;
2187	}
2188	else
2189	{
2190	/* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2191	AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2192	}
2193	}
2194	}
2195
2196	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2197	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2198	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2199	("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2200	pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2201	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2202	("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2203
2204	int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2205	AssertRC(rc);
2206	}
2207
2208
2209	/**
2210	* Flushes the guest TLB entry based on CPU capabilities.
2211	*
2212	* @param pVCpu The cross context virtual CPU structure.
2213	* @param pCpu Pointer to the global HM CPU struct.
2214	*/
2215	DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2216	{
2217	#ifdef HMVMX_ALWAYS_FLUSH_TLB
2218	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2219	#endif
2220	PVM pVM = pVCpu->CTX_SUFF(pVM);
2221	switch (pVM->hm.s.vmx.uFlushTaggedTlb)
2222	{
2223	case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu, pCpu); break;
2224	case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu, pCpu); break;
2225	case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu, pCpu); break;
2226	case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu, pCpu); break;
2227	default:
2228	AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2229	break;
2230	}
2231
2232	/* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2233	}
2234
2235
2236	/**
2237	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2238	* TLB entries from the host TLB before VM-entry.
2239	*
2240	* @returns VBox status code.
2241	* @param pVM The cross context VM structure.
2242	*/
2243	static int hmR0VmxSetupTaggedTlb(PVM pVM)
2244	{
2245	/*
2246	* Determine optimal flush type for Nested Paging.
2247	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2248	* guest execution (see hmR3InitFinalizeR0()).
2249	*/
2250	if (pVM->hm.s.fNestedPaging)
2251	{
2252	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2253	{
2254	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2255	pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_SINGLE_CONTEXT;
2256	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2257	pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_ALL_CONTEXTS;
2258	else
2259	{
2260	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2261	pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2262	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
2263	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2264	}
2265
2266	/* Make sure the write-back cacheable memory type for EPT is supported. */
2267	if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
2268	{
2269	pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2270	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
2271	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2272	}
2273
2274	/* EPT requires a page-walk length of 4. */
2275	if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
2276	{
2277	pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2278	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
2279	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2280	}
2281	}
2282	else
2283	{
2284	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2285	pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2286	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
2287	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2288	}
2289	}
2290
2291	/*
2292	* Determine optimal flush type for VPID.
2293	*/
2294	if (pVM->hm.s.vmx.fVpid)
2295	{
2296	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2297	{
2298	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2299	pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_SINGLE_CONTEXT;
2300	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2301	pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_ALL_CONTEXTS;
2302	else
2303	{
2304	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2305	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2306	LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
2307	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2308	LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2309	pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2310	pVM->hm.s.vmx.fVpid = false;
2311	}
2312	}
2313	else
2314	{
2315	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2316	Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
2317	pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2318	pVM->hm.s.vmx.fVpid = false;
2319	}
2320	}
2321
2322	/*
2323	* Setup the handler for flushing tagged-TLBs.
2324	*/
2325	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2326	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
2327	else if (pVM->hm.s.fNestedPaging)
2328	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
2329	else if (pVM->hm.s.vmx.fVpid)
2330	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
2331	else
2332	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
2333	return VINF_SUCCESS;
2334	}
2335
2336
2337	/**
2338	* Sets up pin-based VM-execution controls in the VMCS.
2339	*
2340	* @returns VBox status code.
2341	* @param pVM The cross context VM structure.
2342	* @param pVCpu The cross context virtual CPU structure.
2343	*/
2344	static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
2345	{
2346	AssertPtr(pVM);
2347	AssertPtr(pVCpu);
2348
2349	uint32_t val = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
2350	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2351
2352	val \|= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2353	\| VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2354
2355	if (pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
2356	val \|= VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2357
2358	/* Enable the VMX preemption timer. */
2359	if (pVM->hm.s.vmx.fUsePreemptTimer)
2360	{
2361	Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
2362	val \|= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
2363	}
2364
2365	#if 0
2366	/* Enable posted-interrupt processing. */
2367	if (pVM->hm.s.fPostedIntrs)
2368	{
2369	Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR);
2370	Assert(pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT);
2371	val \|= VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR;
2372	}
2373	#endif
2374
2375	if ((val & zap) != val)
2376	{
2377	LogRel(("hmR0VmxSetupPinCtls: Invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2378	pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, val, zap));
2379	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2380	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2381	}
2382
2383	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
2384	AssertRCReturn(rc, rc);
2385
2386	pVCpu->hm.s.vmx.u32PinCtls = val;
2387	return rc;
2388	}
2389
2390
2391	/**
2392	* Sets up processor-based VM-execution controls in the VMCS.
2393	*
2394	* @returns VBox status code.
2395	* @param pVM The cross context VM structure.
2396	* @param pVCpu The cross context virtual CPU structure.
2397	*/
2398	static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
2399	{
2400	AssertPtr(pVM);
2401	AssertPtr(pVCpu);
2402
2403	int rc = VERR_INTERNAL_ERROR_5;
2404	uint32_t val = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2405	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2406
2407	val \|= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
2408	\| VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2409	\| VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2410	\| VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2411	\| VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2412	\| VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2413	\| VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2414
2415	/* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2416	if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
2417	\|\| (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
2418	{
2419	LogRel(("hmR0VmxSetupProcCtls: Unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
2420	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2421	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2422	}
2423
2424	/* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2425	if (!pVM->hm.s.fNestedPaging)
2426	{
2427	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2428	val \|= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
2429	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2430	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2431	}
2432
2433	/* Use TPR shadowing if supported by the CPU. */
2434	if ( PDMHasApic(pVM)
2435	&& pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2436	{
2437	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2438	Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2439	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2440	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2441	AssertRCReturn(rc, rc);
2442
2443	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2444	/* CR8 writes cause a VM-exit based on TPR threshold. */
2445	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
2446	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
2447	}
2448	else
2449	{
2450	/*
2451	* Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2452	* Set this control only for 64-bit guests.
2453	*/
2454	if (pVM->hm.s.fAllow64BitGuests)
2455	{
2456	val \|= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
2457	\| VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
2458	}
2459	}
2460
2461	/* Use MSR-bitmaps if supported by the CPU. */
2462	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2463	{
2464	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
2465
2466	Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2467	Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2468	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2469	AssertRCReturn(rc, rc);
2470
2471	/*
2472	* The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2473	* automatically using dedicated fields in the VMCS.
2474	*/
2475	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2476	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2477	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2478	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2479	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2480
2481	#if HC_ARCH_BITS == 64
2482	/*
2483	* Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2484	*/
2485	if (pVM->hm.s.fAllow64BitGuests)
2486	{
2487	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2488	hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2489	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2490	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2491	}
2492	#endif
2493	/* Though MSR_IA32_PERF_GLOBAL_CTRL is saved/restored lazily, we want intercept reads/write to it for now. */
2494	}
2495
2496	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2497	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2498	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
2499
2500	if ((val & zap) != val)
2501	{
2502	LogRel(("hmR0VmxSetupProcCtls: Invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2503	pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, val, zap));
2504	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2505	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2506	}
2507
2508	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
2509	AssertRCReturn(rc, rc);
2510
2511	pVCpu->hm.s.vmx.u32ProcCtls = val;
2512
2513	/*
2514	* Secondary processor-based VM-execution controls.
2515	*/
2516	if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
2517	{
2518	val = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2519	zap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2520
2521	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
2522	val \|= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
2523
2524	if (pVM->hm.s.fNestedPaging)
2525	val \|= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
2526
2527	/*
2528	* Enable the INVPCID instruction if supported by the hardware and we expose
2529	* it to the guest. Without this, guest executing INVPCID would cause a #UD.
2530	*/
2531	if ( (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
2532	&& pVM->cpum.ro.GuestFeatures.fInvpcid)
2533	{
2534	val \|= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
2535	}
2536
2537	if (pVM->hm.s.vmx.fVpid)
2538	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
2539
2540	if (pVM->hm.s.vmx.fUnrestrictedGuest)
2541	val \|= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
2542
2543	#if 0
2544	if (pVM->hm.s.fVirtApicRegs)
2545	{
2546	Assert(pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT);
2547	val \|= VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT; /* Enable APIC-register virtualization. */
2548
2549	Assert(pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY);
2550	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY; /* Enable virtual-interrupt delivery. */
2551	}
2552	#endif
2553
2554	/* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
2555	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2556	* done dynamically. */
2557	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2558	{
2559	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2560	Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2561	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
2562	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2563	AssertRCReturn(rc, rc);
2564	}
2565
2566	if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2567	val \|= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
2568
2569	if ( pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT
2570	&& pVM->hm.s.vmx.cPleGapTicks
2571	&& pVM->hm.s.vmx.cPleWindowTicks)
2572	{
2573	val \|= VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT; /* Enable pause-loop exiting. */
2574
2575	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks);
2576	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks);
2577	AssertRCReturn(rc, rc);
2578	}
2579
2580	if ((val & zap) != val)
2581	{
2582	LogRel(("hmR0VmxSetupProcCtls: Invalid secondary processor-based VM-execution controls combo! "
2583	"cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, val, zap));
2584	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2585	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2586	}
2587
2588	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
2589	AssertRCReturn(rc, rc);
2590
2591	pVCpu->hm.s.vmx.u32ProcCtls2 = val;
2592	}
2593	else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2594	{
2595	LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
2596	"available\n"));
2597	pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2598	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2599	}
2600
2601	return VINF_SUCCESS;
2602	}
2603
2604
2605	/**
2606	* Sets up miscellaneous (everything other than Pin & Processor-based
2607	* VM-execution) control fields in the VMCS.
2608	*
2609	* @returns VBox status code.
2610	* @param pVM The cross context VM structure.
2611	* @param pVCpu The cross context virtual CPU structure.
2612	*/
2613	static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
2614	{
2615	NOREF(pVM);
2616	AssertPtr(pVM);
2617	AssertPtr(pVCpu);
2618
2619	int rc = VERR_GENERAL_FAILURE;
2620
2621	/* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2622	#if 0
2623	/* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestCR3AndCR4())*/
2624	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0);
2625	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0);
2626
2627	/*
2628	* Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2629	* and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2630	* We thus use the exception bitmap to control it rather than use both.
2631	*/
2632	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0);
2633	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0);
2634
2635	/** @todo Explore possibility of using IO-bitmaps. */
2636	/* All IO & IOIO instructions cause VM-exits. */
2637	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0);
2638	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0);
2639
2640	/* Initialize the MSR-bitmap area. */
2641	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
2642	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0);
2643	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0);
2644	AssertRCReturn(rc, rc);
2645	#endif
2646
2647	/* Setup MSR auto-load/store area. */
2648	Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2649	Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2650	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2651	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2652	AssertRCReturn(rc, rc);
2653
2654	Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2655	Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2656	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2657	AssertRCReturn(rc, rc);
2658
2659	/* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2660	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2661	AssertRCReturn(rc, rc);
2662
2663	/* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2664	#if 0
2665	/* Setup debug controls */
2666	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
2667	rc \|= VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2668	AssertRCReturn(rc, rc);
2669	#endif
2670
2671	return rc;
2672	}
2673
2674
2675	/**
2676	* Sets up the initial exception bitmap in the VMCS based on static conditions.
2677	*
2678	* We shall setup those exception intercepts that don't change during the
2679	* lifetime of the VM here. The rest are done dynamically while loading the
2680	* guest state.
2681	*
2682	* @returns VBox status code.
2683	* @param pVM The cross context VM structure.
2684	* @param pVCpu The cross context virtual CPU structure.
2685	*/
2686	static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
2687	{
2688	AssertPtr(pVM);
2689	AssertPtr(pVCpu);
2690
2691	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
2692
2693	uint32_t u32XcptBitmap = 0;
2694
2695	/* Must always intercept #AC to prevent the guest from hanging the CPU. */
2696	u32XcptBitmap \|= RT_BIT_32(X86_XCPT_AC);
2697
2698	/* Because we need to maintain the DR6 state even when intercepting DRx reads
2699	and writes, and because recursive #DBs can cause the CPU hang, we must always
2700	intercept #DB. */
2701	u32XcptBitmap \|= RT_BIT_32(X86_XCPT_DB);
2702
2703	/* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2704	if (!pVM->hm.s.fNestedPaging)
2705	u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
2706
2707	pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2708	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2709	AssertRCReturn(rc, rc);
2710	return rc;
2711	}
2712
2713
2714	/**
2715	* Sets up the initial guest-state mask. The guest-state mask is consulted
2716	* before reading guest-state fields from the VMCS as VMREADs can be expensive
2717	* for the nested virtualization case (as it would cause a VM-exit).
2718	*
2719	* @param pVCpu The cross context virtual CPU structure.
2720	*/
2721	static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
2722	{
2723	/* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
2724	HMVMXCPU_GST_RESET_TO(pVCpu, HMVMX_UPDATED_GUEST_ALL);
2725	return VINF_SUCCESS;
2726	}
2727
2728
2729	/**
2730	* Does per-VM VT-x initialization.
2731	*
2732	* @returns VBox status code.
2733	* @param pVM The cross context VM structure.
2734	*/
2735	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2736	{
2737	LogFlowFunc(("pVM=%p\n", pVM));
2738
2739	int rc = hmR0VmxStructsAlloc(pVM);
2740	if (RT_FAILURE(rc))
2741	{
2742	LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2743	return rc;
2744	}
2745
2746	return VINF_SUCCESS;
2747	}
2748
2749
2750	/**
2751	* Does per-VM VT-x termination.
2752	*
2753	* @returns VBox status code.
2754	* @param pVM The cross context VM structure.
2755	*/
2756	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2757	{
2758	LogFlowFunc(("pVM=%p\n", pVM));
2759
2760	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2761	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2762	ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2763	#endif
2764	hmR0VmxStructsFree(pVM);
2765	return VINF_SUCCESS;
2766	}
2767
2768
2769	/**
2770	* Sets up the VM for execution under VT-x.
2771	* This function is only called once per-VM during initialization.
2772	*
2773	* @returns VBox status code.
2774	* @param pVM The cross context VM structure.
2775	*/
2776	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2777	{
2778	AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2779	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2780
2781	LogFlowFunc(("pVM=%p\n", pVM));
2782
2783	/*
2784	* Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable must always be allocated.
2785	* We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0Intel().
2786	*/
2787	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2788	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2789	\|\| !pVM->hm.s.vmx.pRealModeTSS))
2790	{
2791	LogRel(("VMXR0SetupVM: Invalid real-on-v86 state.\n"));
2792	return VERR_INTERNAL_ERROR;
2793	}
2794
2795	/* Initialize these always, see hmR3InitFinalizeR0().*/
2796	pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NONE;
2797	pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NONE;
2798
2799	/* Setup the tagged-TLB flush handlers. */
2800	int rc = hmR0VmxSetupTaggedTlb(pVM);
2801	if (RT_FAILURE(rc))
2802	{
2803	LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2804	return rc;
2805	}
2806
2807	/* Check if we can use the VMCS controls for swapping the EFER MSR. */
2808	Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
2809	#if HC_ARCH_BITS == 64
2810	if ( (pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1 & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
2811	&& (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
2812	&& (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR))
2813	{
2814	pVM->hm.s.vmx.fSupportsVmcsEfer = true;
2815	}
2816	#endif
2817
2818	/* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
2819	RTCCUINTREG uHostCR4 = ASMGetCR4();
2820	if (RT_UNLIKELY(!(uHostCR4 & X86_CR4_VMXE)))
2821	return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
2822
2823	for (VMCPUID i = 0; i < pVM->cCpus; i++)
2824	{
2825	PVMCPU pVCpu = &pVM->aCpus[i];
2826	AssertPtr(pVCpu);
2827	AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2828
2829	/* Log the VCPU pointers, useful for debugging SMP VMs. */
2830	Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2831
2832	/* Initialize the VM-exit history array with end-of-array markers (UINT16_MAX). */
2833	Assert(!pVCpu->hm.s.idxExitHistoryFree);
2834	HMCPU_EXIT_HISTORY_RESET(pVCpu);
2835
2836	/* Set revision dword at the beginning of the VMCS structure. */
2837	(uint32_t )pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2838
2839	/* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2840	rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2841	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2842	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2843
2844	/* Load this VMCS as the current VMCS. */
2845	rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2846	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2847	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2848
2849	rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2850	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2851	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2852
2853	rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2854	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2855	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2856
2857	rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2858	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2859	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2860
2861	rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2862	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2863	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2864
2865	rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2866	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2867	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2868
2869	#if HC_ARCH_BITS == 32
2870	rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2871	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2872	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2873	#endif
2874
2875	/* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2876	rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2877	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2878	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2879
2880	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2881
2882	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2883	}
2884
2885	return VINF_SUCCESS;
2886	}
2887
2888
2889	/**
2890	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2891	* the VMCS.
2892	*
2893	* @returns VBox status code.
2894	* @param pVM The cross context VM structure.
2895	* @param pVCpu The cross context virtual CPU structure.
2896	*/
2897	DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2898	{
2899	NOREF(pVM); NOREF(pVCpu);
2900
2901	RTCCUINTREG uReg = ASMGetCR0();
2902	int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2903	AssertRCReturn(rc, rc);
2904
2905	uReg = ASMGetCR3();
2906	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2907	AssertRCReturn(rc, rc);
2908
2909	uReg = ASMGetCR4();
2910	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2911	AssertRCReturn(rc, rc);
2912	return rc;
2913	}
2914
2915
2916	#if HC_ARCH_BITS == 64
2917	/**
2918	* Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2919	* requirements. See hmR0VmxSaveHostSegmentRegs().
2920	*/
2921	# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2922	if ((selValue) & (X86_SEL_RPL \| X86_SEL_LDT)) \
2923	{ \
2924	bool fValidSelector = true; \
2925	if ((selValue) & X86_SEL_LDT) \
2926	{ \
2927	uint32_t uAttr = ASMGetSegAttr((selValue)); \
2928	fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
2929	} \
2930	if (fValidSelector) \
2931	{ \
2932	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_##seg; \
2933	pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2934	} \
2935	(selValue) = 0; \
2936	}
2937	#endif
2938
2939
2940	/**
2941	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2942	* the host-state area in the VMCS.
2943	*
2944	* @returns VBox status code.
2945	* @param pVM The cross context VM structure.
2946	* @param pVCpu The cross context virtual CPU structure.
2947	*/
2948	DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2949	{
2950	int rc = VERR_INTERNAL_ERROR_5;
2951
2952	#if HC_ARCH_BITS == 64
2953	/*
2954	* If we've executed guest code using VT-x, the host-state bits will be messed up. We
2955	* should -not- save the messed up state without restoring the original host-state. See @bugref{7240}.
2956	*
2957	* This apparently can happen (most likely the FPU changes), deal with it rather than asserting.
2958	* Was observed booting Solaris10u10 32-bit guest.
2959	*/
2960	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
2961	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
2962	{
2963	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags,
2964	pVCpu->idCpu));
2965	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
2966	}
2967	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2968	#else
2969	RT_NOREF(pVCpu);
2970	#endif
2971
2972	/*
2973	* Host DS, ES, FS and GS segment registers.
2974	*/
2975	#if HC_ARCH_BITS == 64
2976	RTSEL uSelDS = ASMGetDS();
2977	RTSEL uSelES = ASMGetES();
2978	RTSEL uSelFS = ASMGetFS();
2979	RTSEL uSelGS = ASMGetGS();
2980	#else
2981	RTSEL uSelDS = 0;
2982	RTSEL uSelES = 0;
2983	RTSEL uSelFS = 0;
2984	RTSEL uSelGS = 0;
2985	#endif
2986
2987	/*
2988	* Host CS and SS segment registers.
2989	*/
2990	RTSEL uSelCS = ASMGetCS();
2991	RTSEL uSelSS = ASMGetSS();
2992
2993	/*
2994	* Host TR segment register.
2995	*/
2996	RTSEL uSelTR = ASMGetTR();
2997
2998	#if HC_ARCH_BITS == 64
2999	/*
3000	* Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
3001	* before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
3002	*/
3003	VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
3004	VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
3005	VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
3006	VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
3007	# undef VMXLOCAL_ADJUST_HOST_SEG
3008	#endif
3009
3010	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
3011	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
3012	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
3013	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
3014	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
3015	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
3016	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
3017	Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
3018	Assert(uSelCS);
3019	Assert(uSelTR);
3020
3021	/* Assertion is right but we would not have updated u32ExitCtls yet. */
3022	#if 0
3023	if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
3024	Assert(uSelSS != 0);
3025	#endif
3026
3027	/* Write these host selector fields into the host-state area in the VMCS. */
3028	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_CS_SEL, uSelCS);
3029	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_SS_SEL, uSelSS);
3030	#if HC_ARCH_BITS == 64
3031	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_DS_SEL, uSelDS);
3032	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_ES_SEL, uSelES);
3033	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FS_SEL, uSelFS);
3034	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_GS_SEL, uSelGS);
3035	#else
3036	NOREF(uSelDS);
3037	NOREF(uSelES);
3038	NOREF(uSelFS);
3039	NOREF(uSelGS);
3040	#endif
3041	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_TR_SEL, uSelTR);
3042	AssertRCReturn(rc, rc);
3043
3044	/*
3045	* Host GDTR and IDTR.
3046	*/
3047	RTGDTR Gdtr;
3048	RTIDTR Idtr;
3049	RT_ZERO(Gdtr);
3050	RT_ZERO(Idtr);
3051	ASMGetGDTR(&Gdtr);
3052	ASMGetIDTR(&Idtr);
3053	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
3054	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
3055	AssertRCReturn(rc, rc);
3056
3057	#if HC_ARCH_BITS == 64
3058	/*
3059	* Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
3060	* maximum limit (0xffff) on every VM-exit.
3061	*/
3062	if (Gdtr.cbGdt != 0xffff)
3063	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDTR;
3064
3065	/*
3066	* IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT"
3067	* and Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit as 0xfff, VT-x
3068	* bloating the limit to 0xffff shouldn't cause any different CPU behavior. However, several hosts either insists
3069	* on 0xfff being the limit (Windows Patch Guard) or uses the limit for other purposes (darwin puts the CPU ID in there
3070	* but botches sidt alignment in at least one consumer). So, we're only allowing IDTR.LIMIT to be left at 0xffff on
3071	* hosts where we are pretty sure it won't cause trouble.
3072	*/
3073	# if defined(RT_OS_LINUX) \|\| defined(RT_OS_SOLARIS)
3074	if (Idtr.cbIdt < 0x0fff)
3075	# else
3076	if (Idtr.cbIdt != 0xffff)
3077	# endif
3078	{
3079	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_IDTR;
3080	AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
3081	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
3082	}
3083	#endif
3084
3085	/*
3086	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
3087	* is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
3088	*/
3089	AssertMsgReturn((uSelTR \| X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
3090	("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt),
3091	VERR_VMX_INVALID_HOST_STATE);
3092
3093	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
3094	#if HC_ARCH_BITS == 64
3095	uintptr_t uTRBase = X86DESC64_BASE(pDesc);
3096
3097	/*
3098	* VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on all VM-exits.
3099	* The type is the same for 64-bit busy TSS[1]. The limit needs manual restoration if the host has something else.
3100	* Task switching is not supported in 64-bit mode[2], but the limit still matters as IOPM is supported in 64-bit mode.
3101	* Restoring the limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
3102	*
3103	* [1] See Intel spec. 3.5 "System Descriptor Types".
3104	* [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
3105	*/
3106	Assert(pDesc->System.u4Type == 11);
3107	if ( pDesc->System.u16LimitLow != 0x67
3108	\|\| pDesc->System.u4LimitHigh)
3109	{
3110	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_TR;
3111	/* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
3112	if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
3113	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDT_READ_ONLY;
3114	pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
3115	}
3116
3117	/*
3118	* Store the GDTR as we need it when restoring the GDT and while restoring the TR.
3119	*/
3120	if (pVCpu->hm.s.vmx.fRestoreHostFlags & (VMX_RESTORE_HOST_GDTR \| VMX_RESTORE_HOST_SEL_TR))
3121	{
3122	AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
3123	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3124	if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_NEED_WRITABLE)
3125	{
3126	/* The GDT is read-only but the writable GDT is available. */
3127	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDT_NEED_WRITABLE;
3128	pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.cb = Gdtr.cbGdt;
3129	rc = SUPR0GetCurrentGdtRw(&pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.uAddr);
3130	AssertRCReturn(rc, rc);
3131	}
3132	}
3133	#else
3134	NOREF(pVM);
3135	uintptr_t uTRBase = X86DESC_BASE(pDesc);
3136	#endif
3137	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
3138	AssertRCReturn(rc, rc);
3139
3140	/*
3141	* Host FS base and GS base.
3142	*/
3143	#if HC_ARCH_BITS == 64
3144	uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
3145	uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
3146	rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase);
3147	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase);
3148	AssertRCReturn(rc, rc);
3149
3150	/* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
3151	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
3152	pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
3153	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
3154	pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
3155	#endif
3156	return rc;
3157	}
3158
3159
3160	/**
3161	* Saves certain host MSRs in the VM-exit MSR-load area and some in the
3162	* host-state area of the VMCS. Theses MSRs will be automatically restored on
3163	* the host after every successful VM-exit.
3164	*
3165	* @returns VBox status code.
3166	* @param pVM The cross context VM structure.
3167	* @param pVCpu The cross context virtual CPU structure.
3168	*
3169	* @remarks No-long-jump zone!!!
3170	*/
3171	DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
3172	{
3173	NOREF(pVM);
3174
3175	AssertPtr(pVCpu);
3176	AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3177
3178	/*
3179	* Save MSRs that we restore lazily (due to preemption or transition to ring-3)
3180	* rather than swapping them on every VM-entry.
3181	*/
3182	hmR0VmxLazySaveHostMsrs(pVCpu);
3183
3184	/*
3185	* Host Sysenter MSRs.
3186	*/
3187	int rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3188	#if HC_ARCH_BITS == 32
3189	rc \|= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3190	rc \|= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3191	#else
3192	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3193	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3194	#endif
3195	AssertRCReturn(rc, rc);
3196
3197	/*
3198	* Host EFER MSR.
3199	* If the CPU supports the newer VMCS controls for managing EFER, use it.
3200	* Otherwise it's done as part of auto-load/store MSR area in the VMCS, see hmR0VmxLoadGuestMsrs().
3201	*/
3202	if (pVM->hm.s.vmx.fSupportsVmcsEfer)
3203	{
3204	rc = VMXWriteVmcs64(VMX_VMCS64_HOST_EFER_FULL, pVM->hm.s.vmx.u64HostEfer);
3205	AssertRCReturn(rc, rc);
3206	}
3207
3208	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see
3209	* hmR0VmxLoadGuestExitCtls() !! */
3210
3211	return rc;
3212	}
3213
3214
3215	/**
3216	* Figures out if we need to swap the EFER MSR which is particularly expensive.
3217	*
3218	* We check all relevant bits. For now, that's everything besides LMA/LME, as
3219	* these two bits are handled by VM-entry, see hmR0VmxLoadGuestExitCtls() and
3220	* hmR0VMxLoadGuestEntryCtls().
3221	*
3222	* @returns true if we need to load guest EFER, false otherwise.
3223	* @param pVCpu The cross context virtual CPU structure.
3224	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3225	* out-of-sync. Make sure to update the required fields
3226	* before using them.
3227	*
3228	* @remarks Requires EFER, CR4.
3229	* @remarks No-long-jump zone!!!
3230	*/
3231	static bool hmR0VmxShouldSwapEferMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3232	{
3233	#ifdef HMVMX_ALWAYS_SWAP_EFER
3234	return true;
3235	#endif
3236
3237	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
3238	/* For 32-bit hosts running 64-bit guests, we always swap EFER in the world-switcher. Nothing to do here. */
3239	if (CPUMIsGuestInLongMode(pVCpu))
3240	return false;
3241	#endif
3242
3243	PVM pVM = pVCpu->CTX_SUFF(pVM);
3244	uint64_t u64HostEfer = pVM->hm.s.vmx.u64HostEfer;
3245	uint64_t u64GuestEfer = pMixedCtx->msrEFER;
3246
3247	/*
3248	* For 64-bit guests, if EFER.SCE bit differs, we need to swap to ensure that the
3249	* guest's SYSCALL behaviour isn't screwed. See @bugref{7386}.
3250	*/
3251	if ( CPUMIsGuestInLongMode(pVCpu)
3252	&& (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
3253	{
3254	return true;
3255	}
3256
3257	/*
3258	* If the guest uses PAE and EFER.NXE bit differs, we need to swap EFER as it
3259	* affects guest paging. 64-bit paging implies CR4.PAE as well.
3260	* See Intel spec. 4.5 "IA-32e Paging" and Intel spec. 4.1.1 "Three Paging Modes".
3261	*/
3262	if ( (pMixedCtx->cr4 & X86_CR4_PAE)
3263	&& (pMixedCtx->cr0 & X86_CR0_PG)
3264	&& (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
3265	{
3266	/* Assert that host is PAE capable. */
3267	Assert(pVM->hm.s.cpuid.u32AMDFeatureEDX & X86_CPUID_EXT_FEATURE_EDX_NX);
3268	return true;
3269	}
3270
3271	/** @todo Check the latest Intel spec. for any other bits,
3272	* like SMEP/SMAP? */
3273	return false;
3274	}
3275
3276
3277	/**
3278	* Sets up VM-entry controls in the VMCS. These controls can affect things done
3279	* on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
3280	* controls".
3281	*
3282	* @returns VBox status code.
3283	* @param pVCpu The cross context virtual CPU structure.
3284	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3285	* out-of-sync. Make sure to update the required fields
3286	* before using them.
3287	*
3288	* @remarks Requires EFER.
3289	* @remarks No-long-jump zone!!!
3290	*/
3291	DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3292	{
3293	int rc = VINF_SUCCESS;
3294	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS))
3295	{
3296	PVM pVM = pVCpu->CTX_SUFF(pVM);
3297	uint32_t val = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
3298	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3299
3300	/* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3301	val \|= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
3302
3303	/* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3304	if (CPUMIsGuestInLongModeEx(pMixedCtx))
3305	{
3306	val \|= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
3307	Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST\n", pVCpu->idCpu));
3308	}
3309	else
3310	Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
3311
3312	/* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it. */
3313	if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3314	&& hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3315	{
3316	val \|= VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR;
3317	Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR\n", pVCpu->idCpu));
3318	}
3319
3320	/*
3321	* The following should -not- be set (since we're not in SMM mode):
3322	* - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
3323	* - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
3324	*/
3325
3326	/** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
3327	* VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR. */
3328
3329	if ((val & zap) != val)
3330	{
3331	LogRel(("hmR0VmxLoadGuestEntryCtls: Invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3332	pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, val, zap));
3333	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3334	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3335	}
3336
3337	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
3338	AssertRCReturn(rc, rc);
3339
3340	pVCpu->hm.s.vmx.u32EntryCtls = val;
3341	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS);
3342	}
3343	return rc;
3344	}
3345
3346
3347	/**
3348	* Sets up the VM-exit controls in the VMCS.
3349	*
3350	* @returns VBox status code.
3351	* @param pVCpu The cross context virtual CPU structure.
3352	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3353	* out-of-sync. Make sure to update the required fields
3354	* before using them.
3355	*
3356	* @remarks Requires EFER.
3357	*/
3358	DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3359	{
3360	NOREF(pMixedCtx);
3361
3362	int rc = VINF_SUCCESS;
3363	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_EXIT_CTLS))
3364	{
3365	PVM pVM = pVCpu->CTX_SUFF(pVM);
3366	uint32_t val = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
3367	uint32_t zap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3368
3369	/* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3370	val \|= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
3371
3372	/*
3373	* Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3374	* On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
3375	*/
3376	#if HC_ARCH_BITS == 64
3377	val \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3378	Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3379	#else
3380	Assert( pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64
3381	\|\| pVCpu->hm.s.vmx.pfnStartVM == VMXR0StartVM32);
3382	/* Set the host address-space size based on the switcher, not guest state. See @bugref{8432}. */
3383	if (pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64)
3384	{
3385	/* The switcher returns to long mode, EFER is managed by the switcher. */
3386	val \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3387	Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3388	}
3389	else
3390	Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3391	#endif
3392
3393	/* If the newer VMCS fields for managing EFER exists, use it. */
3394	if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3395	&& hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3396	{
3397	val \|= VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR
3398	\| VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR;
3399	Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR, VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR\n", pVCpu->idCpu));
3400	}
3401
3402	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3403	Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
3404
3405	/** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
3406	* VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
3407	* VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR. */
3408
3409	if ( pVM->hm.s.vmx.fUsePreemptTimer
3410	&& (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER))
3411	val \|= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
3412
3413	if ((val & zap) != val)
3414	{
3415	LogRel(("hmR0VmxSetupProcCtls: Invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3416	pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, val, zap));
3417	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3418	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3419	}
3420
3421	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
3422	AssertRCReturn(rc, rc);
3423
3424	pVCpu->hm.s.vmx.u32ExitCtls = val;
3425	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_EXIT_CTLS);
3426	}
3427	return rc;
3428	}
3429
3430
3431	/**
3432	* Sets the TPR threshold in the VMCS.
3433	*
3434	* @returns VBox status code.
3435	* @param pVCpu The cross context virtual CPU structure.
3436	* @param u32TprThreshold The TPR threshold (task-priority class only).
3437	*/
3438	DECLINLINE(int) hmR0VmxApicSetTprThreshold(PVMCPU pVCpu, uint32_t u32TprThreshold)
3439	{
3440	Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3441	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW); RT_NOREF_PV(pVCpu);
3442	return VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3443	}
3444
3445
3446	/**
3447	* Loads the guest APIC and related state.
3448	*
3449	* @returns VBox status code.
3450	* @param pVCpu The cross context virtual CPU structure.
3451	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3452	* out-of-sync. Make sure to update the required fields
3453	* before using them.
3454	*
3455	* @remarks No-long-jump zone!!!
3456	*/
3457	DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3458	{
3459	NOREF(pMixedCtx);
3460
3461	int rc = VINF_SUCCESS;
3462	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE))
3463	{
3464	if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
3465	&& APICIsEnabled(pVCpu))
3466	{
3467	/*
3468	* Setup TPR shadowing.
3469	*/
3470	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
3471	{
3472	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3473
3474	bool fPendingIntr = false;
3475	uint8_t u8Tpr = 0;
3476	uint8_t u8PendingIntr = 0;
3477	rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3478	AssertRCReturn(rc, rc);
3479
3480	/*
3481	* If there are interrupts pending but masked by the TPR, instruct VT-x to cause a TPR-below-threshold VM-exit
3482	* when the guest lowers its TPR below the priority of the pending interrupt so we can deliver the interrupt.
3483	* If there are no interrupts pending, set threshold to 0 to not cause any TPR-below-threshold VM-exits.
3484	*/
3485	pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR] = u8Tpr;
3486	uint32_t u32TprThreshold = 0;
3487	if (fPendingIntr)
3488	{
3489	/* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3490	const uint8_t u8PendingPriority = u8PendingIntr >> 4;
3491	const uint8_t u8TprPriority = u8Tpr >> 4;
3492	if (u8PendingPriority <= u8TprPriority)
3493	u32TprThreshold = u8PendingPriority;
3494	}
3495
3496	rc = hmR0VmxApicSetTprThreshold(pVCpu, u32TprThreshold);
3497	AssertRCReturn(rc, rc);
3498	}
3499	}
3500	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
3501	}
3502
3503	return rc;
3504	}
3505
3506
3507	/**
3508	* Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3509	*
3510	* @returns Guest's interruptibility-state.
3511	* @param pVCpu The cross context virtual CPU structure.
3512	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3513	* out-of-sync. Make sure to update the required fields
3514	* before using them.
3515	*
3516	* @remarks No-long-jump zone!!!
3517	*/
3518	DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3519	{
3520	/*
3521	* Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
3522	*/
3523	uint32_t uIntrState = 0;
3524	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3525	{
3526	/* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
3527	AssertMsg(HMVMXCPU_GST_IS_SET(pVCpu, HMVMX_UPDATED_GUEST_RIP \| HMVMX_UPDATED_GUEST_RFLAGS),
3528	("%#x\n", HMVMXCPU_GST_VALUE(pVCpu)));
3529	if (pMixedCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
3530	{
3531	if (pMixedCtx->eflags.Bits.u1IF)
3532	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
3533	else
3534	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
3535	}
3536	else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3537	{
3538	/*
3539	* We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
3540	* VT-x, the flag's condition to be cleared is met and thus the cleared state is correct.
3541	*/
3542	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3543	}
3544	}
3545
3546	/*
3547	* NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
3548	* bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
3549	* setting this would block host-NMIs and IRET will not clear the blocking.
3550	*
3551	* See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
3552	*/
3553	if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
3554	&& (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
3555	{
3556	uIntrState \|= VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI;
3557	}
3558
3559	return uIntrState;
3560	}
3561
3562
3563	/**
3564	* Loads the guest's interruptibility-state into the guest-state area in the
3565	* VMCS.
3566	*
3567	* @returns VBox status code.
3568	* @param pVCpu The cross context virtual CPU structure.
3569	* @param uIntrState The interruptibility-state to set.
3570	*/
3571	static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
3572	{
3573	NOREF(pVCpu);
3574	AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
3575	Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3576	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
3577	AssertRC(rc);
3578	return rc;
3579	}
3580
3581
3582	/**
3583	* Loads the exception intercepts required for guest execution in the VMCS.
3584	*
3585	* @returns VBox status code.
3586	* @param pVCpu The cross context virtual CPU structure.
3587	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3588	* out-of-sync. Make sure to update the required fields
3589	* before using them.
3590	*/
3591	static int hmR0VmxLoadGuestXcptIntercepts(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3592	{
3593	NOREF(pMixedCtx);
3594	int rc = VINF_SUCCESS;
3595	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS))
3596	{
3597	/* The remaining exception intercepts are handled elsewhere, e.g. in hmR0VmxLoadSharedCR0(). */
3598	if (pVCpu->hm.s.fGIMTrapXcptUD)
3599	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_UD);
3600	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3601	else
3602	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_UD);
3603	#endif
3604
3605	Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_AC));
3606	Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
3607
3608	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3609	AssertRCReturn(rc, rc);
3610
3611	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
3612	Log4(("Load[%RU32]: VMX_VMCS32_CTRL_EXCEPTION_BITMAP=%#RX64 fContextUseFlags=%#RX32\n", pVCpu->idCpu,
3613	pVCpu->hm.s.vmx.u32XcptBitmap, HMCPU_CF_VALUE(pVCpu)));
3614	}
3615	return rc;
3616	}
3617
3618
3619	/**
3620	* Loads the guest's RIP into the guest-state area in the VMCS.
3621	*
3622	* @returns VBox status code.
3623	* @param pVCpu The cross context virtual CPU structure.
3624	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3625	* out-of-sync. Make sure to update the required fields
3626	* before using them.
3627	*
3628	* @remarks No-long-jump zone!!!
3629	*/
3630	static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3631	{
3632	int rc = VINF_SUCCESS;
3633	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP))
3634	{
3635	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
3636	AssertRCReturn(rc, rc);
3637
3638	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RIP);
3639	Log4(("Load[%RU32]: VMX_VMCS_GUEST_RIP=%#RX64 fContextUseFlags=%#RX32\n", pVCpu->idCpu, pMixedCtx->rip,
3640	HMCPU_CF_VALUE(pVCpu)));
3641	}
3642	return rc;
3643	}
3644
3645
3646	/**
3647	* Loads the guest's RSP into the guest-state area in the VMCS.
3648	*
3649	* @returns VBox status code.
3650	* @param pVCpu The cross context virtual CPU structure.
3651	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3652	* out-of-sync. Make sure to update the required fields
3653	* before using them.
3654	*
3655	* @remarks No-long-jump zone!!!
3656	*/
3657	static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3658	{
3659	int rc = VINF_SUCCESS;
3660	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RSP))
3661	{
3662	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
3663	AssertRCReturn(rc, rc);
3664
3665	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RSP);
3666	Log4(("Load[%RU32]: VMX_VMCS_GUEST_RSP=%#RX64\n", pVCpu->idCpu, pMixedCtx->rsp));
3667	}
3668	return rc;
3669	}
3670
3671
3672	/**
3673	* Loads the guest's RFLAGS into the guest-state area in the VMCS.
3674	*
3675	* @returns VBox status code.
3676	* @param pVCpu The cross context virtual CPU structure.
3677	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3678	* out-of-sync. Make sure to update the required fields
3679	* before using them.
3680	*
3681	* @remarks No-long-jump zone!!!
3682	*/
3683	static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3684	{
3685	int rc = VINF_SUCCESS;
3686	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
3687	{
3688	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3689	Let us assert it as such and use 32-bit VMWRITE. */
3690	Assert(!(pMixedCtx->rflags.u64 >> 32));
3691	X86EFLAGS Eflags = pMixedCtx->eflags;
3692	/** @todo r=bird: There shall be no need to OR in X86_EFL_1 here, nor
3693	* shall there be any reason for clearing bits 63:22, 15, 5 and 3.
3694	* These will never be cleared/set, unless some other part of the VMM
3695	* code is buggy - in which case we're better of finding and fixing
3696	* those bugs than hiding them. */
3697	Assert(Eflags.u32 & X86_EFL_RA1_MASK);
3698	Assert(!(Eflags.u32 & ~(X86_EFL_1 \| X86_EFL_LIVE_MASK)));
3699	Eflags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
3700	Eflags.u32 \|= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
3701
3702	/*
3703	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM-exit.
3704	* Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
3705	*/
3706	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3707	{
3708	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3709	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3710	pVCpu->hm.s.vmx.RealMode.Eflags.u32 = Eflags.u32; /* Save the original eflags of the real-mode guest. */
3711	Eflags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3712	Eflags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3713	}
3714
3715	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, Eflags.u32);
3716	AssertRCReturn(rc, rc);
3717
3718	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3719	Log4(("Load[%RU32]: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", pVCpu->idCpu, Eflags.u32));
3720	}
3721	return rc;
3722	}
3723
3724
3725	/**
3726	* Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
3727	*
3728	* @returns VBox status code.
3729	* @param pVCpu The cross context virtual CPU structure.
3730	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3731	* out-of-sync. Make sure to update the required fields
3732	* before using them.
3733	*
3734	* @remarks No-long-jump zone!!!
3735	*/
3736	DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3737	{
3738	int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
3739	rc \|= hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
3740	rc \|= hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
3741	AssertRCReturn(rc, rc);
3742	return rc;
3743	}
3744
3745
3746	/**
3747	* Loads the guest CR0 control register into the guest-state area in the VMCS.
3748	* CR0 is partially shared with the host and we have to consider the FPU bits.
3749	*
3750	* @returns VBox status code.
3751	* @param pVCpu The cross context virtual CPU structure.
3752	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3753	* out-of-sync. Make sure to update the required fields
3754	* before using them.
3755	*
3756	* @remarks No-long-jump zone!!!
3757	*/
3758	static int hmR0VmxLoadSharedCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3759	{
3760	/*
3761	* Guest CR0.
3762	* Guest FPU.
3763	*/
3764	int rc = VINF_SUCCESS;
3765	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
3766	{
3767	Assert(!(pMixedCtx->cr0 >> 32));
3768	uint32_t u32GuestCR0 = pMixedCtx->cr0;
3769	PVM pVM = pVCpu->CTX_SUFF(pVM);
3770
3771	/* The guest's view (read access) of its CR0 is unblemished. */
3772	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
3773	AssertRCReturn(rc, rc);
3774	Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR0));
3775
3776	/* Setup VT-x's view of the guest CR0. */
3777	/* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
3778	if (pVM->hm.s.fNestedPaging)
3779	{
3780	if (CPUMIsGuestPagingEnabledEx(pMixedCtx))
3781	{
3782	/* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3783	pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3784	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3785	}
3786	else
3787	{
3788	/* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3789	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3790	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3791	}
3792
3793	/* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3794	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3795	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3796
3797	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3798	AssertRCReturn(rc, rc);
3799	}
3800	else
3801	u32GuestCR0 \|= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3802
3803	/*
3804	* Guest FPU bits.
3805	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
3806	* CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3807	*/
3808	u32GuestCR0 \|= X86_CR0_NE;
3809	bool fInterceptNM = false;
3810	if (CPUMIsGuestFPUStateActive(pVCpu))
3811	{
3812	fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
3813	/* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
3814	We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
3815	}
3816	else
3817	{
3818	fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
3819	u32GuestCR0 \|= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
3820	\| X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
3821	}
3822
3823	/* Catch floating point exceptions if we need to report them to the guest in a different way. */
3824	bool fInterceptMF = false;
3825	if (!(pMixedCtx->cr0 & X86_CR0_NE))
3826	fInterceptMF = true;
3827
3828	/* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
3829	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3830	{
3831	Assert(PDMVmmDevHeapIsEnabled(pVM));
3832	Assert(pVM->hm.s.vmx.pRealModeTSS);
3833	pVCpu->hm.s.vmx.u32XcptBitmap \|= HMVMX_REAL_MODE_XCPT_MASK;
3834	fInterceptNM = true;
3835	fInterceptMF = true;
3836	}
3837	else
3838	{
3839	/* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
3840	pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3841	}
3842	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
3843
3844	if (fInterceptNM)
3845	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_NM);
3846	else
3847	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
3848
3849	if (fInterceptMF)
3850	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_MF);
3851	else
3852	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
3853
3854	/* Additional intercepts for debugging, define these yourself explicitly. */
3855	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3856	pVCpu->hm.s.vmx.u32XcptBitmap \|= 0
3857	\| RT_BIT(X86_XCPT_BP)
3858	\| RT_BIT(X86_XCPT_DE)
3859	\| RT_BIT(X86_XCPT_NM)
3860	\| RT_BIT(X86_XCPT_TS)
3861	\| RT_BIT(X86_XCPT_UD)
3862	\| RT_BIT(X86_XCPT_NP)
3863	\| RT_BIT(X86_XCPT_SS)
3864	\| RT_BIT(X86_XCPT_GP)
3865	\| RT_BIT(X86_XCPT_PF)
3866	\| RT_BIT(X86_XCPT_MF)
3867	;
3868	#elif defined(HMVMX_ALWAYS_TRAP_PF)
3869	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
3870	#endif
3871
3872	Assert(pVM->hm.s.fNestedPaging \|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3873
3874	/* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
3875	uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3876	uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3877	if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3878	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
3879	else
3880	Assert((uSetCR0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
3881
3882	u32GuestCR0 \|= uSetCR0;
3883	u32GuestCR0 &= uZapCR0;
3884	u32GuestCR0 &= ~(X86_CR0_CD \| X86_CR0_NW); /* Always enable caching. */
3885
3886	/* Write VT-x's view of the guest CR0 into the VMCS. */
3887	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
3888	AssertRCReturn(rc, rc);
3889	Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", pVCpu->idCpu, u32GuestCR0, uSetCR0,
3890	uZapCR0));
3891
3892	/*
3893	* CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3894	* by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3895	* we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3896	*/
3897	uint32_t u32CR0Mask = 0;
3898	u32CR0Mask = X86_CR0_PE
3899	\| X86_CR0_NE
3900	\| X86_CR0_WP
3901	\| X86_CR0_PG
3902	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3903	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3904	\| X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3905
3906	/** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3907	* enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3908	* and @bugref{6944}. */
3909	#if 0
3910	if (pVM->hm.s.vmx.fUnrestrictedGuest)
3911	u32CR0Mask &= ~X86_CR0_PE;
3912	#endif
3913	if (pVM->hm.s.fNestedPaging)
3914	u32CR0Mask &= ~X86_CR0_WP;
3915
3916	/* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
3917	if (fInterceptNM)
3918	{
3919	u32CR0Mask \|= X86_CR0_TS
3920	\| X86_CR0_MP;
3921	}
3922
3923	/* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3924	pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3925	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3926	AssertRCReturn(rc, rc);
3927	Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_MASK=%#RX32\n", pVCpu->idCpu, u32CR0Mask));
3928
3929	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR0);
3930	}
3931	return rc;
3932	}
3933
3934
3935	/**
3936	* Loads the guest control registers (CR3, CR4) into the guest-state area
3937	* in the VMCS.
3938	*
3939	* @returns VBox strict status code.
3940	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
3941	* without unrestricted guest access and the VMMDev is not presently
3942	* mapped (e.g. EFI32).
3943	*
3944	* @param pVCpu The cross context virtual CPU structure.
3945	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3946	* out-of-sync. Make sure to update the required fields
3947	* before using them.
3948	*
3949	* @remarks No-long-jump zone!!!
3950	*/
3951	static VBOXSTRICTRC hmR0VmxLoadGuestCR3AndCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3952	{
3953	int rc = VINF_SUCCESS;
3954	PVM pVM = pVCpu->CTX_SUFF(pVM);
3955
3956	/*
3957	* Guest CR2.
3958	* It's always loaded in the assembler code. Nothing to do here.
3959	*/
3960
3961	/*
3962	* Guest CR3.
3963	*/
3964	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3))
3965	{
3966	RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3967	if (pVM->hm.s.fNestedPaging)
3968	{
3969	pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3970
3971	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3972	Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3973	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3974	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3975
3976	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3977	pVCpu->hm.s.vmx.HCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
3978	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3979
3980	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3981	AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3982	&& ((pVCpu->hm.s.vmx.HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
3983	("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3984	AssertMsg( !((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
3985	\|\| (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
3986	("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3987
3988	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3989	AssertRCReturn(rc, rc);
3990	Log4(("Load[%RU32]: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->idCpu, pVCpu->hm.s.vmx.HCPhysEPTP));
3991
3992	if ( pVM->hm.s.vmx.fUnrestrictedGuest
3993	\|\| CPUMIsGuestPagingEnabledEx(pMixedCtx))
3994	{
3995	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3996	if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3997	{
3998	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
3999	AssertRCReturn(rc, rc);
4000	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
4001	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
4002	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
4003	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
4004	AssertRCReturn(rc, rc);
4005	}
4006
4007	/* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
4008	have Unrestricted Execution to handle the guest when it's not using paging. */
4009	GCPhysGuestCR3 = pMixedCtx->cr3;
4010	}
4011	else
4012	{
4013	/*
4014	* The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
4015	* directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
4016	* EPT takes care of translating it to host-physical addresses.
4017	*/
4018	RTGCPHYS GCPhys;
4019	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
4020
4021	/* We obtain it here every time as the guest could have relocated this PCI region. */
4022	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
4023	if (RT_SUCCESS(rc))
4024	{ /* likely */ }
4025	else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
4026	{
4027	Log4(("Load[%RU32]: VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n", pVCpu->idCpu));
4028	return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
4029	}
4030	else
4031	AssertMsgFailedReturn(("%Rrc\n", rc), rc);
4032
4033	GCPhysGuestCR3 = GCPhys;
4034	}
4035
4036	Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RGp (GstN)\n", pVCpu->idCpu, GCPhysGuestCR3));
4037	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
4038	}
4039	else
4040	{
4041	/* Non-nested paging case, just use the hypervisor's CR3. */
4042	RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
4043
4044	Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", pVCpu->idCpu, HCPhysGuestCR3));
4045	rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
4046	}
4047	AssertRCReturn(rc, rc);
4048
4049	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR3);
4050	}
4051
4052	/*
4053	* Guest CR4.
4054	* ASSUMES this is done everytime we get in from ring-3! (XCR0)
4055	*/
4056	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4))
4057	{
4058	Assert(!(pMixedCtx->cr4 >> 32));
4059	uint32_t u32GuestCR4 = pMixedCtx->cr4;
4060
4061	/* The guest's view of its CR4 is unblemished. */
4062	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
4063	AssertRCReturn(rc, rc);
4064	Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR4));
4065
4066	/* Setup VT-x's view of the guest CR4. */
4067	/*
4068	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
4069	* interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
4070	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
4071	*/
4072	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4073	{
4074	Assert(pVM->hm.s.vmx.pRealModeTSS);
4075	Assert(PDMVmmDevHeapIsEnabled(pVM));
4076	u32GuestCR4 &= ~X86_CR4_VME;
4077	}
4078
4079	if (pVM->hm.s.fNestedPaging)
4080	{
4081	if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
4082	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
4083	{
4084	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
4085	u32GuestCR4 \|= X86_CR4_PSE;
4086	/* Our identity mapping is a 32-bit page directory. */
4087	u32GuestCR4 &= ~X86_CR4_PAE;
4088	}
4089	/* else use guest CR4.*/
4090	}
4091	else
4092	{
4093	/*
4094	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
4095	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
4096	*/
4097	switch (pVCpu->hm.s.enmShadowMode)
4098	{
4099	case PGMMODE_REAL: /* Real-mode. */
4100	case PGMMODE_PROTECTED: /* Protected mode without paging. */
4101	case PGMMODE_32_BIT: /* 32-bit paging. */
4102	{
4103	u32GuestCR4 &= ~X86_CR4_PAE;
4104	break;
4105	}
4106
4107	case PGMMODE_PAE: /* PAE paging. */
4108	case PGMMODE_PAE_NX: /* PAE paging with NX. */
4109	{
4110	u32GuestCR4 \|= X86_CR4_PAE;
4111	break;
4112	}
4113
4114	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
4115	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
4116	#ifdef VBOX_ENABLE_64_BITS_GUESTS
4117	break;
4118	#endif
4119	default:
4120	AssertFailed();
4121	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4122	}
4123	}
4124
4125	/* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
4126	uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4127	uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4128	u32GuestCR4 \|= uSetCR4;
4129	u32GuestCR4 &= uZapCR4;
4130
4131	/* Write VT-x's view of the guest CR4 into the VMCS. */
4132	Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", pVCpu->idCpu, u32GuestCR4, uSetCR4, uZapCR4));
4133	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
4134	AssertRCReturn(rc, rc);
4135
4136	/* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM-exit. */
4137	uint32_t u32CR4Mask = X86_CR4_VME
4138	\| X86_CR4_PAE
4139	\| X86_CR4_PGE
4140	\| X86_CR4_PSE
4141	\| X86_CR4_VMXE;
4142	if (pVM->cpum.ro.HostFeatures.fXSaveRstor)
4143	u32CR4Mask \|= X86_CR4_OSXSAVE;
4144	if (pVM->cpum.ro.GuestFeatures.fPcid)
4145	u32CR4Mask \|= X86_CR4_PCIDE;
4146	pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
4147	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
4148	AssertRCReturn(rc, rc);
4149
4150	/* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
4151	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
4152
4153	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR4);
4154	}
4155	return rc;
4156	}
4157
4158
4159	/**
4160	* Loads the guest debug registers into the guest-state area in the VMCS.
4161	*
4162	* This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
4163	*
4164	* The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
4165	*
4166	* @returns VBox status code.
4167	* @param pVCpu The cross context virtual CPU structure.
4168	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4169	* out-of-sync. Make sure to update the required fields
4170	* before using them.
4171	*
4172	* @remarks No-long-jump zone!!!
4173	*/
4174	static int hmR0VmxLoadSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4175	{
4176	if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
4177	return VINF_SUCCESS;
4178
4179	#ifdef VBOX_STRICT
4180	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
4181	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
4182	{
4183	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
4184	Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK \| X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
4185	Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
4186	}
4187	#endif
4188
4189	int rc;
4190	PVM pVM = pVCpu->CTX_SUFF(pVM);
4191	bool fSteppingDB = false;
4192	bool fInterceptMovDRx = false;
4193	if (pVCpu->hm.s.fSingleInstruction)
4194	{
4195	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
4196	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
4197	{
4198	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
4199	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4200	AssertRCReturn(rc, rc);
4201	Assert(fSteppingDB == false);
4202	}
4203	else
4204	{
4205	pMixedCtx->eflags.u32 \|= X86_EFL_TF;
4206	pVCpu->hm.s.fClearTrapFlag = true;
4207	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
4208	fSteppingDB = true;
4209	}
4210	}
4211
4212	if ( fSteppingDB
4213	\|\| (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
4214	{
4215	/*
4216	* Use the combined guest and host DRx values found in the hypervisor
4217	* register set because the debugger has breakpoints active or someone
4218	* is single stepping on the host side without a monitor trap flag.
4219	*
4220	* Note! DBGF expects a clean DR6 state before executing guest code.
4221	*/
4222	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4223	if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4224	&& !CPUMIsHyperDebugStateActivePending(pVCpu))
4225	{
4226	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4227	Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
4228	Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
4229	}
4230	else
4231	#endif
4232	if (!CPUMIsHyperDebugStateActive(pVCpu))
4233	{
4234	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4235	Assert(CPUMIsHyperDebugStateActive(pVCpu));
4236	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
4237	}
4238
4239	/* Update DR7. (The other DRx values are handled by CPUM one way or the other.) */
4240	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)CPUMGetHyperDR7(pVCpu));
4241	AssertRCReturn(rc, rc);
4242
4243	pVCpu->hm.s.fUsingHyperDR7 = true;
4244	fInterceptMovDRx = true;
4245	}
4246	else
4247	{
4248	/*
4249	* If the guest has enabled debug registers, we need to load them prior to
4250	* executing guest code so they'll trigger at the right time.
4251	*/
4252	if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD)) /** @todo Why GD? */
4253	{
4254	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4255	if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4256	&& !CPUMIsGuestDebugStateActivePending(pVCpu))
4257	{
4258	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4259	Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
4260	Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
4261	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4262	}
4263	else
4264	#endif
4265	if (!CPUMIsGuestDebugStateActive(pVCpu))
4266	{
4267	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4268	Assert(CPUMIsGuestDebugStateActive(pVCpu));
4269	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
4270	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4271	}
4272	Assert(!fInterceptMovDRx);
4273	}
4274	/*
4275	* If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
4276	* must intercept #DB in order to maintain a correct DR6 guest value, and
4277	* because we need to intercept it to prevent nested #DBs from hanging the
4278	* CPU, we end up always having to intercept it. See hmR0VmxInitXcptBitmap.
4279	*/
4280	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4281	else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
4282	&& !CPUMIsGuestDebugStateActive(pVCpu))
4283	#else
4284	else if (!CPUMIsGuestDebugStateActive(pVCpu))
4285	#endif
4286	{
4287	fInterceptMovDRx = true;
4288	}
4289
4290	/* Update guest DR7. */
4291	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
4292	AssertRCReturn(rc, rc);
4293
4294	pVCpu->hm.s.fUsingHyperDR7 = false;
4295	}
4296
4297	/*
4298	* Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions.
4299	*/
4300	if (fInterceptMovDRx)
4301	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4302	else
4303	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4304	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4305	AssertRCReturn(rc, rc);
4306
4307	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_DEBUG);
4308	return VINF_SUCCESS;
4309	}
4310
4311
4312	#ifdef VBOX_STRICT
4313	/**
4314	* Strict function to validate segment registers.
4315	*
4316	* @remarks ASSUMES CR0 is up to date.
4317	*/
4318	static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4319	{
4320	/* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
4321	/* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
4322	* only updates the VMCS' copy of the value with the unusable bit and doesn't change the guest-context value. */
4323	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4324	&& ( !CPUMIsGuestInRealModeEx(pCtx)
4325	&& !CPUMIsGuestInV86ModeEx(pCtx)))
4326	{
4327	/* Protected mode checks */
4328	/* CS */
4329	Assert(pCtx->cs.Attr.n.u1Present);
4330	Assert(!(pCtx->cs.Attr.u & 0xf00));
4331	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4332	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4333	\|\| !(pCtx->cs.Attr.n.u1Granularity));
4334	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4335	\|\| (pCtx->cs.Attr.n.u1Granularity));
4336	/* CS cannot be loaded with NULL in protected mode. */
4337	Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
4338	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
4339	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4340	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
4341	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4342	else
4343	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4344	/* SS */
4345	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4346	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4347	if ( !(pCtx->cr0 & X86_CR0_PE)
4348	\|\| pCtx->cs.Attr.n.u4Type == 3)
4349	{
4350	Assert(!pCtx->ss.Attr.n.u2Dpl);
4351	}
4352	if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4353	{
4354	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4355	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
4356	Assert(pCtx->ss.Attr.n.u1Present);
4357	Assert(!(pCtx->ss.Attr.u & 0xf00));
4358	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4359	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4360	\|\| !(pCtx->ss.Attr.n.u1Granularity));
4361	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4362	\|\| (pCtx->ss.Attr.n.u1Granularity));
4363	}
4364	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
4365	if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4366	{
4367	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4368	Assert(pCtx->ds.Attr.n.u1Present);
4369	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4370	Assert(!(pCtx->ds.Attr.u & 0xf00));
4371	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4372	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4373	\|\| !(pCtx->ds.Attr.n.u1Granularity));
4374	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4375	\|\| (pCtx->ds.Attr.n.u1Granularity));
4376	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4377	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4378	}
4379	if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4380	{
4381	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4382	Assert(pCtx->es.Attr.n.u1Present);
4383	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4384	Assert(!(pCtx->es.Attr.u & 0xf00));
4385	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4386	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4387	\|\| !(pCtx->es.Attr.n.u1Granularity));
4388	Assert( !(pCtx->es.u32Limit & 0xfff00000)
4389	\|\| (pCtx->es.Attr.n.u1Granularity));
4390	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4391	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4392	}
4393	if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4394	{
4395	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4396	Assert(pCtx->fs.Attr.n.u1Present);
4397	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4398	Assert(!(pCtx->fs.Attr.u & 0xf00));
4399	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4400	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4401	\|\| !(pCtx->fs.Attr.n.u1Granularity));
4402	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4403	\|\| (pCtx->fs.Attr.n.u1Granularity));
4404	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4405	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4406	}
4407	if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4408	{
4409	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4410	Assert(pCtx->gs.Attr.n.u1Present);
4411	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4412	Assert(!(pCtx->gs.Attr.u & 0xf00));
4413	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4414	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4415	\|\| !(pCtx->gs.Attr.n.u1Granularity));
4416	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4417	\|\| (pCtx->gs.Attr.n.u1Granularity));
4418	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4419	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4420	}
4421	/* 64-bit capable CPUs. */
4422	# if HC_ARCH_BITS == 64
4423	Assert(!(pCtx->cs.u64Base >> 32));
4424	Assert(!pCtx->ss.Attr.u \|\| !(pCtx->ss.u64Base >> 32));
4425	Assert(!pCtx->ds.Attr.u \|\| !(pCtx->ds.u64Base >> 32));
4426	Assert(!pCtx->es.Attr.u \|\| !(pCtx->es.u64Base >> 32));
4427	# endif
4428	}
4429	else if ( CPUMIsGuestInV86ModeEx(pCtx)
4430	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
4431	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
4432	{
4433	/* Real and v86 mode checks. */
4434	/* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4435	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4436	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4437	{
4438	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4439	}
4440	else
4441	{
4442	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4443	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4444	}
4445
4446	/* CS */
4447	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4448	Assert(pCtx->cs.u32Limit == 0xffff);
4449	Assert(u32CSAttr == 0xf3);
4450	/* SS */
4451	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4452	Assert(pCtx->ss.u32Limit == 0xffff);
4453	Assert(u32SSAttr == 0xf3);
4454	/* DS */
4455	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4456	Assert(pCtx->ds.u32Limit == 0xffff);
4457	Assert(u32DSAttr == 0xf3);
4458	/* ES */
4459	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4460	Assert(pCtx->es.u32Limit == 0xffff);
4461	Assert(u32ESAttr == 0xf3);
4462	/* FS */
4463	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4464	Assert(pCtx->fs.u32Limit == 0xffff);
4465	Assert(u32FSAttr == 0xf3);
4466	/* GS */
4467	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4468	Assert(pCtx->gs.u32Limit == 0xffff);
4469	Assert(u32GSAttr == 0xf3);
4470	/* 64-bit capable CPUs. */
4471	# if HC_ARCH_BITS == 64
4472	Assert(!(pCtx->cs.u64Base >> 32));
4473	Assert(!u32SSAttr \|\| !(pCtx->ss.u64Base >> 32));
4474	Assert(!u32DSAttr \|\| !(pCtx->ds.u64Base >> 32));
4475	Assert(!u32ESAttr \|\| !(pCtx->es.u64Base >> 32));
4476	# endif
4477	}
4478	}
4479	#endif /* VBOX_STRICT */
4480
4481
4482	/**
4483	* Writes a guest segment register into the guest-state area in the VMCS.
4484	*
4485	* @returns VBox status code.
4486	* @param pVCpu The cross context virtual CPU structure.
4487	* @param idxSel Index of the selector in the VMCS.
4488	* @param idxLimit Index of the segment limit in the VMCS.
4489	* @param idxBase Index of the segment base in the VMCS.
4490	* @param idxAccess Index of the access rights of the segment in the VMCS.
4491	* @param pSelReg Pointer to the segment selector.
4492	*
4493	* @remarks No-long-jump zone!!!
4494	*/
4495	static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
4496	uint32_t idxAccess, PCPUMSELREG pSelReg)
4497	{
4498	int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4499	rc \|= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4500	rc \|= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4501	AssertRCReturn(rc, rc);
4502
4503	uint32_t u32Access = pSelReg->Attr.u;
4504	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4505	{
4506	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4507	u32Access = 0xf3;
4508	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4509	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4510	}
4511	else
4512	{
4513	/*
4514	* The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
4515	* real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
4516	* protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
4517	* loaded in protected-mode have their attribute as 0.
4518	*/
4519	if (!u32Access)
4520	u32Access = X86DESCATTR_UNUSABLE;
4521	}
4522
4523	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4524	AssertMsg((u32Access & X86DESCATTR_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
4525	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4526
4527	rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4528	AssertRCReturn(rc, rc);
4529	return rc;
4530	}
4531
4532
4533	/**
4534	* Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4535	* into the guest-state area in the VMCS.
4536	*
4537	* @returns VBox status code.
4538	* @param pVCpu The cross context virtual CPU structure.
4539	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4540	* out-of-sync. Make sure to update the required fields
4541	* before using them.
4542	*
4543	* @remarks ASSUMES pMixedCtx->cr0 is up to date (strict builds validation).
4544	* @remarks No-long-jump zone!!!
4545	*/
4546	static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4547	{
4548	int rc = VERR_INTERNAL_ERROR_5;
4549	PVM pVM = pVCpu->CTX_SUFF(pVM);
4550
4551	/*
4552	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
4553	*/
4554	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS))
4555	{
4556	/* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
4557	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4558	{
4559	pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
4560	pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
4561	pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
4562	pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
4563	pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
4564	pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
4565	}
4566
4567	#ifdef VBOX_WITH_REM
4568	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4569	{
4570	Assert(pVM->hm.s.vmx.pRealModeTSS);
4571	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4572	if ( pVCpu->hm.s.vmx.fWasInRealMode
4573	&& PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4574	{
4575	/* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4576	in real-mode (e.g. OpenBSD 4.0) */
4577	REMFlushTBs(pVM);
4578	Log4(("Load[%RU32]: Switch to protected mode detected!\n", pVCpu->idCpu));
4579	pVCpu->hm.s.vmx.fWasInRealMode = false;
4580	}
4581	}
4582	#endif
4583	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_CS_SEL, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
4584	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs);
4585	AssertRCReturn(rc, rc);
4586	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_SS_SEL, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
4587	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss);
4588	AssertRCReturn(rc, rc);
4589	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_DS_SEL, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
4590	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds);
4591	AssertRCReturn(rc, rc);
4592	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_ES_SEL, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
4593	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es);
4594	AssertRCReturn(rc, rc);
4595	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FS_SEL, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
4596	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs);
4597	AssertRCReturn(rc, rc);
4598	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_GS_SEL, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
4599	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs);
4600	AssertRCReturn(rc, rc);
4601
4602	#ifdef VBOX_STRICT
4603	/* Validate. */
4604	hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
4605	#endif
4606
4607	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS);
4608	Log4(("Load[%RU32]: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pVCpu->idCpu, pMixedCtx->cs.Sel,
4609	pMixedCtx->cs.u64Base, pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
4610	}
4611
4612	/*
4613	* Guest TR.
4614	*/
4615	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR))
4616	{
4617	/*
4618	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
4619	* using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
4620	* See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4621	*/
4622	uint16_t u16Sel = 0;
4623	uint32_t u32Limit = 0;
4624	uint64_t u64Base = 0;
4625	uint32_t u32AccessRights = 0;
4626
4627	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4628	{
4629	u16Sel = pMixedCtx->tr.Sel;
4630	u32Limit = pMixedCtx->tr.u32Limit;
4631	u64Base = pMixedCtx->tr.u64Base;
4632	u32AccessRights = pMixedCtx->tr.Attr.u;
4633	}
4634	else
4635	{
4636	Assert(pVM->hm.s.vmx.pRealModeTSS);
4637	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
4638
4639	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4640	RTGCPHYS GCPhys;
4641	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4642	AssertRCReturn(rc, rc);
4643
4644	X86DESCATTR DescAttr;
4645	DescAttr.u = 0;
4646	DescAttr.n.u1Present = 1;
4647	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4648
4649	u16Sel = 0;
4650	u32Limit = HM_VTX_TSS_SIZE;
4651	u64Base = GCPhys; /* in real-mode phys = virt. */
4652	u32AccessRights = DescAttr.u;
4653	}
4654
4655	/* Validate. */
4656	Assert(!(u16Sel & RT_BIT(2)));
4657	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4658	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4659	AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4660	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4661	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4662	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4663	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4664	Assert( (u32Limit & 0xfff) == 0xfff
4665	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4666	Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
4667	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4668
4669	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_TR_SEL, u16Sel);
4670	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
4671	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
4672	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
4673	AssertRCReturn(rc, rc);
4674
4675	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_TR);
4676	Log4(("Load[%RU32]: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", pVCpu->idCpu, u64Base));
4677	}
4678
4679	/*
4680	* Guest GDTR.
4681	*/
4682	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR))
4683	{
4684	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt);
4685	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt);
4686	AssertRCReturn(rc, rc);
4687
4688	/* Validate. */
4689	Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4690
4691	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_GDTR);
4692	Log4(("Load[%RU32]: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->gdtr.pGdt));
4693	}
4694
4695	/*
4696	* Guest LDTR.
4697	*/
4698	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR))
4699	{
4700	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4701	uint32_t u32Access = 0;
4702	if (!pMixedCtx->ldtr.Attr.u)
4703	u32Access = X86DESCATTR_UNUSABLE;
4704	else
4705	u32Access = pMixedCtx->ldtr.Attr.u;
4706
4707	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, pMixedCtx->ldtr.Sel);
4708	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit);
4709	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base);
4710	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
4711	AssertRCReturn(rc, rc);
4712
4713	/* Validate. */
4714	if (!(u32Access & X86DESCATTR_UNUSABLE))
4715	{
4716	Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4717	Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4718	Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4719	Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4720	Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4721	Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4722	Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
4723	\|\| !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4724	Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
4725	\|\| pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4726	}
4727
4728	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LDTR);
4729	Log4(("Load[%RU32]: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->ldtr.u64Base));
4730	}
4731
4732	/*
4733	* Guest IDTR.
4734	*/
4735	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR))
4736	{
4737	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt);
4738	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt);
4739	AssertRCReturn(rc, rc);
4740
4741	/* Validate. */
4742	Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4743
4744	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_IDTR);
4745	Log4(("Load[%RU32]: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->idtr.pIdt));
4746	}
4747
4748	return VINF_SUCCESS;
4749	}
4750
4751
4752	/**
4753	* Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4754	* areas.
4755	*
4756	* These MSRs will automatically be loaded to the host CPU on every successful
4757	* VM-entry and stored from the host CPU on every successful VM-exit. This also
4758	* creates/updates MSR slots for the host MSRs. The actual host MSR values are
4759	* -not- updated here for performance reasons. See hmR0VmxSaveHostMsrs().
4760	*
4761	* Also loads the sysenter MSRs into the guest-state area in the VMCS.
4762	*
4763	* @returns VBox status code.
4764	* @param pVCpu The cross context virtual CPU structure.
4765	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4766	* out-of-sync. Make sure to update the required fields
4767	* before using them.
4768	*
4769	* @remarks No-long-jump zone!!!
4770	*/
4771	static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4772	{
4773	AssertPtr(pVCpu);
4774	AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4775
4776	/*
4777	* MSRs that we use the auto-load/store MSR area in the VMCS.
4778	*/
4779	PVM pVM = pVCpu->CTX_SUFF(pVM);
4780	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS))
4781	{
4782	/* For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs(). */
4783	#if HC_ARCH_BITS == 32
4784	if (pVM->hm.s.fAllow64BitGuests)
4785	{
4786	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pMixedCtx->msrLSTAR, false, NULL);
4787	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pMixedCtx->msrSTAR, false, NULL);
4788	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pMixedCtx->msrSFMASK, false, NULL);
4789	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE, false, NULL);
4790	AssertRCReturn(rc, rc);
4791	# ifdef LOG_ENABLED
4792	PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4793	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4794	{
4795	Log4(("Load[%RU32]: MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", pVCpu->idCpu, i, pMsr->u32Msr,
4796	pMsr->u64Value));
4797	}
4798	# endif
4799	}
4800	#endif
4801	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4802	}
4803
4804	/*
4805	* Guest Sysenter MSRs.
4806	* These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4807	* VM-exits on WRMSRs for these MSRs.
4808	*/
4809	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR))
4810	{
4811	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
4812	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4813	}
4814
4815	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR))
4816	{
4817	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
4818	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4819	}
4820
4821	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR))
4822	{
4823	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
4824	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4825	}
4826
4827	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_EFER_MSR))
4828	{
4829	if (hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
4830	{
4831	/*
4832	* If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
4833	* but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
4834	*/
4835	if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4836	{
4837	int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pMixedCtx->msrEFER);
4838	AssertRCReturn(rc,rc);
4839	Log4(("Load[%RU32]: VMX_VMCS64_GUEST_EFER_FULL=%#RX64\n", pVCpu->idCpu, pMixedCtx->msrEFER));
4840	}
4841	else
4842	{
4843	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_EFER, pMixedCtx->msrEFER, false /* fUpdateHostMsr */,
4844	NULL /* pfAddedAndUpdated */);
4845	AssertRCReturn(rc, rc);
4846
4847	/* We need to intercept reads too, see @bugref{7386#c16}. */
4848	if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
4849	hmR0VmxSetMsrPermission(pVCpu, MSR_K6_EFER, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
4850	Log4(("Load[%RU32]: MSR[--]: u32Msr=%#RX32 u64Value=%#RX64 cMsrs=%u\n", pVCpu->idCpu, MSR_K6_EFER,
4851	pMixedCtx->msrEFER, pVCpu->hm.s.vmx.cMsrs));
4852	}
4853	}
4854	else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
4855	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K6_EFER);
4856	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_EFER_MSR);
4857	}
4858
4859	return VINF_SUCCESS;
4860	}
4861
4862
4863	/**
4864	* Loads the guest activity state into the guest-state area in the VMCS.
4865	*
4866	* @returns VBox status code.
4867	* @param pVCpu The cross context virtual CPU structure.
4868	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4869	* out-of-sync. Make sure to update the required fields
4870	* before using them.
4871	*
4872	* @remarks No-long-jump zone!!!
4873	*/
4874	static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4875	{
4876	NOREF(pMixedCtx);
4877	/** @todo See if we can make use of other states, e.g.
4878	* VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
4879	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE))
4880	{
4881	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
4882	AssertRCReturn(rc, rc);
4883
4884	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE);
4885	}
4886	return VINF_SUCCESS;
4887	}
4888
4889
4890	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
4891	/**
4892	* Check if guest state allows safe use of 32-bit switcher again.
4893	*
4894	* Segment bases and protected mode structures must be 32-bit addressable
4895	* because the 32-bit switcher will ignore high dword when writing these VMCS
4896	* fields. See @bugref{8432} for details.
4897	*
4898	* @returns true if safe, false if must continue to use the 64-bit switcher.
4899	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4900	* out-of-sync. Make sure to update the required fields
4901	* before using them.
4902	*
4903	* @remarks No-long-jump zone!!!
4904	*/
4905	static bool hmR0VmxIs32BitSwitcherSafe(PCPUMCTX pMixedCtx)
4906	{
4907	if (pMixedCtx->gdtr.pGdt & UINT64_C(0xffffffff00000000))
4908	return false;
4909	if (pMixedCtx->idtr.pIdt & UINT64_C(0xffffffff00000000))
4910	return false;
4911	if (pMixedCtx->ldtr.u64Base & UINT64_C(0xffffffff00000000))
4912	return false;
4913	if (pMixedCtx->tr.u64Base & UINT64_C(0xffffffff00000000))
4914	return false;
4915	if (pMixedCtx->es.u64Base & UINT64_C(0xffffffff00000000))
4916	return false;
4917	if (pMixedCtx->cs.u64Base & UINT64_C(0xffffffff00000000))
4918	return false;
4919	if (pMixedCtx->ss.u64Base & UINT64_C(0xffffffff00000000))
4920	return false;
4921	if (pMixedCtx->ds.u64Base & UINT64_C(0xffffffff00000000))
4922	return false;
4923	if (pMixedCtx->fs.u64Base & UINT64_C(0xffffffff00000000))
4924	return false;
4925	if (pMixedCtx->gs.u64Base & UINT64_C(0xffffffff00000000))
4926	return false;
4927	/* All good, bases are 32-bit. */
4928	return true;
4929	}
4930	#endif
4931
4932
4933	/**
4934	* Sets up the appropriate function to run guest code.
4935	*
4936	* @returns VBox status code.
4937	* @param pVCpu The cross context virtual CPU structure.
4938	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4939	* out-of-sync. Make sure to update the required fields
4940	* before using them.
4941	*
4942	* @remarks No-long-jump zone!!!
4943	*/
4944	static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4945	{
4946	if (CPUMIsGuestInLongModeEx(pMixedCtx))
4947	{
4948	#ifndef VBOX_ENABLE_64_BITS_GUESTS
4949	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4950	#endif
4951	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4952	#if HC_ARCH_BITS == 32
4953	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4954	if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4955	{
4956	if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4957	{
4958	/* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4959	AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_VMX_EXIT_CTLS
4960	\| HM_CHANGED_VMX_ENTRY_CTLS
4961	\| HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4962	}
4963	pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4964
4965	/* Mark that we've switched to 64-bit handler, we can't safely switch back to 32-bit for
4966	the rest of the VM run (until VM reset). See @bugref{8432#c7}. */
4967	pVCpu->hm.s.vmx.fSwitchedTo64on32 = true;
4968	Log4(("Load[%RU32]: hmR0VmxSetupVMRunHandler: selected 64-bit switcher\n", pVCpu->idCpu));
4969	}
4970	#else
4971	/* 64-bit host. */
4972	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4973	#endif
4974	}
4975	else
4976	{
4977	/* Guest is not in long mode, use the 32-bit handler. */
4978	#if HC_ARCH_BITS == 32
4979	if ( pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32
4980	&& !pVCpu->hm.s.vmx.fSwitchedTo64on32 /* If set, guest mode change does not imply switcher change. */
4981	&& pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4982	{
4983	/* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4984	AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_VMX_EXIT_CTLS
4985	\| HM_CHANGED_VMX_ENTRY_CTLS
4986	\| HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4987	}
4988	# ifdef VBOX_ENABLE_64_BITS_GUESTS
4989	/*
4990	* Keep using the 64-bit switcher even though we're in 32-bit because of bad Intel design, see @bugref{8432#c7}.
4991	* If real-on-v86 mode is active, clear the 64-bit switcher flag because now we know the guest is in a sane
4992	* state where it's safe to use the 32-bit switcher. Otherwise check the guest state if it's safe to use
4993	* the much faster 32-bit switcher again.
4994	*/
4995	if (!pVCpu->hm.s.vmx.fSwitchedTo64on32)
4996	{
4997	if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4998	Log4(("Load[%RU32]: hmR0VmxSetupVMRunHandler: selected 32-bit switcher\n", pVCpu->idCpu));
4999	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
5000	}
5001	else
5002	{
5003	Assert(pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64);
5004	if ( pVCpu->hm.s.vmx.RealMode.fRealOnV86Active
5005	\|\| hmR0VmxIs32BitSwitcherSafe(pMixedCtx))
5006	{
5007	pVCpu->hm.s.vmx.fSwitchedTo64on32 = false;
5008	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
5009	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_EFER_MSR
5010	\| HM_CHANGED_VMX_ENTRY_CTLS
5011	\| HM_CHANGED_VMX_EXIT_CTLS
5012	\| HM_CHANGED_HOST_CONTEXT);
5013	Log4(("Load[%RU32]: hmR0VmxSetupVMRunHandler: selected 32-bit switcher (safe)\n", pVCpu->idCpu));
5014	}
5015	}
5016	# else
5017	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
5018	# endif
5019	#else
5020	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
5021	#endif
5022	}
5023	Assert(pVCpu->hm.s.vmx.pfnStartVM);
5024	return VINF_SUCCESS;
5025	}
5026
5027
5028	/**
5029	* Wrapper for running the guest code in VT-x.
5030	*
5031	* @returns VBox status code, no informational status codes.
5032	* @param pVM The cross context VM structure.
5033	* @param pVCpu The cross context virtual CPU structure.
5034	* @param pCtx Pointer to the guest-CPU context.
5035	*
5036	* @remarks No-long-jump zone!!!
5037	*/
5038	DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
5039	{
5040	/*
5041	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
5042	* using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
5043	* Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
5044	*/
5045	bool const fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
5046	/** @todo Add stats for resume vs launch. */
5047	#ifdef VBOX_WITH_KERNEL_USING_XMM
5048	int rc = hmR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
5049	#else
5050	int rc = pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
5051	#endif
5052	AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
5053	return rc;
5054	}
5055
5056
5057	/**
5058	* Reports world-switch error and dumps some useful debug info.
5059	*
5060	* @param pVM The cross context VM structure.
5061	* @param pVCpu The cross context virtual CPU structure.
5062	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
5063	* @param pCtx Pointer to the guest-CPU context.
5064	* @param pVmxTransient Pointer to the VMX transient structure (only
5065	* exitReason updated).
5066	*/
5067	static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
5068	{
5069	Assert(pVM);
5070	Assert(pVCpu);
5071	Assert(pCtx);
5072	Assert(pVmxTransient);
5073	HMVMX_ASSERT_PREEMPT_SAFE();
5074
5075	Log4(("VM-entry failure: %Rrc\n", rcVMRun));
5076	switch (rcVMRun)
5077	{
5078	case VERR_VMX_INVALID_VMXON_PTR:
5079	AssertFailed();
5080	break;
5081	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
5082	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
5083	{
5084	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
5085	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
5086	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
5087	AssertRC(rc);
5088
5089	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
5090	/* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
5091	Cannot do it here as we may have been long preempted. */
5092
5093	#ifdef VBOX_STRICT
5094	Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
5095	pVmxTransient->uExitReason));
5096	Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
5097	Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
5098	if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
5099	Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
5100	else
5101	Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
5102	Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
5103	Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
5104
5105	/* VMX control bits. */
5106	uint32_t u32Val;
5107	uint64_t u64Val;
5108	RTHCUINTREG uHCReg;
5109	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
5110	Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
5111	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
5112	Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
5113	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
5114	{
5115	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
5116	Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
5117	}
5118	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
5119	Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
5120	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
5121	Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
5122	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
5123	Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
5124	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
5125	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
5126	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
5127	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
5128	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
5129	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
5130	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
5131	Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
5132	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
5133	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
5134	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5135	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
5136	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5137	Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
5138	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
5139	Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
5140	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
5141	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
5142	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
5143	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
5144	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
5145	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
5146	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
5147	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5148	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
5149	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
5150	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
5151	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5152	if (pVM->hm.s.fNestedPaging)
5153	{
5154	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
5155	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
5156	}
5157
5158	/* Guest bits. */
5159	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
5160	Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
5161	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
5162	Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
5163	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
5164	Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
5165	if (pVM->hm.s.vmx.fVpid)
5166	{
5167	rc = VMXReadVmcs32(VMX_VMCS16_VPID, &u32Val); AssertRC(rc);
5168	Log4(("VMX_VMCS16_VPID %u\n", u32Val));
5169	}
5170
5171	/* Host bits. */
5172	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
5173	Log4(("Host CR0 %#RHr\n", uHCReg));
5174	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
5175	Log4(("Host CR3 %#RHr\n", uHCReg));
5176	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
5177	Log4(("Host CR4 %#RHr\n", uHCReg));
5178
5179	RTGDTR HostGdtr;
5180	PCX86DESCHC pDesc;
5181	ASMGetGDTR(&HostGdtr);
5182	rc = VMXReadVmcs32(VMX_VMCS16_HOST_CS_SEL, &u32Val); AssertRC(rc);
5183	Log4(("Host CS %#08x\n", u32Val));
5184	if (u32Val < HostGdtr.cbGdt)
5185	{
5186	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5187	hmR0DumpDescriptor(pDesc, u32Val, "CS: ");
5188	}
5189
5190	rc = VMXReadVmcs32(VMX_VMCS16_HOST_DS_SEL, &u32Val); AssertRC(rc);
5191	Log4(("Host DS %#08x\n", u32Val));
5192	if (u32Val < HostGdtr.cbGdt)
5193	{
5194	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5195	hmR0DumpDescriptor(pDesc, u32Val, "DS: ");
5196	}
5197
5198	rc = VMXReadVmcs32(VMX_VMCS16_HOST_ES_SEL, &u32Val); AssertRC(rc);
5199	Log4(("Host ES %#08x\n", u32Val));
5200	if (u32Val < HostGdtr.cbGdt)
5201	{
5202	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5203	hmR0DumpDescriptor(pDesc, u32Val, "ES: ");
5204	}
5205
5206	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FS_SEL, &u32Val); AssertRC(rc);
5207	Log4(("Host FS %#08x\n", u32Val));
5208	if (u32Val < HostGdtr.cbGdt)
5209	{
5210	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5211	hmR0DumpDescriptor(pDesc, u32Val, "FS: ");
5212	}
5213
5214	rc = VMXReadVmcs32(VMX_VMCS16_HOST_GS_SEL, &u32Val); AssertRC(rc);
5215	Log4(("Host GS %#08x\n", u32Val));
5216	if (u32Val < HostGdtr.cbGdt)
5217	{
5218	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5219	hmR0DumpDescriptor(pDesc, u32Val, "GS: ");
5220	}
5221
5222	rc = VMXReadVmcs32(VMX_VMCS16_HOST_SS_SEL, &u32Val); AssertRC(rc);
5223	Log4(("Host SS %#08x\n", u32Val));
5224	if (u32Val < HostGdtr.cbGdt)
5225	{
5226	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5227	hmR0DumpDescriptor(pDesc, u32Val, "SS: ");
5228	}
5229
5230	rc = VMXReadVmcs32(VMX_VMCS16_HOST_TR_SEL, &u32Val); AssertRC(rc);
5231	Log4(("Host TR %#08x\n", u32Val));
5232	if (u32Val < HostGdtr.cbGdt)
5233	{
5234	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5235	hmR0DumpDescriptor(pDesc, u32Val, "TR: ");
5236	}
5237
5238	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
5239	Log4(("Host TR Base %#RHv\n", uHCReg));
5240	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
5241	Log4(("Host GDTR Base %#RHv\n", uHCReg));
5242	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
5243	Log4(("Host IDTR Base %#RHv\n", uHCReg));
5244	rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
5245	Log4(("Host SYSENTER CS %#08x\n", u32Val));
5246	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
5247	Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
5248	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
5249	Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
5250	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
5251	Log4(("Host RSP %#RHv\n", uHCReg));
5252	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
5253	Log4(("Host RIP %#RHv\n", uHCReg));
5254	# if HC_ARCH_BITS == 64
5255	Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
5256	Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
5257	Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
5258	Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
5259	Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
5260	Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
5261	# endif
5262	#endif /* VBOX_STRICT */
5263	break;
5264	}
5265
5266	default:
5267	/* Impossible */
5268	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
5269	break;
5270	}
5271	NOREF(pVM); NOREF(pCtx);
5272	}
5273
5274
5275	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
5276	#ifndef VMX_USE_CACHED_VMCS_ACCESSES
5277	# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
5278	#endif
5279	#ifdef VBOX_STRICT
5280	static bool hmR0VmxIsValidWriteField(uint32_t idxField)
5281	{
5282	switch (idxField)
5283	{
5284	case VMX_VMCS_GUEST_RIP:
5285	case VMX_VMCS_GUEST_RSP:
5286	case VMX_VMCS_GUEST_SYSENTER_EIP:
5287	case VMX_VMCS_GUEST_SYSENTER_ESP:
5288	case VMX_VMCS_GUEST_GDTR_BASE:
5289	case VMX_VMCS_GUEST_IDTR_BASE:
5290	case VMX_VMCS_GUEST_CS_BASE:
5291	case VMX_VMCS_GUEST_DS_BASE:
5292	case VMX_VMCS_GUEST_ES_BASE:
5293	case VMX_VMCS_GUEST_FS_BASE:
5294	case VMX_VMCS_GUEST_GS_BASE:
5295	case VMX_VMCS_GUEST_SS_BASE:
5296	case VMX_VMCS_GUEST_LDTR_BASE:
5297	case VMX_VMCS_GUEST_TR_BASE:
5298	case VMX_VMCS_GUEST_CR3:
5299	return true;
5300	}
5301	return false;
5302	}
5303
5304	static bool hmR0VmxIsValidReadField(uint32_t idxField)
5305	{
5306	switch (idxField)
5307	{
5308	/* Read-only fields. */
5309	case VMX_VMCS_RO_EXIT_QUALIFICATION:
5310	return true;
5311	}
5312	/* Remaining readable fields should also be writable. */
5313	return hmR0VmxIsValidWriteField(idxField);
5314	}
5315	#endif /* VBOX_STRICT */
5316
5317
5318	/**
5319	* Executes the specified handler in 64-bit mode.
5320	*
5321	* @returns VBox status code (no informational status codes).
5322	* @param pVM The cross context VM structure.
5323	* @param pVCpu The cross context virtual CPU structure.
5324	* @param pCtx Pointer to the guest CPU context.
5325	* @param enmOp The operation to perform.
5326	* @param cParams Number of parameters.
5327	* @param paParam Array of 32-bit parameters.
5328	*/
5329	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp,
5330	uint32_t cParams, uint32_t *paParam)
5331	{
5332	NOREF(pCtx);
5333
5334	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
5335	Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
5336	Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
5337	Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
5338
5339	#ifdef VBOX_STRICT
5340	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
5341	Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
5342
5343	for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
5344	Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
5345	#endif
5346
5347	/* Disable interrupts. */
5348	RTCCUINTREG fOldEFlags = ASMIntDisableFlags();
5349
5350	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
5351	RTCPUID idHostCpu = RTMpCpuId();
5352	CPUMR0SetLApic(pVCpu, idHostCpu);
5353	#endif
5354
5355	PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
5356	RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5357
5358	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
5359	VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5360	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
5361
5362	/* Leave VMX Root Mode. */
5363	VMXDisable();
5364
5365	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5366
5367	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
5368	CPUMSetHyperEIP(pVCpu, enmOp);
5369	for (int i = (int)cParams - 1; i >= 0; i--)
5370	CPUMPushHyper(pVCpu, paParam[i]);
5371
5372	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
5373
5374	/* Call the switcher. */
5375	int rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
5376	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
5377
5378	/** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
5379	/* Make sure the VMX instructions don't cause #UD faults. */
5380	SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
5381
5382	/* Re-enter VMX Root Mode */
5383	int rc2 = VMXEnable(HCPhysCpuPage);
5384	if (RT_FAILURE(rc2))
5385	{
5386	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5387	ASMSetFlags(fOldEFlags);
5388	pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
5389	return rc2;
5390	}
5391
5392	rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5393	AssertRC(rc2);
5394	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
5395	Assert(!(ASMGetFlags() & X86_EFL_IF));
5396	ASMSetFlags(fOldEFlags);
5397	return rc;
5398	}
5399
5400
5401	/**
5402	* Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
5403	* supporting 64-bit guests.
5404	*
5405	* @returns VBox status code.
5406	* @param fResume Whether to VMLAUNCH or VMRESUME.
5407	* @param pCtx Pointer to the guest-CPU context.
5408	* @param pCache Pointer to the VMCS cache.
5409	* @param pVM The cross context VM structure.
5410	* @param pVCpu The cross context virtual CPU structure.
5411	*/
5412	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
5413	{
5414	NOREF(fResume);
5415
5416	PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
5417	RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5418
5419	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5420	pCache->uPos = 1;
5421	pCache->interPD = PGMGetInterPaeCR3(pVM);
5422	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
5423	#endif
5424
5425	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5426	pCache->TestIn.HCPhysCpuPage = 0;
5427	pCache->TestIn.HCPhysVmcs = 0;
5428	pCache->TestIn.pCache = 0;
5429	pCache->TestOut.HCPhysVmcs = 0;
5430	pCache->TestOut.pCache = 0;
5431	pCache->TestOut.pCtx = 0;
5432	pCache->TestOut.eflags = 0;
5433	#else
5434	NOREF(pCache);
5435	#endif
5436
5437	uint32_t aParam[10];
5438	aParam[0] = RT_LO_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5439	aParam[1] = RT_HI_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Hi. */
5440	aParam[2] = RT_LO_U32(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
5441	aParam[3] = RT_HI_U32(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Hi. */
5442	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
5443	aParam[5] = 0;
5444	aParam[6] = VM_RC_ADDR(pVM, pVM);
5445	aParam[7] = 0;
5446	aParam[8] = VM_RC_ADDR(pVM, pVCpu);
5447	aParam[9] = 0;
5448
5449	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5450	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
5451	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
5452	#endif
5453	int rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, RT_ELEMENTS(aParam), &aParam[0]);
5454
5455	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5456	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
5457	Assert(pCtx->dr[4] == 10);
5458	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
5459	#endif
5460
5461	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5462	AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5463	AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5464	pVCpu->hm.s.vmx.HCPhysVmcs));
5465	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5466	pCache->TestOut.HCPhysVmcs));
5467	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5468	pCache->TestOut.pCache));
5469	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5470	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5471	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5472	pCache->TestOut.pCtx));
5473	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5474	#endif
5475	return rc;
5476	}
5477
5478
5479	/**
5480	* Initialize the VMCS-Read cache.
5481	*
5482	* The VMCS cache is used for 32-bit hosts running 64-bit guests (except 32-bit
5483	* Darwin which runs with 64-bit paging in 32-bit mode) for 64-bit fields that
5484	* cannot be accessed in 32-bit mode. Some 64-bit fields -can- be accessed
5485	* (those that have a 32-bit FULL & HIGH part).
5486	*
5487	* @returns VBox status code.
5488	* @param pVM The cross context VM structure.
5489	* @param pVCpu The cross context virtual CPU structure.
5490	*/
5491	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
5492	{
5493	#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5494	{ \
5495	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5496	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5497	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5498	++cReadFields; \
5499	}
5500
5501	AssertPtr(pVM);
5502	AssertPtr(pVCpu);
5503	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5504	uint32_t cReadFields = 0;
5505
5506	/*
5507	* Don't remove the #if 0'd fields in this code. They're listed here for consistency
5508	* and serve to indicate exceptions to the rules.
5509	*/
5510
5511	/* Guest-natural selector base fields. */
5512	#if 0
5513	/* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5514	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5515	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5516	#endif
5517	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5518	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5519	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5520	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5521	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5522	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5523	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5524	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5525	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5526	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5527	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5528	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5529	#if 0
5530	/* Unused natural width guest-state fields. */
5531	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5532	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5533	#endif
5534	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5535	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5536
5537	/* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
5538	#if 0
5539	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5540	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5541	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5542	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5543	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5544	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5545	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5546	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5547	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5548	#endif
5549
5550	/* Natural width guest-state fields. */
5551	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5552	#if 0
5553	/* Currently unused field. */
5554	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
5555	#endif
5556
5557	if (pVM->hm.s.fNestedPaging)
5558	{
5559	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5560	AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5561	VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5562	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5563	}
5564	else
5565	{
5566	AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5567	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5568	}
5569
5570	#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5571	return VINF_SUCCESS;
5572	}
5573
5574
5575	/**
5576	* Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5577	* queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5578	* darwin, running 64-bit guests).
5579	*
5580	* @returns VBox status code.
5581	* @param pVCpu The cross context virtual CPU structure.
5582	* @param idxField The VMCS field encoding.
5583	* @param u64Val 16, 32 or 64-bit value.
5584	*/
5585	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5586	{
5587	int rc;
5588	switch (idxField)
5589	{
5590	/*
5591	* These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5592	*/
5593	/* 64-bit Control fields. */
5594	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5595	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5596	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5597	case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5598	case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5599	case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5600	case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5601	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5602	case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
5603	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5604	case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5605	case VMX_VMCS64_CTRL_EPTP_FULL:
5606	case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5607	/* 64-bit Guest-state fields. */
5608	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5609	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5610	case VMX_VMCS64_GUEST_PAT_FULL:
5611	case VMX_VMCS64_GUEST_EFER_FULL:
5612	case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5613	case VMX_VMCS64_GUEST_PDPTE0_FULL:
5614	case VMX_VMCS64_GUEST_PDPTE1_FULL:
5615	case VMX_VMCS64_GUEST_PDPTE2_FULL:
5616	case VMX_VMCS64_GUEST_PDPTE3_FULL:
5617	/* 64-bit Host-state fields. */
5618	case VMX_VMCS64_HOST_PAT_FULL:
5619	case VMX_VMCS64_HOST_EFER_FULL:
5620	case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5621	{
5622	rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
5623	rc \|= VMXWriteVmcs32(idxField + 1, RT_HI_U32(u64Val));
5624	break;
5625	}
5626
5627	/*
5628	* These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5629	* values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5630	*/
5631	/* Natural-width Guest-state fields. */
5632	case VMX_VMCS_GUEST_CR3:
5633	case VMX_VMCS_GUEST_ES_BASE:
5634	case VMX_VMCS_GUEST_CS_BASE:
5635	case VMX_VMCS_GUEST_SS_BASE:
5636	case VMX_VMCS_GUEST_DS_BASE:
5637	case VMX_VMCS_GUEST_FS_BASE:
5638	case VMX_VMCS_GUEST_GS_BASE:
5639	case VMX_VMCS_GUEST_LDTR_BASE:
5640	case VMX_VMCS_GUEST_TR_BASE:
5641	case VMX_VMCS_GUEST_GDTR_BASE:
5642	case VMX_VMCS_GUEST_IDTR_BASE:
5643	case VMX_VMCS_GUEST_RSP:
5644	case VMX_VMCS_GUEST_RIP:
5645	case VMX_VMCS_GUEST_SYSENTER_ESP:
5646	case VMX_VMCS_GUEST_SYSENTER_EIP:
5647	{
5648	if (!(RT_HI_U32(u64Val)))
5649	{
5650	/* If this field is 64-bit, VT-x will zero out the top bits. */
5651	rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
5652	}
5653	else
5654	{
5655	/* Assert that only the 32->64 switcher case should ever come here. */
5656	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5657	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5658	}
5659	break;
5660	}
5661
5662	default:
5663	{
5664	AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5665	rc = VERR_INVALID_PARAMETER;
5666	break;
5667	}
5668	}
5669	AssertRCReturn(rc, rc);
5670	return rc;
5671	}
5672
5673
5674	/**
5675	* Queue up a VMWRITE by using the VMCS write cache.
5676	* This is only used on 32-bit hosts (except darwin) for 64-bit guests.
5677	*
5678	* @param pVCpu The cross context virtual CPU structure.
5679	* @param idxField The VMCS field encoding.
5680	* @param u64Val 16, 32 or 64-bit value.
5681	*/
5682	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5683	{
5684	AssertPtr(pVCpu);
5685	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5686
5687	AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5688	("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5689
5690	/* Make sure there are no duplicates. */
5691	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5692	{
5693	if (pCache->Write.aField[i] == idxField)
5694	{
5695	pCache->Write.aFieldVal[i] = u64Val;
5696	return VINF_SUCCESS;
5697	}
5698	}
5699
5700	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5701	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5702	pCache->Write.cValidEntries++;
5703	return VINF_SUCCESS;
5704	}
5705	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */
5706
5707
5708	/**
5709	* Sets up the usage of TSC-offsetting and updates the VMCS.
5710	*
5711	* If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
5712	* VMX preemption timer.
5713	*
5714	* @returns VBox status code.
5715	* @param pVM The cross context VM structure.
5716	* @param pVCpu The cross context virtual CPU structure.
5717	*
5718	* @remarks No-long-jump zone!!!
5719	*/
5720	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVM pVM, PVMCPU pVCpu)
5721	{
5722	int rc;
5723	bool fOffsettedTsc;
5724	bool fParavirtTsc;
5725	if (pVM->hm.s.vmx.fUsePreemptTimer)
5726	{
5727	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset,
5728	&fOffsettedTsc, &fParavirtTsc);
5729
5730	/* Make sure the returned values have sane upper and lower boundaries. */
5731	uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
5732	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5733	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5734	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5735
5736	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5737	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
5738	}
5739	else
5740	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset, &fParavirtTsc);
5741
5742	/** @todo later optimize this to be done elsewhere and not before every
5743	* VM-entry. */
5744	if (fParavirtTsc)
5745	{
5746	/* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
5747	information before every VM-entry, hence disable it for performance sake. */
5748	#if 0
5749	rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
5750	AssertRC(rc);
5751	#endif
5752	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
5753	}
5754
5755	if (fOffsettedTsc && RT_LIKELY(!pVCpu->hm.s.fDebugWantRdTscExit))
5756	{
5757	/* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
5758	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
5759
5760	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5761	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5762	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5763	}
5764	else
5765	{
5766	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5767	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5768	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5769	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5770	}
5771	}
5772
5773
5774	#ifdef HMVMX_USE_IEM_EVENT_REFLECTION
5775	/**
5776	* Gets the IEM exception flags for the specified vector and IDT vectoring /
5777	* VM-exit interruption info type.
5778	*
5779	* @returns The IEM exception flags.
5780	* @param uVector The event vector.
5781	* @param uVmxVectorType The VMX event type.
5782	*
5783	* @remarks This function currently only constructs flags required for
5784	* IEMEvaluateRecursiveXcpt and not the complete flags (e.g, error-code
5785	* and CR2 aspects of an exception are not included).
5786	*/
5787	static uint32_t hmR0VmxGetIemXcptFlags(uint8_t uVector, uint32_t uVmxVectorType)
5788	{
5789	uint32_t fIemXcptFlags;
5790	switch (uVmxVectorType)
5791	{
5792	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
5793	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
5794	fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
5795	break;
5796
5797	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
5798	fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
5799	break;
5800
5801	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
5802	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT \| IEM_XCPT_FLAGS_ICEBP_INSTR;
5803	break;
5804
5805	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT:
5806	{
5807	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
5808	if (uVector == X86_XCPT_BP)
5809	fIemXcptFlags \|= IEM_XCPT_FLAGS_BP_INSTR;
5810	else if (uVector == X86_XCPT_OF)
5811	fIemXcptFlags \|= IEM_XCPT_FLAGS_OF_INSTR;
5812	else
5813	{
5814	fIemXcptFlags = 0;
5815	AssertMsgFailed(("Unexpected vector for software int. uVector=%#x", uVector));
5816	}
5817	break;
5818	}
5819
5820	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
5821	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
5822	break;
5823
5824	default:
5825	fIemXcptFlags = 0;
5826	AssertMsgFailed(("Unexpected vector type! uVmxVectorType=%#x uVector=%#x", uVmxVectorType, uVector));
5827	break;
5828	}
5829	return fIemXcptFlags;
5830	}
5831
5832	#else
5833	/**
5834	* Determines if an exception is a contributory exception.
5835	*
5836	* Contributory exceptions are ones which can cause double-faults unless the
5837	* original exception was a benign exception. Page-fault is intentionally not
5838	* included here as it's a conditional contributory exception.
5839	*
5840	* @returns true if the exception is contributory, false otherwise.
5841	* @param uVector The exception vector.
5842	*/
5843	DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
5844	{
5845	switch (uVector)
5846	{
5847	case X86_XCPT_GP:
5848	case X86_XCPT_SS:
5849	case X86_XCPT_NP:
5850	case X86_XCPT_TS:
5851	case X86_XCPT_DE:
5852	return true;
5853	default:
5854	break;
5855	}
5856	return false;
5857	}
5858	#endif /* HMVMX_USE_IEM_EVENT_REFLECTION */
5859
5860
5861	/**
5862	* Sets an event as a pending event to be injected into the guest.
5863	*
5864	* @param pVCpu The cross context virtual CPU structure.
5865	* @param u32IntInfo The VM-entry interruption-information field.
5866	* @param cbInstr The VM-entry instruction length in bytes (for software
5867	* interrupts, exceptions and privileged software
5868	* exceptions).
5869	* @param u32ErrCode The VM-entry exception error code.
5870	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5871	* page-fault.
5872	*
5873	* @remarks Statistics counter assumes this is a guest event being injected or
5874	* re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5875	* always incremented.
5876	*/
5877	DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5878	RTGCUINTPTR GCPtrFaultAddress)
5879	{
5880	Assert(!pVCpu->hm.s.Event.fPending);
5881	pVCpu->hm.s.Event.fPending = true;
5882	pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5883	pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5884	pVCpu->hm.s.Event.cbInstr = cbInstr;
5885	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5886	}
5887
5888
5889	/**
5890	* Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
5891	*
5892	* @param pVCpu The cross context virtual CPU structure.
5893	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5894	* out-of-sync. Make sure to update the required fields
5895	* before using them.
5896	*/
5897	DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5898	{
5899	NOREF(pMixedCtx);
5900	uint32_t u32IntInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
5901	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5902	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5903	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
5904	}
5905
5906
5907	/**
5908	* Handle a condition that occurred while delivering an event through the guest
5909	* IDT.
5910	*
5911	* @returns Strict VBox status code (i.e. informational status codes too).
5912	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
5913	* @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
5914	* to continue execution of the guest which will delivery the \#DF.
5915	* @retval VINF_EM_RESET if we detected a triple-fault condition.
5916	* @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
5917	*
5918	* @param pVCpu The cross context virtual CPU structure.
5919	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5920	* out-of-sync. Make sure to update the required fields
5921	* before using them.
5922	* @param pVmxTransient Pointer to the VMX transient structure.
5923	*
5924	* @remarks No-long-jump zone!!!
5925	*/
5926	static VBOXSTRICTRC hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
5927	{
5928	uint32_t const uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5929
5930	int rc2 = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient); AssertRCReturn(rc2, rc2);
5931	rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient); AssertRCReturn(rc2, rc2);
5932
5933	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
5934	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5935	{
5936	uint32_t const uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5937	uint32_t const uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5938	#ifdef HMVMX_USE_IEM_EVENT_REFLECTION
5939	/*
5940	* If the event was a software interrupt (generated with INT n) or a software exception (generated
5941	* by INT3/INTO) or a privileged software exception (generated by INT1), we can handle the VM-exit
5942	* and continue guest execution which will re-execute the instruction rather than re-injecting the
5943	* exception, as that can cause premature trips to ring-3 before injection and involve TRPM which
5944	* currently has no way of storing that these exceptions were caused by these instructions
5945	* (ICEBP's #DB poses the problem).
5946	*/
5947	IEMXCPTRAISE enmRaise;
5948	IEMXCPTRAISEINFO fRaiseInfo;
5949	if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5950	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5951	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
5952	{
5953	enmRaise = IEMXCPTRAISE_REEXEC_INSTR;
5954	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
5955	}
5956	else if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5957	{
5958	uint32_t const uExitVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uExitIntInfo);
5959	uint32_t const fIdtVectorFlags = hmR0VmxGetIemXcptFlags(uIdtVector, uIdtVectorType);
5960	uint32_t const fExitVectorFlags = hmR0VmxGetIemXcptFlags(uExitVector, uExitVectorType);
5961	/** @todo Make AssertMsgReturn as just AssertMsg later. */
5962	AssertMsgReturn(uExitVectorType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT,
5963	("hmR0VmxCheckExitDueToEventDelivery: Unexpected VM-exit interruption info. %#x!\n",
5964	uExitVectorType), VERR_VMX_IPE_5);
5965	enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector, &fRaiseInfo);
5966
5967	/* Determine a vectoring #PF condition, see comment in hmR0VmxExitXcptPF(). */
5968	if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF \| IEMXCPTRAISEINFO_NMI_PF))
5969	{
5970	pVmxTransient->fVectoringPF = true;
5971	enmRaise = IEMXCPTRAISE_PREV_EVENT;
5972	}
5973	}
5974	else
5975	{
5976	/*
5977	* If an exception or hardware interrupt delivery caused an EPT violation/misconfig or APIC access
5978	* VM-exit, then the VM-exit interruption-information will not be valid and we end up here.
5979	* It is sufficient to reflect the original event to the guest after handling the VM-exit.
5980	*/
5981	Assert( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5982	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5983	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT);
5984	enmRaise = IEMXCPTRAISE_PREV_EVENT;
5985	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
5986	}
5987
5988	/*
5989	* On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig
5990	* etc.) occurred while delivering the NMI, we need to clear the block-by-NMI field in the guest
5991	* interruptibility-state before re-delivering the NMI after handling the VM-exit. Otherwise the
5992	* subsequent VM-entry would fail.
5993	*
5994	* See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
5995	*/
5996	if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
5997	&& uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5998	&& ( enmRaise == IEMXCPTRAISE_PREV_EVENT
5999	\|\| (fRaiseInfo & IEMXCPTRAISEINFO_NMI_PF))
6000	&& (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
6001	{
6002	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6003	}
6004
6005	switch (enmRaise)
6006	{
6007	case IEMXCPTRAISE_CURRENT_XCPT:
6008	{
6009	Log4(("IDT: vcpu[%RU32] Pending secondary xcpt: uIdtVectoringInfo=%#RX64 uExitIntInfo=%#RX64\n", pVCpu->idCpu,
6010	pVmxTransient->uIdtVectoringInfo, pVmxTransient->uExitIntInfo));
6011	Assert(rcStrict == VINF_SUCCESS);
6012	break;
6013	}
6014
6015	case IEMXCPTRAISE_PREV_EVENT:
6016	{
6017	uint32_t u32ErrCode;
6018	if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
6019	{
6020	rc2 = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
6021	AssertRCReturn(rc2, rc2);
6022	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
6023	}
6024	else
6025	u32ErrCode = 0;
6026
6027	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF, see hmR0VmxExitXcptPF(). */
6028	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
6029	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
6030	0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
6031
6032	Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
6033	pVCpu->hm.s.Event.u32ErrCode));
6034	Assert(rcStrict == VINF_SUCCESS);
6035	break;
6036	}
6037
6038	case IEMXCPTRAISE_REEXEC_INSTR:
6039	Assert(rcStrict == VINF_SUCCESS);
6040	break;
6041
6042	case IEMXCPTRAISE_DOUBLE_FAULT:
6043	{
6044	/*
6045	* Determing a vectoring double #PF condition. Used later, when PGM evaluates the
6046	* second #PF as a guest #PF (and not a shadow #PF) and needs to be converted into a #DF.
6047	*/
6048	if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
6049	{
6050	pVmxTransient->fVectoringDoublePF = true;
6051	Log4(("IDT: vcpu[%RU32] Vectoring double #PF %#RX64 cr2=%#RX64\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
6052	pMixedCtx->cr2));
6053	rcStrict = VINF_SUCCESS;
6054	}
6055	else
6056	{
6057	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
6058	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
6059	Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
6060	pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
6061	rcStrict = VINF_HM_DOUBLE_FAULT;
6062	}
6063	break;
6064	}
6065
6066	case IEMXCPTRAISE_TRIPLE_FAULT:
6067	{
6068	Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
6069	uExitVector));
6070	rcStrict = VINF_EM_RESET;
6071	break;
6072	}
6073
6074	case IEMXCPTRAISE_CPU_HANG:
6075	{
6076	Log4(("IDT: vcpu[%RU32] Bad guest! Entering CPU hang. fRaiseInfo=%#x\n", pVCpu->idCpu, fRaiseInfo));
6077	rcStrict = VERR_EM_GUEST_CPU_HANG;
6078	break;
6079	}
6080
6081	default:
6082	{
6083	AssertMsgFailed(("IDT: vcpu[%RU32] Unexpected/invalid value! enmRaise=%#x\n", pVCpu->idCpu, enmRaise));
6084	rcStrict = VERR_VMX_IPE_2;
6085	break;
6086	}
6087	}
6088	#else
6089	typedef enum
6090	{
6091	VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
6092	VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
6093	VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
6094	VMXREFLECTXCPT_HANG, /* Indicate bad VM trying to deadlock the CPU. */
6095	VMXREFLECTXCPT_NONE /* Nothing to reflect. */
6096	} VMXREFLECTXCPT;
6097
6098	/* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
6099	VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
6100	if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
6101	{
6102	if (uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
6103	{
6104	enmReflect = VMXREFLECTXCPT_XCPT;
6105	#ifdef VBOX_STRICT
6106	if ( hmR0VmxIsContributoryXcpt(uIdtVector)
6107	&& uExitVector == X86_XCPT_PF)
6108	{
6109	Log4(("IDT: vcpu[%RU32] Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
6110	}
6111	#endif
6112	if ( uExitVector == X86_XCPT_PF
6113	&& uIdtVector == X86_XCPT_PF)
6114	{
6115	pVmxTransient->fVectoringDoublePF = true;
6116	Log4(("IDT: vcpu[%RU32] Vectoring Double #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
6117	}
6118	else if ( uExitVector == X86_XCPT_AC
6119	&& uIdtVector == X86_XCPT_AC)
6120	{
6121	enmReflect = VMXREFLECTXCPT_HANG;
6122	Log4(("IDT: Nested #AC - Bad guest\n"));
6123	}
6124	else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
6125	&& hmR0VmxIsContributoryXcpt(uExitVector)
6126	&& ( hmR0VmxIsContributoryXcpt(uIdtVector)
6127	\|\| uIdtVector == X86_XCPT_PF))
6128	{
6129	enmReflect = VMXREFLECTXCPT_DF;
6130	}
6131	else if (uIdtVector == X86_XCPT_DF)
6132	enmReflect = VMXREFLECTXCPT_TF;
6133	}
6134	else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
6135	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
6136	{
6137	/*
6138	* Ignore software interrupts (INT n), software exceptions (#BP, #OF) and
6139	* privileged software exception (#DB from ICEBP) as they reoccur when restarting the instruction.
6140	*/
6141	enmReflect = VMXREFLECTXCPT_XCPT;
6142
6143	if (uExitVector == X86_XCPT_PF)
6144	{
6145	pVmxTransient->fVectoringPF = true;
6146	Log4(("IDT: vcpu[%RU32] Vectoring #PF due to Ext-Int/NMI. uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
6147	}
6148	}
6149	}
6150	else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6151	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
6152	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
6153	{
6154	/*
6155	* If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
6156	* interruption-information will not be valid as it's not an exception and we end up here. In such cases,
6157	* it is sufficient to reflect the original exception to the guest after handling the VM-exit.
6158	*/
6159	enmReflect = VMXREFLECTXCPT_XCPT;
6160	}
6161
6162	/*
6163	* On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig etc.) occurred
6164	* while delivering the NMI, we need to clear the block-by-NMI field in the guest interruptibility-state before
6165	* re-delivering the NMI after handling the VM-exit. Otherwise the subsequent VM-entry would fail.
6166	*
6167	* See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
6168	*/
6169	if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
6170	&& enmReflect == VMXREFLECTXCPT_XCPT
6171	&& (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
6172	&& VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6173	{
6174	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6175	}
6176
6177	switch (enmReflect)
6178	{
6179	case VMXREFLECTXCPT_XCPT:
6180	{
6181	Assert( uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6182	&& uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6183	&& uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
6184
6185	uint32_t u32ErrCode = 0;
6186	if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
6187	{
6188	rc2 = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
6189	AssertRCReturn(rc2, rc2);
6190	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
6191	}
6192
6193	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
6194	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
6195	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
6196	0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
6197	rcStrict = VINF_SUCCESS;
6198	Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu,
6199	pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.u32ErrCode));
6200
6201	break;
6202	}
6203
6204	case VMXREFLECTXCPT_DF:
6205	{
6206	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
6207	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
6208	rcStrict = VINF_HM_DOUBLE_FAULT;
6209	Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
6210	pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
6211
6212	break;
6213	}
6214
6215	case VMXREFLECTXCPT_TF:
6216	{
6217	rcStrict = VINF_EM_RESET;
6218	Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
6219	uExitVector));
6220	break;
6221	}
6222
6223	case VMXREFLECTXCPT_HANG:
6224	{
6225	rcStrict = VERR_EM_GUEST_CPU_HANG;
6226	break;
6227	}
6228
6229	default:
6230	Assert(rcStrict == VINF_SUCCESS);
6231	break;
6232	}
6233	#endif /* HMVMX_USE_IEM_EVENT_REFLECTION */
6234	}
6235	else if ( VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo)
6236	&& VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
6237	&& uExitVector != X86_XCPT_DF
6238	&& (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
6239	{
6240	/*
6241	* Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
6242	* We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
6243	* See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
6244	*/
6245	if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6246	{
6247	Log4(("hmR0VmxCheckExitDueToEventDelivery: vcpu[%RU32] Setting VMCPU_FF_BLOCK_NMIS. Valid=%RTbool uExitReason=%u\n",
6248	pVCpu->idCpu, VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo), pVmxTransient->uExitReason));
6249	VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6250	}
6251	}
6252
6253	Assert( rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_HM_DOUBLE_FAULT
6254	\|\| rcStrict == VINF_EM_RESET \|\| rcStrict == VERR_EM_GUEST_CPU_HANG);
6255	return rcStrict;
6256	}
6257
6258
6259	/**
6260	* Saves the guest's CR0 register from the VMCS into the guest-CPU context.
6261	*
6262	* @returns VBox status code.
6263	* @param pVCpu The cross context virtual CPU structure.
6264	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6265	* out-of-sync. Make sure to update the required fields
6266	* before using them.
6267	*
6268	* @remarks No-long-jump zone!!!
6269	*/
6270	static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6271	{
6272	NOREF(pMixedCtx);
6273
6274	/*
6275	* While in the middle of saving guest-CR0, we could get preempted and re-invoked from the preemption hook,
6276	* see hmR0VmxLeave(). Safer to just make this code non-preemptible.
6277	*/
6278	VMMRZCallRing3Disable(pVCpu);
6279	HM_DISABLE_PREEMPT();
6280
6281	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0))
6282	{
6283	#ifndef DEBUG_bird /** @todo this triggers running bs3-cpu-generated-1.img with --debug-command-line
6284	* and 'dbgc-init' containing:
6285	* sxe "xcpt_de"
6286	* sxe "xcpt_bp"
6287	* sxi "xcpt_gp"
6288	* sxi "xcpt_ss"
6289	* sxi "xcpt_np"
6290	*/
6291	/** @todo r=ramshankar: Should be fixed after r119291. */
6292	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0));
6293	#endif
6294	uint32_t uVal = 0;
6295	uint32_t uShadow = 0;
6296	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
6297	rc \|= VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
6298	AssertRCReturn(rc, rc);
6299
6300	uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) \| (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
6301	CPUMSetGuestCR0(pVCpu, uVal);
6302	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0);
6303	}
6304
6305	HM_RESTORE_PREEMPT();
6306	VMMRZCallRing3Enable(pVCpu);
6307	return VINF_SUCCESS;
6308	}
6309
6310
6311	/**
6312	* Saves the guest's CR4 register from the VMCS into the guest-CPU context.
6313	*
6314	* @returns VBox status code.
6315	* @param pVCpu The cross context virtual CPU structure.
6316	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6317	* out-of-sync. Make sure to update the required fields
6318	* before using them.
6319	*
6320	* @remarks No-long-jump zone!!!
6321	*/
6322	static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6323	{
6324	NOREF(pMixedCtx);
6325
6326	int rc = VINF_SUCCESS;
6327	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4))
6328	{
6329	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4));
6330	uint32_t uVal = 0;
6331	uint32_t uShadow = 0;
6332	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
6333	rc \|= VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
6334	AssertRCReturn(rc, rc);
6335
6336	uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) \| (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
6337	CPUMSetGuestCR4(pVCpu, uVal);
6338	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4);
6339	}
6340	return rc;
6341	}
6342
6343
6344	/**
6345	* Saves the guest's RIP register from the VMCS into the guest-CPU context.
6346	*
6347	* @returns VBox status code.
6348	* @param pVCpu The cross context virtual CPU structure.
6349	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6350	* out-of-sync. Make sure to update the required fields
6351	* before using them.
6352	*
6353	* @remarks No-long-jump zone!!!
6354	*/
6355	static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6356	{
6357	int rc = VINF_SUCCESS;
6358	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP))
6359	{
6360	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP));
6361	uint64_t u64Val = 0;
6362	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
6363	AssertRCReturn(rc, rc);
6364
6365	pMixedCtx->rip = u64Val;
6366	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP);
6367	}
6368	return rc;
6369	}
6370
6371
6372	/**
6373	* Saves the guest's RSP register from the VMCS into the guest-CPU context.
6374	*
6375	* @returns VBox status code.
6376	* @param pVCpu The cross context virtual CPU structure.
6377	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6378	* out-of-sync. Make sure to update the required fields
6379	* before using them.
6380	*
6381	* @remarks No-long-jump zone!!!
6382	*/
6383	static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6384	{
6385	int rc = VINF_SUCCESS;
6386	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP))
6387	{
6388	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RSP));
6389	uint64_t u64Val = 0;
6390	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
6391	AssertRCReturn(rc, rc);
6392
6393	pMixedCtx->rsp = u64Val;
6394	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP);
6395	}
6396	return rc;
6397	}
6398
6399
6400	/**
6401	* Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
6402	*
6403	* @returns VBox status code.
6404	* @param pVCpu The cross context virtual CPU structure.
6405	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6406	* out-of-sync. Make sure to update the required fields
6407	* before using them.
6408	*
6409	* @remarks No-long-jump zone!!!
6410	*/
6411	static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6412	{
6413	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS))
6414	{
6415	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS));
6416	uint32_t uVal = 0;
6417	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
6418	AssertRCReturn(rc, rc);
6419
6420	pMixedCtx->eflags.u32 = uVal;
6421	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
6422	{
6423	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
6424	Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
6425
6426	pMixedCtx->eflags.Bits.u1VM = 0;
6427	pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
6428	}
6429
6430	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS);
6431	}
6432	return VINF_SUCCESS;
6433	}
6434
6435
6436	/**
6437	* Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
6438	* guest-CPU context.
6439	*/
6440	DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6441	{
6442	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6443	rc \|= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
6444	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6445	return rc;
6446	}
6447
6448
6449	/**
6450	* Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
6451	* from the guest-state area in the VMCS.
6452	*
6453	* @param pVCpu The cross context virtual CPU structure.
6454	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6455	* out-of-sync. Make sure to update the required fields
6456	* before using them.
6457	*
6458	* @remarks No-long-jump zone!!!
6459	*/
6460	static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6461	{
6462	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE))
6463	{
6464	uint32_t uIntrState = 0;
6465	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
6466	AssertRC(rc);
6467
6468	if (!uIntrState)
6469	{
6470	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6471	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6472
6473	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6474	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6475	}
6476	else
6477	{
6478	if (uIntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
6479	\| VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI))
6480	{
6481	rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6482	AssertRC(rc);
6483	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
6484	AssertRC(rc);
6485
6486	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
6487	Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
6488	}
6489	else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6490	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6491
6492	if (uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI)
6493	{
6494	if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6495	VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6496	}
6497	else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6498	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6499	}
6500
6501	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE);
6502	}
6503	}
6504
6505
6506	/**
6507	* Saves the guest's activity state.
6508	*
6509	* @returns VBox status code.
6510	* @param pVCpu The cross context virtual CPU structure.
6511	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6512	* out-of-sync. Make sure to update the required fields
6513	* before using them.
6514	*
6515	* @remarks No-long-jump zone!!!
6516	*/
6517	static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6518	{
6519	NOREF(pMixedCtx);
6520	/* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
6521	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_ACTIVITY_STATE);
6522	return VINF_SUCCESS;
6523	}
6524
6525
6526	/**
6527	* Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
6528	* the current VMCS into the guest-CPU context.
6529	*
6530	* @returns VBox status code.
6531	* @param pVCpu The cross context virtual CPU structure.
6532	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6533	* out-of-sync. Make sure to update the required fields
6534	* before using them.
6535	*
6536	* @remarks No-long-jump zone!!!
6537	*/
6538	static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6539	{
6540	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
6541	{
6542	Assert(!HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR));
6543	uint32_t u32Val = 0;
6544	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
6545	pMixedCtx->SysEnter.cs = u32Val;
6546	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR);
6547	}
6548
6549	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
6550	{
6551	Assert(!HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR));
6552	uint64_t u64Val = 0;
6553	int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
6554	pMixedCtx->SysEnter.eip = u64Val;
6555	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR);
6556	}
6557	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
6558	{
6559	Assert(!HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR));
6560	uint64_t u64Val = 0;
6561	int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
6562	pMixedCtx->SysEnter.esp = u64Val;
6563	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR);
6564	}
6565	return VINF_SUCCESS;
6566	}
6567
6568
6569	/**
6570	* Saves the set of guest MSRs (that we restore lazily while leaving VT-x) from
6571	* the CPU back into the guest-CPU context.
6572	*
6573	* @returns VBox status code.
6574	* @param pVCpu The cross context virtual CPU structure.
6575	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6576	* out-of-sync. Make sure to update the required fields
6577	* before using them.
6578	*
6579	* @remarks No-long-jump zone!!!
6580	*/
6581	static int hmR0VmxSaveGuestLazyMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6582	{
6583	/* Since this can be called from our preemption hook it's safer to make the guest-MSRs update non-preemptible. */
6584	VMMRZCallRing3Disable(pVCpu);
6585	HM_DISABLE_PREEMPT();
6586
6587	/* Doing the check here ensures we don't overwrite already-saved guest MSRs from a preemption hook. */
6588	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS))
6589	{
6590	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS));
6591	hmR0VmxLazySaveGuestMsrs(pVCpu, pMixedCtx);
6592	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6593	}
6594
6595	HM_RESTORE_PREEMPT();
6596	VMMRZCallRing3Enable(pVCpu);
6597
6598	return VINF_SUCCESS;
6599	}
6600
6601
6602	/**
6603	* Saves the auto load/store'd guest MSRs from the current VMCS into
6604	* the guest-CPU context.
6605	*
6606	* @returns VBox status code.
6607	* @param pVCpu The cross context virtual CPU structure.
6608	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6609	* out-of-sync. Make sure to update the required fields
6610	* before using them.
6611	*
6612	* @remarks No-long-jump zone!!!
6613	*/
6614	static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6615	{
6616	if (HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS))
6617	return VINF_SUCCESS;
6618
6619	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS));
6620	PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
6621	uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
6622	Log4(("hmR0VmxSaveGuestAutoLoadStoreMsrs: cMsrs=%u\n", cMsrs));
6623	for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
6624	{
6625	switch (pMsr->u32Msr)
6626	{
6627	case MSR_K8_TSC_AUX: CPUMR0SetGuestTscAux(pVCpu, pMsr->u64Value); break;
6628	case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
6629	case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
6630	case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
6631	case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
6632	case MSR_K6_EFER: /* Nothing to do here since we intercept writes, see hmR0VmxLoadGuestMsrs(). */
6633	break;
6634
6635	default:
6636	{
6637	AssertMsgFailed(("Unexpected MSR in auto-load/store area. uMsr=%#RX32 cMsrs=%u\n", pMsr->u32Msr, cMsrs));
6638	pVCpu->hm.s.u32HMError = pMsr->u32Msr;
6639	return VERR_HM_UNEXPECTED_LD_ST_MSR;
6640	}
6641	}
6642	}
6643
6644	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS);
6645	return VINF_SUCCESS;
6646	}
6647
6648
6649	/**
6650	* Saves the guest control registers from the current VMCS into the guest-CPU
6651	* context.
6652	*
6653	* @returns VBox status code.
6654	* @param pVCpu The cross context virtual CPU structure.
6655	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6656	* out-of-sync. Make sure to update the required fields
6657	* before using them.
6658	*
6659	* @remarks No-long-jump zone!!!
6660	*/
6661	static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6662	{
6663	/* Guest CR0. Guest FPU. */
6664	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6665	AssertRCReturn(rc, rc);
6666
6667	/* Guest CR4. */
6668	rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
6669	AssertRCReturn(rc, rc);
6670
6671	/* Guest CR2 - updated always during the world-switch or in #PF. */
6672	/* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
6673	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3))
6674	{
6675	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3));
6676	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
6677	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4));
6678
6679	PVM pVM = pVCpu->CTX_SUFF(pVM);
6680	if ( pVM->hm.s.vmx.fUnrestrictedGuest
6681	\|\| ( pVM->hm.s.fNestedPaging
6682	&& CPUMIsGuestPagingEnabledEx(pMixedCtx)))
6683	{
6684	uint64_t u64Val = 0;
6685	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
6686	if (pMixedCtx->cr3 != u64Val)
6687	{
6688	CPUMSetGuestCR3(pVCpu, u64Val);
6689	if (VMMRZCallRing3IsEnabled(pVCpu))
6690	{
6691	PGMUpdateCR3(pVCpu, u64Val);
6692	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6693	}
6694	else
6695	{
6696	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
6697	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
6698	}
6699	}
6700
6701	/* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
6702	if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
6703	{
6704	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
6705	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
6706	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
6707	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
6708	AssertRCReturn(rc, rc);
6709
6710	if (VMMRZCallRing3IsEnabled(pVCpu))
6711	{
6712	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6713	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6714	}
6715	else
6716	{
6717	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
6718	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
6719	}
6720	}
6721	}
6722
6723	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3);
6724	}
6725
6726	/*
6727	* Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
6728	* -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
6729	* -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
6730	*
6731	* The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
6732	* if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
6733	* calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
6734	* -NOT- check if HMVMX_UPDATED_GUEST_CR3 is already set or not!
6735	*
6736	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6737	*/
6738	if (VMMRZCallRing3IsEnabled(pVCpu))
6739	{
6740	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6741	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6742
6743	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6744	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6745
6746	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6747	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6748	}
6749
6750	return rc;
6751	}
6752
6753
6754	/**
6755	* Reads a guest segment register from the current VMCS into the guest-CPU
6756	* context.
6757	*
6758	* @returns VBox status code.
6759	* @param pVCpu The cross context virtual CPU structure.
6760	* @param idxSel Index of the selector in the VMCS.
6761	* @param idxLimit Index of the segment limit in the VMCS.
6762	* @param idxBase Index of the segment base in the VMCS.
6763	* @param idxAccess Index of the access rights of the segment in the VMCS.
6764	* @param pSelReg Pointer to the segment selector.
6765	*
6766	* @remarks No-long-jump zone!!!
6767	* @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
6768	* macro as that takes care of whether to read from the VMCS cache or
6769	* not.
6770	*/
6771	DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
6772	PCPUMSELREG pSelReg)
6773	{
6774	NOREF(pVCpu);
6775
6776	uint32_t u32Val = 0;
6777	int rc = VMXReadVmcs32(idxSel, &u32Val);
6778	AssertRCReturn(rc, rc);
6779	pSelReg->Sel = (uint16_t)u32Val;
6780	pSelReg->ValidSel = (uint16_t)u32Val;
6781	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
6782
6783	rc = VMXReadVmcs32(idxLimit, &u32Val);
6784	AssertRCReturn(rc, rc);
6785	pSelReg->u32Limit = u32Val;
6786
6787	uint64_t u64Val = 0;
6788	rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
6789	AssertRCReturn(rc, rc);
6790	pSelReg->u64Base = u64Val;
6791
6792	rc = VMXReadVmcs32(idxAccess, &u32Val);
6793	AssertRCReturn(rc, rc);
6794	pSelReg->Attr.u = u32Val;
6795
6796	/*
6797	* If VT-x marks the segment as unusable, most other bits remain undefined:
6798	* - For CS the L, D and G bits have meaning.
6799	* - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
6800	* - For the remaining data segments no bits are defined.
6801	*
6802	* The present bit and the unusable bit has been observed to be set at the
6803	* same time (the selector was supposed to be invalid as we started executing
6804	* a V8086 interrupt in ring-0).
6805	*
6806	* What should be important for the rest of the VBox code, is that the P bit is
6807	* cleared. Some of the other VBox code recognizes the unusable bit, but
6808	* AMD-V certainly don't, and REM doesn't really either. So, to be on the
6809	* safe side here, we'll strip off P and other bits we don't care about. If
6810	* any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
6811	*
6812	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
6813	*/
6814	if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6815	{
6816	Assert(idxSel != VMX_VMCS16_GUEST_TR_SEL); /* TR is the only selector that can never be unusable. */
6817
6818	/* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6819	pSelReg->Attr.u &= X86DESCATTR_UNUSABLE \| X86DESCATTR_L \| X86DESCATTR_D \| X86DESCATTR_G
6820	\| X86DESCATTR_DPL \| X86DESCATTR_TYPE \| X86DESCATTR_DT;
6821
6822	Log4(("hmR0VmxReadSegmentReg: Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Val, pSelReg->Attr.u));
6823	#ifdef DEBUG_bird
6824	AssertMsg((u32Val & ~X86DESCATTR_P) == pSelReg->Attr.u,
6825	("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6826	idxSel, u32Val, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6827	#endif
6828	}
6829	return VINF_SUCCESS;
6830	}
6831
6832
6833	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
6834	# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6835	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6836	VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6837	#else
6838	# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6839	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6840	VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6841	#endif
6842
6843
6844	/**
6845	* Saves the guest segment registers from the current VMCS into the guest-CPU
6846	* context.
6847	*
6848	* @returns VBox status code.
6849	* @param pVCpu The cross context virtual CPU structure.
6850	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6851	* out-of-sync. Make sure to update the required fields
6852	* before using them.
6853	*
6854	* @remarks No-long-jump zone!!!
6855	*/
6856	static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6857	{
6858	/* Guest segment registers. */
6859	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS))
6860	{
6861	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS));
6862	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6863	AssertRCReturn(rc, rc);
6864
6865	rc = VMXLOCAL_READ_SEG(CS, cs);
6866	rc \|= VMXLOCAL_READ_SEG(SS, ss);
6867	rc \|= VMXLOCAL_READ_SEG(DS, ds);
6868	rc \|= VMXLOCAL_READ_SEG(ES, es);
6869	rc \|= VMXLOCAL_READ_SEG(FS, fs);
6870	rc \|= VMXLOCAL_READ_SEG(GS, gs);
6871	AssertRCReturn(rc, rc);
6872
6873	/* Restore segment attributes for real-on-v86 mode hack. */
6874	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6875	{
6876	pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6877	pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6878	pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6879	pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6880	pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6881	pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6882	}
6883	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS);
6884	}
6885
6886	return VINF_SUCCESS;
6887	}
6888
6889
6890	/**
6891	* Saves the guest descriptor table registers and task register from the current
6892	* VMCS into the guest-CPU context.
6893	*
6894	* @returns VBox status code.
6895	* @param pVCpu The cross context virtual CPU structure.
6896	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6897	* out-of-sync. Make sure to update the required fields
6898	* before using them.
6899	*
6900	* @remarks No-long-jump zone!!!
6901	*/
6902	static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6903	{
6904	int rc = VINF_SUCCESS;
6905
6906	/* Guest LDTR. */
6907	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR))
6908	{
6909	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR));
6910	rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
6911	AssertRCReturn(rc, rc);
6912	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR);
6913	}
6914
6915	/* Guest GDTR. */
6916	uint64_t u64Val = 0;
6917	uint32_t u32Val = 0;
6918	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR))
6919	{
6920	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR));
6921	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
6922	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6923	pMixedCtx->gdtr.pGdt = u64Val;
6924	pMixedCtx->gdtr.cbGdt = u32Val;
6925	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR);
6926	}
6927
6928	/* Guest IDTR. */
6929	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR))
6930	{
6931	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR));
6932	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
6933	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6934	pMixedCtx->idtr.pIdt = u64Val;
6935	pMixedCtx->idtr.cbIdt = u32Val;
6936	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR);
6937	}
6938
6939	/* Guest TR. */
6940	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR))
6941	{
6942	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR));
6943	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6944	AssertRCReturn(rc, rc);
6945
6946	/* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
6947	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6948	{
6949	rc = VMXLOCAL_READ_SEG(TR, tr);
6950	AssertRCReturn(rc, rc);
6951	}
6952	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR);
6953	}
6954	return rc;
6955	}
6956
6957	#undef VMXLOCAL_READ_SEG
6958
6959
6960	/**
6961	* Saves the guest debug-register DR7 from the current VMCS into the guest-CPU
6962	* context.
6963	*
6964	* @returns VBox status code.
6965	* @param pVCpu The cross context virtual CPU structure.
6966	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6967	* out-of-sync. Make sure to update the required fields
6968	* before using them.
6969	*
6970	* @remarks No-long-jump zone!!!
6971	*/
6972	static int hmR0VmxSaveGuestDR7(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6973	{
6974	if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DR7))
6975	{
6976	if (!pVCpu->hm.s.fUsingHyperDR7)
6977	{
6978	/* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6979	uint32_t u32Val;
6980	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
6981	pMixedCtx->dr[7] = u32Val;
6982	}
6983
6984	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DR7);
6985	}
6986	return VINF_SUCCESS;
6987	}
6988
6989
6990	/**
6991	* Saves the guest APIC state from the current VMCS into the guest-CPU context.
6992	*
6993	* @returns VBox status code.
6994	* @param pVCpu The cross context virtual CPU structure.
6995	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6996	* out-of-sync. Make sure to update the required fields
6997	* before using them.
6998	*
6999	* @remarks No-long-jump zone!!!
7000	*/
7001	static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7002	{
7003	NOREF(pMixedCtx);
7004
7005	/* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
7006	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_APIC_STATE);
7007	return VINF_SUCCESS;
7008	}
7009
7010
7011	/**
7012	* Saves the entire guest state from the currently active VMCS into the
7013	* guest-CPU context.
7014	*
7015	* This essentially VMREADs all guest-data.
7016	*
7017	* @returns VBox status code.
7018	* @param pVCpu The cross context virtual CPU structure.
7019	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7020	* out-of-sync. Make sure to update the required fields
7021	* before using them.
7022	*/
7023	static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7024	{
7025	Assert(pVCpu);
7026	Assert(pMixedCtx);
7027
7028	if (HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL)
7029	return VINF_SUCCESS;
7030
7031	/* Though we can longjmp to ring-3 due to log-flushes here and get recalled
7032	again on the ring-3 callback path, there is no real need to. */
7033	if (VMMRZCallRing3IsEnabled(pVCpu))
7034	VMMR0LogFlushDisable(pVCpu);
7035	else
7036	Assert(VMMR0IsLogFlushDisabled(pVCpu));
7037	Log4Func(("vcpu[%RU32]\n", pVCpu->idCpu));
7038
7039	int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
7040	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7041
7042	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
7043	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7044
7045	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7046	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7047
7048	rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
7049	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7050
7051	rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
7052	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDR7 failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7053
7054	rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
7055	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7056
7057	rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
7058	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestLazyMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7059
7060	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
7061	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7062
7063	rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
7064	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7065
7066	rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
7067	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestApicState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7068
7069	AssertMsg(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL,
7070	("Missed guest state bits while saving state; missing %RX32 (got %RX32, want %RX32) - check log for any previous errors!\n",
7071	HMVMX_UPDATED_GUEST_ALL ^ HMVMXCPU_GST_VALUE(pVCpu), HMVMXCPU_GST_VALUE(pVCpu), HMVMX_UPDATED_GUEST_ALL));
7072
7073	if (VMMRZCallRing3IsEnabled(pVCpu))
7074	VMMR0LogFlushEnable(pVCpu);
7075
7076	return VINF_SUCCESS;
7077	}
7078
7079
7080	/**
7081	* Saves basic guest registers needed for IEM instruction execution.
7082	*
7083	* @returns VBox status code (OR-able).
7084	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
7085	* @param pMixedCtx Pointer to the CPU context of the guest.
7086	* @param fMemory Whether the instruction being executed operates on
7087	* memory or not. Only CR0 is synced up if clear.
7088	* @param fNeedRsp Need RSP (any instruction working on GPRs or stack).
7089	*/
7090	static int hmR0VmxSaveGuestRegsForIemExec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fMemory, bool fNeedRsp)
7091	{
7092	/*
7093	* We assume all general purpose registers other than RSP are available.
7094	*
7095	* - RIP is a must, as it will be incremented or otherwise changed.
7096	* - RFLAGS are always required to figure the CPL.
7097	* - RSP isn't always required, however it's a GPR, so frequently required.
7098	* - SS and CS are the only segment register needed if IEM doesn't do memory
7099	* access (CPL + 16/32/64-bit mode), but we can only get all segment registers.
7100	* - CR0 is always required by IEM for the CPL, while CR3 and CR4 will only
7101	* be required for memory accesses.
7102	*
7103	* Note! Before IEM dispatches an exception, it will call us to sync in everything.
7104	*/
7105	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
7106	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7107	if (fNeedRsp)
7108	rc \|= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
7109	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7110	if (!fMemory)
7111	rc \|= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7112	else
7113	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
7114	AssertRCReturn(rc, rc);
7115	return rc;
7116	}
7117
7118
7119	/**
7120	* Ensures that we've got a complete basic guest-context.
7121	*
7122	* This excludes the FPU, SSE, AVX, and similar extended state. The interface
7123	* is for the interpreter.
7124	*
7125	* @returns VBox status code.
7126	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
7127	* @param pMixedCtx Pointer to the guest-CPU context which may have data
7128	* needing to be synced in.
7129	* @thread EMT(pVCpu)
7130	*/
7131	VMMR0_INT_DECL(int) HMR0EnsureCompleteBasicContext(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7132	{
7133	/* Note! Since this is only applicable to VT-x, the implementation is placed
7134	in the VT-x part of the sources instead of the generic stuff. */
7135	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported)
7136	{
7137	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7138	/*
7139	* For now, imply that the caller might change everything too. Do this after
7140	* saving the guest state so as to not trigger assertions.
7141	*/
7142	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7143	return rc;
7144	}
7145	return VINF_SUCCESS;
7146	}
7147
7148
7149	/**
7150	* Check per-VM and per-VCPU force flag actions that require us to go back to
7151	* ring-3 for one reason or another.
7152	*
7153	* @returns Strict VBox status code (i.e. informational status codes too)
7154	* @retval VINF_SUCCESS if we don't have any actions that require going back to
7155	* ring-3.
7156	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
7157	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
7158	* interrupts)
7159	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
7160	* all EMTs to be in ring-3.
7161	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
7162	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
7163	* to the EM loop.
7164	*
7165	* @param pVM The cross context VM structure.
7166	* @param pVCpu The cross context virtual CPU structure.
7167	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7168	* out-of-sync. Make sure to update the required fields
7169	* before using them.
7170	* @param fStepping Running in hmR0VmxRunGuestCodeStep().
7171	*/
7172	static VBOXSTRICTRC hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping)
7173	{
7174	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7175
7176	/*
7177	* Anything pending? Should be more likely than not if we're doing a good job.
7178	*/
7179	if ( !fStepping
7180	? !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_MASK)
7181	&& !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
7182	: !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
7183	&& !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
7184	return VINF_SUCCESS;
7185
7186	/* We need the control registers now, make sure the guest-CPU context is updated. */
7187	int rc3 = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
7188	AssertRCReturn(rc3, rc3);
7189
7190	/* Pending HM CR3 sync. */
7191	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
7192	{
7193	int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
7194	AssertMsgReturn(rc2 == VINF_SUCCESS \|\| rc2 == VINF_PGM_SYNC_CR3,
7195	("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
7196	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
7197	}
7198
7199	/* Pending HM PAE PDPEs. */
7200	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
7201	{
7202	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
7203	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
7204	}
7205
7206	/* Pending PGM C3 sync. */
7207	if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
7208	{
7209	VBOXSTRICTRC rcStrict2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
7210	VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
7211	if (rcStrict2 != VINF_SUCCESS)
7212	{
7213	AssertRC(VBOXSTRICTRC_VAL(rcStrict2));
7214	Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
7215	return rcStrict2;
7216	}
7217	}
7218
7219	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
7220	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
7221	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
7222	{
7223	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
7224	int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
7225	Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
7226	return rc2;
7227	}
7228
7229	/* Pending VM request packets, such as hardware interrupts. */
7230	if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
7231	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
7232	{
7233	Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
7234	return VINF_EM_PENDING_REQUEST;
7235	}
7236
7237	/* Pending PGM pool flushes. */
7238	if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
7239	{
7240	Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
7241	return VINF_PGM_POOL_FLUSH_PENDING;
7242	}
7243
7244	/* Pending DMA requests. */
7245	if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
7246	{
7247	Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
7248	return VINF_EM_RAW_TO_R3;
7249	}
7250
7251	return VINF_SUCCESS;
7252	}
7253
7254
7255	/**
7256	* Converts any TRPM trap into a pending HM event. This is typically used when
7257	* entering from ring-3 (not longjmp returns).
7258	*
7259	* @param pVCpu The cross context virtual CPU structure.
7260	*/
7261	static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
7262	{
7263	Assert(TRPMHasTrap(pVCpu));
7264	Assert(!pVCpu->hm.s.Event.fPending);
7265
7266	uint8_t uVector;
7267	TRPMEVENT enmTrpmEvent;
7268	RTGCUINT uErrCode;
7269	RTGCUINTPTR GCPtrFaultAddress;
7270	uint8_t cbInstr;
7271
7272	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
7273	AssertRC(rc);
7274
7275	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
7276	uint32_t u32IntInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7277	if (enmTrpmEvent == TRPM_TRAP)
7278	{
7279	switch (uVector)
7280	{
7281	case X86_XCPT_NMI:
7282	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7283	break;
7284
7285	case X86_XCPT_BP:
7286	case X86_XCPT_OF:
7287	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7288	break;
7289
7290	case X86_XCPT_PF:
7291	case X86_XCPT_DF:
7292	case X86_XCPT_TS:
7293	case X86_XCPT_NP:
7294	case X86_XCPT_SS:
7295	case X86_XCPT_GP:
7296	case X86_XCPT_AC:
7297	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7298	RT_FALL_THRU();
7299	default:
7300	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7301	break;
7302	}
7303	}
7304	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
7305	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7306	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
7307	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7308	else
7309	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
7310
7311	rc = TRPMResetTrap(pVCpu);
7312	AssertRC(rc);
7313	Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
7314	u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
7315
7316	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
7317	}
7318
7319
7320	/**
7321	* Converts the pending HM event into a TRPM trap.
7322	*
7323	* @param pVCpu The cross context virtual CPU structure.
7324	*/
7325	static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
7326	{
7327	Assert(pVCpu->hm.s.Event.fPending);
7328
7329	uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7330	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
7331	bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntInfo);
7332	uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
7333
7334	/* If a trap was already pending, we did something wrong! */
7335	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
7336
7337	TRPMEVENT enmTrapType;
7338	switch (uVectorType)
7339	{
7340	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
7341	enmTrapType = TRPM_HARDWARE_INT;
7342	break;
7343
7344	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
7345	enmTrapType = TRPM_SOFTWARE_INT;
7346	break;
7347
7348	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
7349	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
7350	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
7351	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
7352	enmTrapType = TRPM_TRAP;
7353	break;
7354
7355	default:
7356	AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
7357	enmTrapType = TRPM_32BIT_HACK;
7358	break;
7359	}
7360
7361	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
7362
7363	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
7364	AssertRC(rc);
7365
7366	if (fErrorCodeValid)
7367	TRPMSetErrorCode(pVCpu, uErrorCode);
7368
7369	if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
7370	&& uVector == X86_XCPT_PF)
7371	{
7372	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
7373	}
7374	else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
7375	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
7376	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
7377	{
7378	AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
7379	\|\| (uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF),
7380	("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
7381	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
7382	}
7383
7384	/* Clear any pending events from the VMCS. */
7385	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0); AssertRC(rc);
7386	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0); AssertRC(rc);
7387
7388	/* We're now done converting the pending event. */
7389	pVCpu->hm.s.Event.fPending = false;
7390	}
7391
7392
7393	/**
7394	* Does the necessary state syncing before returning to ring-3 for any reason
7395	* (longjmp, preemption, voluntary exits to ring-3) from VT-x.
7396	*
7397	* @returns VBox status code.
7398	* @param pVCpu The cross context virtual CPU structure.
7399	* @param pMixedCtx Pointer to the guest-CPU context. The data may
7400	* be out-of-sync. Make sure to update the required
7401	* fields before using them.
7402	* @param fSaveGuestState Whether to save the guest state or not.
7403	*
7404	* @remarks No-long-jmp zone!!!
7405	*/
7406	static int hmR0VmxLeave(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fSaveGuestState)
7407	{
7408	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7409	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7410
7411	RTCPUID idCpu = RTMpCpuId();
7412	Log4Func(("HostCpuId=%u\n", idCpu));
7413
7414	/*
7415	* !!! IMPORTANT !!!
7416	* If you modify code here, check whether hmR0VmxCallRing3Callback() needs to be updated too.
7417	*/
7418
7419	/* Save the guest state if necessary. */
7420	if ( fSaveGuestState
7421	&& HMVMXCPU_GST_VALUE(pVCpu) != HMVMX_UPDATED_GUEST_ALL)
7422	{
7423	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7424	AssertRCReturn(rc, rc);
7425	Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
7426	}
7427
7428	/* Restore host FPU state if necessary and resync on next R0 reentry .*/
7429	if (CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu))
7430	{
7431	/* We shouldn't reload CR0 without saving it first. */
7432	if (!fSaveGuestState)
7433	{
7434	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7435	AssertRCReturn(rc, rc);
7436	}
7437	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
7438	}
7439
7440	/* Restore host debug registers if necessary and resync on next R0 reentry. */
7441	#ifdef VBOX_STRICT
7442	if (CPUMIsHyperDebugStateActive(pVCpu))
7443	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
7444	#endif
7445	if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */))
7446	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
7447	Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
7448	Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
7449
7450	#if HC_ARCH_BITS == 64
7451	/* Restore host-state bits that VT-x only restores partially. */
7452	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7453	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7454	{
7455	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
7456	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7457	}
7458	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7459	#endif
7460
7461	/* Restore the lazy host MSRs as we're leaving VT-x context. */
7462	if (pVCpu->hm.s.vmx.fLazyMsrs)
7463	{
7464	/* We shouldn't reload the guest MSRs without saving it first. */
7465	if (!fSaveGuestState)
7466	{
7467	int rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
7468	AssertRCReturn(rc, rc);
7469	}
7470	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS));
7471	hmR0VmxLazyRestoreHostMsrs(pVCpu);
7472	Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
7473	}
7474
7475	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7476	pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7477
7478	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
7479	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
7480	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
7481	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
7482	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
7483	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
7484	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
7485	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7486
7487	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7488
7489	/** @todo This partially defeats the purpose of having preemption hooks.
7490	* The problem is, deregistering the hooks should be moved to a place that
7491	* lasts until the EMT is about to be destroyed not everytime while leaving HM
7492	* context.
7493	*/
7494	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7495	{
7496	int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7497	AssertRCReturn(rc, rc);
7498
7499	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7500	Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
7501	}
7502	Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
7503	NOREF(idCpu);
7504
7505	return VINF_SUCCESS;
7506	}
7507
7508
7509	/**
7510	* Leaves the VT-x session.
7511	*
7512	* @returns VBox status code.
7513	* @param pVCpu The cross context virtual CPU structure.
7514	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7515	* out-of-sync. Make sure to update the required fields
7516	* before using them.
7517	*
7518	* @remarks No-long-jmp zone!!!
7519	*/
7520	DECLINLINE(int) hmR0VmxLeaveSession(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7521	{
7522	HM_DISABLE_PREEMPT();
7523	HMVMX_ASSERT_CPU_SAFE();
7524	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7525	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7526
7527	/* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
7528	and done this from the VMXR0ThreadCtxCallback(). */
7529	if (!pVCpu->hm.s.fLeaveDone)
7530	{
7531	int rc2 = hmR0VmxLeave(pVCpu, pMixedCtx, true /* fSaveGuestState */);
7532	AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
7533	pVCpu->hm.s.fLeaveDone = true;
7534	}
7535	Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
7536
7537	/*
7538	* !!! IMPORTANT !!!
7539	* If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
7540	*/
7541
7542	/* Deregister hook now that we've left HM context before re-enabling preemption. */
7543	/** @todo Deregistering here means we need to VMCLEAR always
7544	* (longjmp/exit-to-r3) in VT-x which is not efficient. */
7545	/** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7546	VMMR0ThreadCtxHookDisable(pVCpu);
7547
7548	/* Leave HM context. This takes care of local init (term). */
7549	int rc = HMR0LeaveCpu(pVCpu);
7550
7551	HM_RESTORE_PREEMPT();
7552	return rc;
7553	}
7554
7555
7556	/**
7557	* Does the necessary state syncing before doing a longjmp to ring-3.
7558	*
7559	* @returns VBox status code.
7560	* @param pVCpu The cross context virtual CPU structure.
7561	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7562	* out-of-sync. Make sure to update the required fields
7563	* before using them.
7564	*
7565	* @remarks No-long-jmp zone!!!
7566	*/
7567	DECLINLINE(int) hmR0VmxLongJmpToRing3(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7568	{
7569	return hmR0VmxLeaveSession(pVCpu, pMixedCtx);
7570	}
7571
7572
7573	/**
7574	* Take necessary actions before going back to ring-3.
7575	*
7576	* An action requires us to go back to ring-3. This function does the necessary
7577	* steps before we can safely return to ring-3. This is not the same as longjmps
7578	* to ring-3, this is voluntary and prepares the guest so it may continue
7579	* executing outside HM (recompiler/IEM).
7580	*
7581	* @returns VBox status code.
7582	* @param pVM The cross context VM structure.
7583	* @param pVCpu The cross context virtual CPU structure.
7584	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7585	* out-of-sync. Make sure to update the required fields
7586	* before using them.
7587	* @param rcExit The reason for exiting to ring-3. Can be
7588	* VINF_VMM_UNKNOWN_RING3_CALL.
7589	*/
7590	static int hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, VBOXSTRICTRC rcExit)
7591	{
7592	Assert(pVM);
7593	Assert(pVCpu);
7594	Assert(pMixedCtx);
7595	HMVMX_ASSERT_PREEMPT_SAFE();
7596
7597	if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
7598	{
7599	VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
7600	pVCpu->hm.s.vmx.LastError.u32VMCSRevision = (uint32_t )pVCpu->hm.s.vmx.pvVmcs;
7601	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
7602	/* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
7603	}
7604
7605	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
7606	VMMRZCallRing3Disable(pVCpu);
7607	Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, VBOXSTRICTRC_VAL(rcExit)));
7608
7609	/* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
7610	if (pVCpu->hm.s.Event.fPending)
7611	{
7612	hmR0VmxPendingEventToTrpmTrap(pVCpu);
7613	Assert(!pVCpu->hm.s.Event.fPending);
7614	}
7615
7616	/* Clear interrupt-window and NMI-window controls as we re-evaluate it when we return from ring-3. */
7617	hmR0VmxClearIntNmiWindowsVmcs(pVCpu);
7618
7619	/* If we're emulating an instruction, we shouldn't have any TRPM traps pending
7620	and if we're injecting an event we should have a TRPM trap pending. */
7621	AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT \|\| TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
7622	#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a tripple fault in progress. */
7623	AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR \|\| !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
7624	#endif
7625
7626	/* Save guest state and restore host state bits. */
7627	int rc = hmR0VmxLeaveSession(pVCpu, pMixedCtx);
7628	AssertRCReturn(rc, rc);
7629	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7630	/* Thread-context hooks are unregistered at this point!!! */
7631
7632	/* Sync recompiler state. */
7633	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
7634	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
7635	\| CPUM_CHANGED_LDTR
7636	\| CPUM_CHANGED_GDTR
7637	\| CPUM_CHANGED_IDTR
7638	\| CPUM_CHANGED_TR
7639	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
7640	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
7641	if ( pVM->hm.s.fNestedPaging
7642	&& CPUMIsGuestPagingEnabledEx(pMixedCtx))
7643	{
7644	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
7645	}
7646
7647	Assert(!pVCpu->hm.s.fClearTrapFlag);
7648
7649	/* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
7650	if (rcExit != VINF_EM_RAW_INTERRUPT)
7651	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7652
7653	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
7654
7655	/* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
7656	VMMRZCallRing3RemoveNotification(pVCpu);
7657	VMMRZCallRing3Enable(pVCpu);
7658
7659	return rc;
7660	}
7661
7662
7663	/**
7664	* VMMRZCallRing3() callback wrapper which saves the guest state before we
7665	* longjump to ring-3 and possibly get preempted.
7666	*
7667	* @returns VBox status code.
7668	* @param pVCpu The cross context virtual CPU structure.
7669	* @param enmOperation The operation causing the ring-3 longjump.
7670	* @param pvUser Opaque pointer to the guest-CPU context. The data
7671	* may be out-of-sync. Make sure to update the required
7672	* fields before using them.
7673	*/
7674	static DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
7675	{
7676	if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
7677	{
7678	/*
7679	* !!! IMPORTANT !!!
7680	* If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
7681	* This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
7682	*/
7683	VMMRZCallRing3RemoveNotification(pVCpu);
7684	VMMRZCallRing3Disable(pVCpu);
7685	RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
7686	RTThreadPreemptDisable(&PreemptState);
7687
7688	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
7689	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
7690
7691	#if HC_ARCH_BITS == 64
7692	/* Restore host-state bits that VT-x only restores partially. */
7693	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7694	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7695	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7696	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7697	#endif
7698	/* Restore the lazy host MSRs as we're leaving VT-x context. */
7699	if (pVCpu->hm.s.vmx.fLazyMsrs)
7700	hmR0VmxLazyRestoreHostMsrs(pVCpu);
7701
7702	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7703	pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7704	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7705	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7706	{
7707	VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7708	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7709	}
7710
7711	/** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7712	VMMR0ThreadCtxHookDisable(pVCpu);
7713	HMR0LeaveCpu(pVCpu);
7714	RTThreadPreemptRestore(&PreemptState);
7715	return VINF_SUCCESS;
7716	}
7717
7718	Assert(pVCpu);
7719	Assert(pvUser);
7720	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7721	HMVMX_ASSERT_PREEMPT_SAFE();
7722
7723	VMMRZCallRing3Disable(pVCpu);
7724	Assert(VMMR0IsLogFlushDisabled(pVCpu));
7725
7726	Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32 enmOperation=%d\n", pVCpu, pVCpu->idCpu,
7727	enmOperation));
7728
7729	int rc = hmR0VmxLongJmpToRing3(pVCpu, (PCPUMCTX)pvUser);
7730	AssertRCReturn(rc, rc);
7731
7732	VMMRZCallRing3Enable(pVCpu);
7733	return VINF_SUCCESS;
7734	}
7735
7736
7737	/**
7738	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7739	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
7740	*
7741	* @param pVCpu The cross context virtual CPU structure.
7742	*/
7743	DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
7744	{
7745	if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7746	{
7747	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7748	{
7749	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7750	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7751	AssertRC(rc);
7752	Log4(("Setup interrupt-window exiting\n"));
7753	}
7754	} /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
7755	}
7756
7757
7758	/**
7759	* Clears the interrupt-window exiting control in the VMCS.
7760	*
7761	* @param pVCpu The cross context virtual CPU structure.
7762	*/
7763	DECLINLINE(void) hmR0VmxClearIntWindowExitVmcs(PVMCPU pVCpu)
7764	{
7765	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7766	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7767	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7768	AssertRC(rc);
7769	Log4(("Cleared interrupt-window exiting\n"));
7770	}
7771
7772
7773	/**
7774	* Sets the NMI-window exiting control in the VMCS which instructs VT-x to
7775	* cause a VM-exit as soon as the guest is in a state to receive NMIs.
7776	*
7777	* @param pVCpu The cross context virtual CPU structure.
7778	*/
7779	DECLINLINE(void) hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu)
7780	{
7781	if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7782	{
7783	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7784	{
7785	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7786	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7787	AssertRC(rc);
7788	Log4(("Setup NMI-window exiting\n"));
7789	}
7790	} /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
7791	}
7792
7793
7794	/**
7795	* Clears the NMI-window exiting control in the VMCS.
7796	*
7797	* @param pVCpu The cross context virtual CPU structure.
7798	*/
7799	DECLINLINE(void) hmR0VmxClearNmiWindowExitVmcs(PVMCPU pVCpu)
7800	{
7801	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT);
7802	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7803	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7804	AssertRC(rc);
7805	Log4(("Cleared NMI-window exiting\n"));
7806	}
7807
7808
7809	/**
7810	* Evaluates the event to be delivered to the guest and sets it as the pending
7811	* event.
7812	*
7813	* @returns The VT-x guest-interruptibility state.
7814	* @param pVCpu The cross context virtual CPU structure.
7815	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7816	* out-of-sync. Make sure to update the required fields
7817	* before using them.
7818	*/
7819	static uint32_t hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7820	{
7821	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7822	uint32_t const uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7823	bool const fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7824	bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7825	bool const fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7826
7827	Assert(!fBlockSti \|\| HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7828	Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7829	Assert(!fBlockSti \|\| pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7830	Assert(!TRPMHasTrap(pVCpu));
7831
7832	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
7833	APICUpdatePendingInterrupts(pVCpu);
7834
7835	/*
7836	* Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
7837	* to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
7838	*/
7839	/** @todo SMI. SMIs take priority over NMIs. */
7840	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
7841	{
7842	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
7843	if ( !pVCpu->hm.s.Event.fPending
7844	&& !fBlockNmi
7845	&& !fBlockSti
7846	&& !fBlockMovSS)
7847	{
7848	Log4(("Pending NMI vcpu[%RU32]\n", pVCpu->idCpu));
7849	uint32_t u32IntInfo = X86_XCPT_NMI \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7850	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7851
7852	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
7853	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
7854	}
7855	else
7856	hmR0VmxSetNmiWindowExitVmcs(pVCpu);
7857	}
7858	/*
7859	* Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt() returns
7860	* a valid interrupt we must- deliver the interrupt. We can no longer re-request it from the APIC.
7861	*/
7862	else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC))
7863	&& !pVCpu->hm.s.fSingleInstruction)
7864	{
7865	Assert(!DBGFIsStepping(pVCpu));
7866	int rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7867	AssertRC(rc);
7868	bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7869	if ( !pVCpu->hm.s.Event.fPending
7870	&& !fBlockInt
7871	&& !fBlockSti
7872	&& !fBlockMovSS)
7873	{
7874	uint8_t u8Interrupt;
7875	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
7876	if (RT_SUCCESS(rc))
7877	{
7878	Log4(("Pending interrupt vcpu[%RU32] u8Interrupt=%#x \n", pVCpu->idCpu, u8Interrupt));
7879	uint32_t u32IntInfo = u8Interrupt \| VMX_EXIT_INTERRUPTION_INFO_VALID;
7880	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7881
7882	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrfaultAddress */);
7883	}
7884	else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
7885	{
7886	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
7887	hmR0VmxApicSetTprThreshold(pVCpu, u8Interrupt >> 4);
7888	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
7889
7890	/*
7891	* If the CPU doesn't have TPR shadowing, we will always get a VM-exit on TPR changes and
7892	* APICSetTpr() will end up setting the VMCPU_FF_INTERRUPT_APIC if required, so there is no
7893	* need to re-set this force-flag here.
7894	*/
7895	}
7896	else
7897	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
7898	}
7899	else
7900	hmR0VmxSetIntWindowExitVmcs(pVCpu);
7901	}
7902
7903	return uIntrState;
7904	}
7905
7906
7907	/**
7908	* Sets a pending-debug exception to be delivered to the guest if the guest is
7909	* single-stepping in the VMCS.
7910	*
7911	* @param pVCpu The cross context virtual CPU structure.
7912	*/
7913	DECLINLINE(void) hmR0VmxSetPendingDebugXcptVmcs(PVMCPU pVCpu)
7914	{
7915	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS)); NOREF(pVCpu);
7916	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
7917	AssertRC(rc);
7918	}
7919
7920
7921	/**
7922	* Injects any pending events into the guest if the guest is in a state to
7923	* receive them.
7924	*
7925	* @returns Strict VBox status code (i.e. informational status codes too).
7926	* @param pVCpu The cross context virtual CPU structure.
7927	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
7928	* out-of-sync. Make sure to update the required fields
7929	* before using them.
7930	* @param uIntrState The VT-x guest-interruptibility state.
7931	* @param fStepping Running in hmR0VmxRunGuestCodeStep() and we should
7932	* return VINF_EM_DBG_STEPPED if the event was
7933	* dispatched directly.
7934	*/
7935	static VBOXSTRICTRC hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t uIntrState, bool fStepping)
7936	{
7937	HMVMX_ASSERT_PREEMPT_SAFE();
7938	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7939
7940	bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7941	bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7942
7943	Assert(!fBlockSti \|\| HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7944	Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7945	Assert(!fBlockSti \|\| pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7946	Assert(!TRPMHasTrap(pVCpu));
7947
7948	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
7949	if (pVCpu->hm.s.Event.fPending)
7950	{
7951	/*
7952	* Do -not- clear any interrupt-window exiting control here. We might have an interrupt
7953	* pending even while injecting an event and in this case, we want a VM-exit as soon as
7954	* the guest is ready for the next interrupt, see @bugref{6208#c45}.
7955	*
7956	* See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
7957	*/
7958	uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7959	#ifdef VBOX_STRICT
7960	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7961	{
7962	bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7963	Assert(!fBlockInt);
7964	Assert(!fBlockSti);
7965	Assert(!fBlockMovSS);
7966	}
7967	else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7968	{
7969	bool const fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7970	Assert(!fBlockSti);
7971	Assert(!fBlockMovSS);
7972	Assert(!fBlockNmi);
7973	}
7974	#endif
7975	Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#x\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
7976	(uint8_t)uIntType));
7977	rcStrict = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
7978	pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress,
7979	fStepping, &uIntrState);
7980	AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
7981
7982	/* Update the interruptibility-state as it could have been changed by
7983	hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
7984	fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7985	fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7986
7987	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7988	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
7989	else
7990	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
7991	}
7992
7993	/* Deliver pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
7994	if ( fBlockSti
7995	\|\| fBlockMovSS)
7996	{
7997	if (!pVCpu->hm.s.fSingleInstruction)
7998	{
7999	/*
8000	* The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
8001	* VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
8002	* See Intel spec. 27.3.4 "Saving Non-Register State".
8003	*/
8004	Assert(!DBGFIsStepping(pVCpu));
8005	int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8006	AssertRCReturn(rc2, rc2);
8007	if (pMixedCtx->eflags.Bits.u1TF)
8008	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
8009	}
8010	else if (pMixedCtx->eflags.Bits.u1TF)
8011	{
8012	/*
8013	* We are single-stepping in the hypervisor debugger using EFLAGS.TF. Clear interrupt inhibition as setting the
8014	* BS bit would mean delivering a #DB to the guest upon VM-entry when it shouldn't be.
8015	*/
8016	Assert(!(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG));
8017	uIntrState = 0;
8018	}
8019	}
8020
8021	/*
8022	* There's no need to clear the VM-entry interruption-information field here if we're not injecting anything.
8023	* VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
8024	*/
8025	int rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
8026	AssertRC(rc2);
8027
8028	Assert(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
8029	NOREF(fBlockMovSS); NOREF(fBlockSti);
8030	return rcStrict;
8031	}
8032
8033
8034	/**
8035	* Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
8036	*
8037	* @param pVCpu The cross context virtual CPU structure.
8038	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8039	* out-of-sync. Make sure to update the required fields
8040	* before using them.
8041	*/
8042	DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8043	{
8044	NOREF(pMixedCtx);
8045	uint32_t u32IntInfo = X86_XCPT_UD \| VMX_EXIT_INTERRUPTION_INFO_VALID;
8046	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
8047	}
8048
8049
8050	/**
8051	* Injects a double-fault (\#DF) exception into the VM.
8052	*
8053	* @returns Strict VBox status code (i.e. informational status codes too).
8054	* @param pVCpu The cross context virtual CPU structure.
8055	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8056	* out-of-sync. Make sure to update the required fields
8057	* before using them.
8058	* @param fStepping Whether we're running in hmR0VmxRunGuestCodeStep()
8059	* and should return VINF_EM_DBG_STEPPED if the event
8060	* is injected directly (register modified by us, not
8061	* by hardware on VM-entry).
8062	* @param puIntrState Pointer to the current guest interruptibility-state.
8063	* This interruptibility-state will be updated if
8064	* necessary. This cannot not be NULL.
8065	*/
8066	DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping, uint32_t *puIntrState)
8067	{
8068	uint32_t u32IntInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
8069	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8070	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
8071	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */,
8072	fStepping, puIntrState);
8073	}
8074
8075
8076	/**
8077	* Sets a debug (\#DB) exception as pending-for-injection into the VM.
8078	*
8079	* @param pVCpu The cross context virtual CPU structure.
8080	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8081	* out-of-sync. Make sure to update the required fields
8082	* before using them.
8083	*/
8084	DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8085	{
8086	NOREF(pMixedCtx);
8087	uint32_t u32IntInfo = X86_XCPT_DB \| VMX_EXIT_INTERRUPTION_INFO_VALID;
8088	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8089	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
8090	}
8091
8092
8093	/**
8094	* Sets an overflow (\#OF) exception as pending-for-injection into the VM.
8095	*
8096	* @param pVCpu The cross context virtual CPU structure.
8097	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8098	* out-of-sync. Make sure to update the required fields
8099	* before using them.
8100	* @param cbInstr The value of RIP that is to be pushed on the guest
8101	* stack.
8102	*/
8103	DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
8104	{
8105	NOREF(pMixedCtx);
8106	uint32_t u32IntInfo = X86_XCPT_OF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
8107	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8108	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
8109	}
8110
8111
8112	/**
8113	* Injects a general-protection (\#GP) fault into the VM.
8114	*
8115	* @returns Strict VBox status code (i.e. informational status codes too).
8116	* @param pVCpu The cross context virtual CPU structure.
8117	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8118	* out-of-sync. Make sure to update the required fields
8119	* before using them.
8120	* @param fErrorCodeValid Whether the error code is valid (depends on the CPU
8121	* mode, i.e. in real-mode it's not valid).
8122	* @param u32ErrorCode The error code associated with the \#GP.
8123	* @param fStepping Whether we're running in
8124	* hmR0VmxRunGuestCodeStep() and should return
8125	* VINF_EM_DBG_STEPPED if the event is injected
8126	* directly (register modified by us, not by
8127	* hardware on VM-entry).
8128	* @param puIntrState Pointer to the current guest interruptibility-state.
8129	* This interruptibility-state will be updated if
8130	* necessary. This cannot not be NULL.
8131	*/
8132	DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
8133	bool fStepping, uint32_t *puIntrState)
8134	{
8135	uint32_t u32IntInfo = X86_XCPT_GP \| VMX_EXIT_INTERRUPTION_INFO_VALID;
8136	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8137	if (fErrorCodeValid)
8138	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
8139	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr /, u32ErrorCode, 0 / GCPtrFaultAddress */,
8140	fStepping, puIntrState);
8141	}
8142
8143
8144	#if 0 /* unused */
8145	/**
8146	* Sets a general-protection (\#GP) exception as pending-for-injection into the
8147	* VM.
8148	*
8149	* @param pVCpu The cross context virtual CPU structure.
8150	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8151	* out-of-sync. Make sure to update the required fields
8152	* before using them.
8153	* @param u32ErrorCode The error code associated with the \#GP.
8154	*/
8155	DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t u32ErrorCode)
8156	{
8157	NOREF(pMixedCtx);
8158	uint32_t u32IntInfo = X86_XCPT_GP \| VMX_EXIT_INTERRUPTION_INFO_VALID;
8159	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8160	u32IntInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
8161	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, u32ErrorCode, 0 / GCPtrFaultAddress */);
8162	}
8163	#endif /* unused */
8164
8165
8166	/**
8167	* Sets a software interrupt (INTn) as pending-for-injection into the VM.
8168	*
8169	* @param pVCpu The cross context virtual CPU structure.
8170	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8171	* out-of-sync. Make sure to update the required fields
8172	* before using them.
8173	* @param uVector The software interrupt vector number.
8174	* @param cbInstr The value of RIP that is to be pushed on the guest
8175	* stack.
8176	*/
8177	DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
8178	{
8179	NOREF(pMixedCtx);
8180	uint32_t u32IntInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
8181	if ( uVector == X86_XCPT_BP
8182	\|\| uVector == X86_XCPT_OF)
8183	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8184	else
8185	u32IntInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8186	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
8187	}
8188
8189
8190	/**
8191	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
8192	* stack.
8193	*
8194	* @returns Strict VBox status code (i.e. informational status codes too).
8195	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
8196	* @param pVM The cross context VM structure.
8197	* @param pMixedCtx Pointer to the guest-CPU context.
8198	* @param uValue The value to push to the guest stack.
8199	*/
8200	DECLINLINE(VBOXSTRICTRC) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
8201	{
8202	/*
8203	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
8204	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
8205	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
8206	*/
8207	if (pMixedCtx->sp == 1)
8208	return VINF_EM_RESET;
8209	pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
8210	int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
8211	AssertRC(rc);
8212	return rc;
8213	}
8214
8215
8216	/**
8217	* Injects an event into the guest upon VM-entry by updating the relevant fields
8218	* in the VM-entry area in the VMCS.
8219	*
8220	* @returns Strict VBox status code (i.e. informational status codes too).
8221	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
8222	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
8223	*
8224	* @param pVCpu The cross context virtual CPU structure.
8225	* @param pMixedCtx Pointer to the guest-CPU context. The data may
8226	* be out-of-sync. Make sure to update the required
8227	* fields before using them.
8228	* @param u64IntInfo The VM-entry interruption-information field.
8229	* @param cbInstr The VM-entry instruction length in bytes (for
8230	* software interrupts, exceptions and privileged
8231	* software exceptions).
8232	* @param u32ErrCode The VM-entry exception error code.
8233	* @param GCPtrFaultAddress The page-fault address for \#PF exceptions.
8234	* @param puIntrState Pointer to the current guest interruptibility-state.
8235	* This interruptibility-state will be updated if
8236	* necessary. This cannot not be NULL.
8237	* @param fStepping Whether we're running in
8238	* hmR0VmxRunGuestCodeStep() and should return
8239	* VINF_EM_DBG_STEPPED if the event is injected
8240	* directly (register modified by us, not by
8241	* hardware on VM-entry).
8242	*
8243	* @remarks Requires CR0!
8244	*/
8245	static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
8246	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, bool fStepping,
8247	uint32_t *puIntrState)
8248	{
8249	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
8250	AssertMsg(u64IntInfo >> 32 == 0, ("%#RX64\n", u64IntInfo));
8251	Assert(puIntrState);
8252	uint32_t u32IntInfo = (uint32_t)u64IntInfo;
8253
8254	uint32_t const uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntInfo);
8255	uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo);
8256
8257	#ifdef VBOX_STRICT
8258	/*
8259	* Validate the error-code-valid bit for hardware exceptions.
8260	* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling"
8261	*/
8262	if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
8263	&& !CPUMIsGuestInRealModeEx(pMixedCtx))
8264	{
8265	switch (uVector)
8266	{
8267	case X86_XCPT_PF:
8268	case X86_XCPT_DF:
8269	case X86_XCPT_TS:
8270	case X86_XCPT_NP:
8271	case X86_XCPT_SS:
8272	case X86_XCPT_GP:
8273	case X86_XCPT_AC:
8274	AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo),
8275	("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
8276	RT_FALL_THRU();
8277	default:
8278	break;
8279	}
8280	}
8281	#endif
8282
8283	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
8284	Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
8285	\|\| !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
8286
8287	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
8288
8289	/* We require CR0 to check if the guest is in real-mode. */
8290	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8291	AssertRCReturn(rc, rc);
8292
8293	/*
8294	* Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
8295	* mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
8296	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
8297	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
8298	*/
8299	if (CPUMIsGuestInRealModeEx(pMixedCtx))
8300	{
8301	PVM pVM = pVCpu->CTX_SUFF(pVM);
8302	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
8303	{
8304	Assert(PDMVmmDevHeapIsEnabled(pVM));
8305	Assert(pVM->hm.s.vmx.pRealModeTSS);
8306
8307	/* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
8308	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8309	rc \|= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
8310	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8311	AssertRCReturn(rc, rc);
8312	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP));
8313
8314	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
8315	size_t const cbIdtEntry = sizeof(X86IDTR16);
8316	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
8317	{
8318	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
8319	if (uVector == X86_XCPT_DF)
8320	return VINF_EM_RESET;
8321
8322	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
8323	if (uVector == X86_XCPT_GP)
8324	return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, fStepping, puIntrState);
8325
8326	/* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
8327	/* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
8328	return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid /, 0 / u32ErrCode */,
8329	fStepping, puIntrState);
8330	}
8331
8332	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
8333	uint16_t uGuestIp = pMixedCtx->ip;
8334	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
8335	{
8336	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
8337	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
8338	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
8339	}
8340	else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
8341	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
8342
8343	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
8344	X86IDTR16 IdtEntry;
8345	RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
8346	rc = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
8347	AssertRCReturn(rc, rc);
8348
8349	/* Construct the stack frame for the interrupt/exception handler. */
8350	VBOXSTRICTRC rcStrict;
8351	rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
8352	if (rcStrict == VINF_SUCCESS)
8353	rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
8354	if (rcStrict == VINF_SUCCESS)
8355	rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
8356
8357	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
8358	if (rcStrict == VINF_SUCCESS)
8359	{
8360	pMixedCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
8361	pMixedCtx->rip = IdtEntry.offSel;
8362	pMixedCtx->cs.Sel = IdtEntry.uSel;
8363	pMixedCtx->cs.ValidSel = IdtEntry.uSel;
8364	pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
8365	if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
8366	&& uVector == X86_XCPT_PF)
8367	pMixedCtx->cr2 = GCPtrFaultAddress;
8368
8369	/* If any other guest-state bits are changed here, make sure to update
8370	hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
8371	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS
8372	\| HM_CHANGED_GUEST_RIP
8373	\| HM_CHANGED_GUEST_RFLAGS
8374	\| HM_CHANGED_GUEST_RSP);
8375
8376	/* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
8377	if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
8378	{
8379	Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
8380	&& uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
8381	Log4(("Clearing inhibition due to STI.\n"));
8382	*puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
8383	}
8384	Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
8385	u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->eflags.u, pMixedCtx->cs.Sel, pMixedCtx->eip));
8386
8387	/* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
8388	it, if we are returning to ring-3 before executing guest code. */
8389	pVCpu->hm.s.Event.fPending = false;
8390
8391	/* Make hmR0VmxPreRunGuest return if we're stepping since we've changed cs:rip. */
8392	if (fStepping)
8393	rcStrict = VINF_EM_DBG_STEPPED;
8394	}
8395	AssertMsg(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8396	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8397	return rcStrict;
8398	}
8399
8400	/*
8401	* For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
8402	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
8403	*/
8404	u32IntInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
8405	}
8406
8407	/* Validate. */
8408	Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
8409	Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(u32IntInfo)); /* Bit 12 MBZ. */
8410	Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
8411
8412	/* Inject. */
8413	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
8414	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo))
8415	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
8416	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
8417
8418	if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
8419	&& uVector == X86_XCPT_PF)
8420	pMixedCtx->cr2 = GCPtrFaultAddress;
8421
8422	Log4(("Injecting vcpu[%RU32] u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", pVCpu->idCpu,
8423	u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
8424
8425	AssertRCReturn(rc, rc);
8426	return VINF_SUCCESS;
8427	}
8428
8429
8430	/**
8431	* Clears the interrupt-window exiting control in the VMCS and if necessary
8432	* clears the current event in the VMCS as well.
8433	*
8434	* @returns VBox status code.
8435	* @param pVCpu The cross context virtual CPU structure.
8436	*
8437	* @remarks Use this function only to clear events that have not yet been
8438	* delivered to the guest but are injected in the VMCS!
8439	* @remarks No-long-jump zone!!!
8440	*/
8441	static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu)
8442	{
8443	Log4Func(("vcpu[%d]\n", pVCpu->idCpu));
8444
8445	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
8446	hmR0VmxClearIntWindowExitVmcs(pVCpu);
8447
8448	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)
8449	hmR0VmxClearNmiWindowExitVmcs(pVCpu);
8450	}
8451
8452
8453	/**
8454	* Enters the VT-x session.
8455	*
8456	* @returns VBox status code.
8457	* @param pVM The cross context VM structure.
8458	* @param pVCpu The cross context virtual CPU structure.
8459	* @param pCpu Pointer to the CPU info struct.
8460	*/
8461	VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
8462	{
8463	AssertPtr(pVM);
8464	AssertPtr(pVCpu);
8465	Assert(pVM->hm.s.vmx.fSupported);
8466	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8467	NOREF(pCpu); NOREF(pVM);
8468
8469	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8470	Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE));
8471
8472	#ifdef VBOX_STRICT
8473	/* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
8474	RTCCUINTREG uHostCR4 = ASMGetCR4();
8475	if (!(uHostCR4 & X86_CR4_VMXE))
8476	{
8477	LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
8478	return VERR_VMX_X86_CR4_VMXE_CLEARED;
8479	}
8480	#endif
8481
8482	/*
8483	* Load the VCPU's VMCS as the current (and active) one.
8484	*/
8485	Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
8486	int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8487	if (RT_FAILURE(rc))
8488	return rc;
8489
8490	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8491	pVCpu->hm.s.fLeaveDone = false;
8492	Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8493
8494	return VINF_SUCCESS;
8495	}
8496
8497
8498	/**
8499	* The thread-context callback (only on platforms which support it).
8500	*
8501	* @param enmEvent The thread-context event.
8502	* @param pVCpu The cross context virtual CPU structure.
8503	* @param fGlobalInit Whether global VT-x/AMD-V init. was used.
8504	* @thread EMT(pVCpu)
8505	*/
8506	VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
8507	{
8508	NOREF(fGlobalInit);
8509
8510	switch (enmEvent)
8511	{
8512	case RTTHREADCTXEVENT_OUT:
8513	{
8514	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8515	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8516	VMCPU_ASSERT_EMT(pVCpu);
8517
8518	PCPUMCTX pMixedCtx = CPUMQueryGuestCtxPtr(pVCpu);
8519
8520	/* No longjmps (logger flushes, locks) in this fragile context. */
8521	VMMRZCallRing3Disable(pVCpu);
8522	Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
8523
8524	/*
8525	* Restore host-state (FPU, debug etc.)
8526	*/
8527	if (!pVCpu->hm.s.fLeaveDone)
8528	{
8529	/* Do -not- save guest-state here as we might already be in the middle of saving it (esp. bad if we are
8530	holding the PGM lock while saving the guest state (see hmR0VmxSaveGuestControlRegs()). */
8531	hmR0VmxLeave(pVCpu, pMixedCtx, false /* fSaveGuestState */);
8532	pVCpu->hm.s.fLeaveDone = true;
8533	}
8534
8535	/* Leave HM context, takes care of local init (term). */
8536	int rc = HMR0LeaveCpu(pVCpu);
8537	AssertRC(rc); NOREF(rc);
8538
8539	/* Restore longjmp state. */
8540	VMMRZCallRing3Enable(pVCpu);
8541	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
8542	break;
8543	}
8544
8545	case RTTHREADCTXEVENT_IN:
8546	{
8547	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8548	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8549	VMCPU_ASSERT_EMT(pVCpu);
8550
8551	/* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
8552	VMMRZCallRing3Disable(pVCpu);
8553	Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
8554
8555	/* Initialize the bare minimum state required for HM. This takes care of
8556	initializing VT-x if necessary (onlined CPUs, local init etc.) */
8557	int rc = HMR0EnterCpu(pVCpu);
8558	AssertRC(rc);
8559	Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE));
8560
8561	/* Load the active VMCS as the current one. */
8562	if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
8563	{
8564	rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8565	AssertRC(rc); NOREF(rc);
8566	pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8567	Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8568	}
8569	pVCpu->hm.s.fLeaveDone = false;
8570
8571	/* Restore longjmp state. */
8572	VMMRZCallRing3Enable(pVCpu);
8573	break;
8574	}
8575
8576	default:
8577	break;
8578	}
8579	}
8580
8581
8582	/**
8583	* Saves the host state in the VMCS host-state.
8584	* Sets up the VM-exit MSR-load area.
8585	*
8586	* The CPU state will be loaded from these fields on every successful VM-exit.
8587	*
8588	* @returns VBox status code.
8589	* @param pVM The cross context VM structure.
8590	* @param pVCpu The cross context virtual CPU structure.
8591	*
8592	* @remarks No-long-jump zone!!!
8593	*/
8594	static int hmR0VmxSaveHostState(PVM pVM, PVMCPU pVCpu)
8595	{
8596	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8597
8598	int rc = VINF_SUCCESS;
8599	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8600	{
8601	rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
8602	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8603
8604	rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
8605	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8606
8607	rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
8608	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8609
8610	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_HOST_CONTEXT);
8611	}
8612	return rc;
8613	}
8614
8615
8616	/**
8617	* Saves the host state in the VMCS host-state.
8618	*
8619	* @returns VBox status code.
8620	* @param pVM The cross context VM structure.
8621	* @param pVCpu The cross context virtual CPU structure.
8622	*
8623	* @remarks No-long-jump zone!!!
8624	*/
8625	VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
8626	{
8627	AssertPtr(pVM);
8628	AssertPtr(pVCpu);
8629
8630	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8631
8632	/* Save the host state here while entering HM context. When thread-context hooks are used, we might get preempted
8633	and have to resave the host state but most of the time we won't be, so do it here before we disable interrupts. */
8634	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8635	return hmR0VmxSaveHostState(pVM, pVCpu);
8636	}
8637
8638
8639	/**
8640	* Loads the guest state into the VMCS guest-state area.
8641	*
8642	* The will typically be done before VM-entry when the guest-CPU state and the
8643	* VMCS state may potentially be out of sync.
8644	*
8645	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
8646	* VM-entry controls.
8647	* Sets up the appropriate VMX non-root function to execute guest code based on
8648	* the guest CPU mode.
8649	*
8650	* @returns VBox strict status code.
8651	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8652	* without unrestricted guest access and the VMMDev is not presently
8653	* mapped (e.g. EFI32).
8654	*
8655	* @param pVM The cross context VM structure.
8656	* @param pVCpu The cross context virtual CPU structure.
8657	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8658	* out-of-sync. Make sure to update the required fields
8659	* before using them.
8660	*
8661	* @remarks No-long-jump zone!!!
8662	*/
8663	static VBOXSTRICTRC hmR0VmxLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8664	{
8665	AssertPtr(pVM);
8666	AssertPtr(pVCpu);
8667	AssertPtr(pMixedCtx);
8668	HMVMX_ASSERT_PREEMPT_SAFE();
8669
8670	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8671
8672	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
8673
8674	/* Determine real-on-v86 mode. */
8675	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
8676	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
8677	&& CPUMIsGuestInRealModeEx(pMixedCtx))
8678	{
8679	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
8680	}
8681
8682	/*
8683	* Load the guest-state into the VMCS.
8684	* Any ordering dependency among the sub-functions below must be explicitly stated using comments.
8685	* Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
8686	*/
8687	int rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
8688	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8689
8690	/* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
8691	rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
8692	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8693
8694	/* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
8695	rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
8696	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8697
8698	rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
8699	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8700
8701	VBOXSTRICTRC rcStrict = hmR0VmxLoadGuestCR3AndCR4(pVCpu, pMixedCtx);
8702	if (rcStrict == VINF_SUCCESS)
8703	{ /* likely */ }
8704	else
8705	{
8706	Assert(rcStrict == VINF_EM_RESCHEDULE_REM \|\| RT_FAILURE_NP(rcStrict));
8707	return rcStrict;
8708	}
8709
8710	/* Assumes pMixedCtx->cr0 is up-to-date (strict builds require CR0 for segment register validation checks). */
8711	rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
8712	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8713
8714	/* This needs to be done after hmR0VmxLoadGuestEntryCtls() and hmR0VmxLoadGuestExitCtls() as it may alter controls if we
8715	determine we don't have to swap EFER after all. */
8716	rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
8717	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8718
8719	rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
8720	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8721
8722	rc = hmR0VmxLoadGuestXcptIntercepts(pVCpu, pMixedCtx);
8723	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestXcptIntercepts! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8724
8725	/*
8726	* Loading Rflags here is fine, even though Rflags.TF might depend on guest debug state (which is not loaded here).
8727	* It is re-evaluated and updated if necessary in hmR0VmxLoadSharedState().
8728	*/
8729	rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
8730	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestRipRspRflags! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8731
8732	/* Clear any unused and reserved bits. */
8733	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR2);
8734
8735	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
8736	return rc;
8737	}
8738
8739
8740	/**
8741	* Loads the state shared between the host and guest into the VMCS.
8742	*
8743	* @param pVM The cross context VM structure.
8744	* @param pVCpu The cross context virtual CPU structure.
8745	* @param pCtx Pointer to the guest-CPU context.
8746	*
8747	* @remarks No-long-jump zone!!!
8748	*/
8749	static void hmR0VmxLoadSharedState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8750	{
8751	NOREF(pVM);
8752
8753	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8754	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8755
8756	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
8757	{
8758	int rc = hmR0VmxLoadSharedCR0(pVCpu, pCtx);
8759	AssertRC(rc);
8760	}
8761
8762	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
8763	{
8764	int rc = hmR0VmxLoadSharedDebugState(pVCpu, pCtx);
8765	AssertRC(rc);
8766
8767	/* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
8768	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
8769	{
8770	rc = hmR0VmxLoadGuestRflags(pVCpu, pCtx);
8771	AssertRC(rc);
8772	}
8773	}
8774
8775	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS))
8776	{
8777	hmR0VmxLazyLoadGuestMsrs(pVCpu, pCtx);
8778	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
8779	}
8780
8781	/* Loading CR0, debug state might have changed intercepts, update VMCS. */
8782	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS))
8783	{
8784	Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_AC));
8785	Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
8786	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
8787	AssertRC(rc);
8788	HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
8789	}
8790
8791	AssertMsg(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE),
8792	("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8793	}
8794
8795
8796	/**
8797	* Worker for loading the guest-state bits in the inner VT-x execution loop.
8798	*
8799	* @returns Strict VBox status code (i.e. informational status codes too).
8800	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8801	* without unrestricted guest access and the VMMDev is not presently
8802	* mapped (e.g. EFI32).
8803	*
8804	* @param pVM The cross context VM structure.
8805	* @param pVCpu The cross context virtual CPU structure.
8806	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8807	* out-of-sync. Make sure to update the required fields
8808	* before using them.
8809	*
8810	* @remarks No-long-jump zone!!!
8811	*/
8812	static VBOXSTRICTRC hmR0VmxLoadGuestStateOptimal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8813	{
8814	HMVMX_ASSERT_PREEMPT_SAFE();
8815	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8816	Assert(VMMR0IsLogFlushDisabled(pVCpu));
8817
8818	Log5(("LoadFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8819	#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
8820	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
8821	#endif
8822
8823	/*
8824	* RIP is what changes the most often and hence if it's the only bit needing to be
8825	* updated, we shall handle it early for performance reasons.
8826	*/
8827	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
8828	if (HMCPU_CF_IS_SET_ONLY(pVCpu, HM_CHANGED_GUEST_RIP))
8829	{
8830	rcStrict = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
8831	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8832	{ /* likely */}
8833	else
8834	{
8835	AssertMsgFailedReturn(("hmR0VmxLoadGuestStateOptimal: hmR0VmxLoadGuestRip failed! rc=%Rrc\n",
8836	VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
8837	}
8838	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
8839	}
8840	else if (HMCPU_CF_VALUE(pVCpu))
8841	{
8842	rcStrict = hmR0VmxLoadGuestState(pVM, pVCpu, pMixedCtx);
8843	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8844	{ /* likely */}
8845	else
8846	{
8847	AssertMsg(rcStrict == VINF_EM_RESCHEDULE_REM,
8848	("hmR0VmxLoadGuestStateOptimal: hmR0VmxLoadGuestState failed! rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8849	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8850	return rcStrict;
8851	}
8852	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
8853	}
8854
8855	/* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
8856	AssertMsg( !HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_ALL_GUEST)
8857	\|\| HMCPU_CF_IS_PENDING_ONLY(pVCpu, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_HOST_GUEST_SHARED_STATE),
8858	("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8859	return rcStrict;
8860	}
8861
8862
8863	/**
8864	* Does the preparations before executing guest code in VT-x.
8865	*
8866	* This may cause longjmps to ring-3 and may even result in rescheduling to the
8867	* recompiler/IEM. We must be cautious what we do here regarding committing
8868	* guest-state information into the VMCS assuming we assuredly execute the
8869	* guest in VT-x mode.
8870	*
8871	* If we fall back to the recompiler/IEM after updating the VMCS and clearing
8872	* the common-state (TRPM/forceflags), we must undo those changes so that the
8873	* recompiler/IEM can (and should) use them when it resumes guest execution.
8874	* Otherwise such operations must be done when we can no longer exit to ring-3.
8875	*
8876	* @returns Strict VBox status code (i.e. informational status codes too).
8877	* @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
8878	* have been disabled.
8879	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a
8880	* double-fault into the guest.
8881	* @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
8882	* dispatched directly.
8883	* @retval VINF_* scheduling changes, we have to go back to ring-3.
8884	*
8885	* @param pVM The cross context VM structure.
8886	* @param pVCpu The cross context virtual CPU structure.
8887	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
8888	* out-of-sync. Make sure to update the required fields
8889	* before using them.
8890	* @param pVmxTransient Pointer to the VMX transient structure.
8891	* @param fStepping Set if called from hmR0VmxRunGuestCodeStep(). Makes
8892	* us ignore some of the reasons for returning to
8893	* ring-3, and return VINF_EM_DBG_STEPPED if event
8894	* dispatching took place.
8895	*/
8896	static VBOXSTRICTRC hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, bool fStepping)
8897	{
8898	Assert(VMMRZCallRing3IsEnabled(pVCpu));
8899
8900	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
8901	PGMRZDynMapFlushAutoSet(pVCpu);
8902	#endif
8903
8904	/* Check force flag actions that might require us to go back to ring-3. */
8905	VBOXSTRICTRC rcStrict = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx, fStepping);
8906	if (rcStrict == VINF_SUCCESS)
8907	{ /* FFs doesn't get set all the time. */ }
8908	else
8909	return rcStrict;
8910
8911	#ifndef IEM_VERIFICATION_MODE_FULL
8912	/*
8913	* Setup the virtualized-APIC accesses.
8914	*
8915	* Note! This can cause a longjumps to R3 due to the acquisition of the PGM lock
8916	* in both PGMHandlerPhysicalReset() and IOMMMIOMapMMIOHCPage(), see @bugref{8721}.
8917	*
8918	* This is the reason we do it here and not in hmR0VmxLoadGuestState().
8919	*/
8920	if ( !pVCpu->hm.s.vmx.u64MsrApicBase
8921	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
8922	&& PDMHasApic(pVM))
8923	{
8924	uint64_t const u64MsrApicBase = APICGetBaseMsrNoCheck(pVCpu);
8925	Assert(u64MsrApicBase);
8926	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
8927
8928	RTGCPHYS const GCPhysApicBase = u64MsrApicBase & PAGE_BASE_GC_MASK;
8929
8930	/* Unalias any existing mapping. */
8931	int rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
8932	AssertRCReturn(rc, rc);
8933
8934	/* Map the HC APIC-access page in place of the MMIO page, also updates the shadow page tables if necessary. */
8935	Log4(("hmR0VmxPreRunGuest: VCPU%u: Mapped HC APIC-access page at %#RGp\n", pVCpu->idCpu, GCPhysApicBase));
8936	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
8937	AssertRCReturn(rc, rc);
8938
8939	/* Update the per-VCPU cache of the APIC base MSR. */
8940	pVCpu->hm.s.vmx.u64MsrApicBase = u64MsrApicBase;
8941	}
8942	#endif /* !IEM_VERIFICATION_MODE_FULL */
8943
8944	if (TRPMHasTrap(pVCpu))
8945	hmR0VmxTrpmTrapToPendingEvent(pVCpu);
8946	uint32_t uIntrState = hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
8947
8948	/*
8949	* Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus needs to be done with
8950	* longjmps or interrupts + preemption enabled. Event injection might also result in triple-faulting the VM.
8951	*/
8952	rcStrict = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx, uIntrState, fStepping);
8953	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8954	{ /* likely */ }
8955	else
8956	{
8957	AssertMsg(rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8958	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8959	return rcStrict;
8960	}
8961
8962	/*
8963	* No longjmps to ring-3 from this point on!!!
8964	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
8965	* This also disables flushing of the R0-logger instance (if any).
8966	*/
8967	VMMRZCallRing3Disable(pVCpu);
8968
8969	/*
8970	* Load the guest state bits.
8971	*
8972	* We cannot perform longjmps while loading the guest state because we do not preserve the
8973	* host/guest state (although the VMCS will be preserved) across longjmps which can cause
8974	* CPU migration.
8975	*
8976	* If we are injecting events to a real-on-v86 mode guest, we will have to update
8977	* RIP and some segment registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
8978	* Hence, loading of the guest state needs to be done -after- injection of events.
8979	*/
8980	rcStrict = hmR0VmxLoadGuestStateOptimal(pVM, pVCpu, pMixedCtx);
8981	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8982	{ /* likely */ }
8983	else
8984	{
8985	VMMRZCallRing3Enable(pVCpu);
8986	return rcStrict;
8987	}
8988
8989	/*
8990	* We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.)
8991	* when thread-context hooks aren't used and we've been running with preemption disabled for a while.
8992	*
8993	* We need to check for force-flags that could've possible been altered since we last checked them (e.g.
8994	* by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
8995	*
8996	* We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
8997	* executing guest code.
8998	*/
8999	pVmxTransient->fEFlags = ASMIntDisableFlags();
9000
9001	if ( ( !VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS \| VM_FF_TM_VIRTUAL_SYNC)
9002	&& !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
9003	\|\| ( fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
9004	&& !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER \| VMCPU_FF_PDM_CRITSECT))) )
9005	{
9006	if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
9007	{
9008	pVCpu->hm.s.Event.fPending = false;
9009
9010	/*
9011	* We've injected any pending events. This is really the point of no return (to ring-3).
9012	*
9013	* Note! The caller expects to continue with interrupts & longjmps disabled on successful
9014	* returns from this function, so don't enable them here.
9015	*/
9016	return VINF_SUCCESS;
9017	}
9018
9019	STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
9020	rcStrict = VINF_EM_RAW_INTERRUPT;
9021	}
9022	else
9023	{
9024	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
9025	rcStrict = VINF_EM_RAW_TO_R3;
9026	}
9027
9028	ASMSetFlags(pVmxTransient->fEFlags);
9029	VMMRZCallRing3Enable(pVCpu);
9030
9031	return rcStrict;
9032	}
9033
9034
9035	/**
9036	* Prepares to run guest code in VT-x and we've committed to doing so. This
9037	* means there is no backing out to ring-3 or anywhere else at this
9038	* point.
9039	*
9040	* @param pVM The cross context VM structure.
9041	* @param pVCpu The cross context virtual CPU structure.
9042	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
9043	* out-of-sync. Make sure to update the required fields
9044	* before using them.
9045	* @param pVmxTransient Pointer to the VMX transient structure.
9046	*
9047	* @remarks Called with preemption disabled.
9048	* @remarks No-long-jump zone!!!
9049	*/
9050	static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9051	{
9052	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9053	Assert(VMMR0IsLogFlushDisabled(pVCpu));
9054	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9055
9056	/*
9057	* Indicate start of guest execution and where poking EMT out of guest-context is recognized.
9058	*/
9059	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
9060	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
9061
9062	#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
9063	if (!CPUMIsGuestFPUStateActive(pVCpu))
9064	if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
9065	HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT);
9066	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9067	#endif
9068
9069	if ( pVCpu->hm.s.fPreloadGuestFpu
9070	&& !CPUMIsGuestFPUStateActive(pVCpu))
9071	{
9072	if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
9073	HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT);
9074	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
9075	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9076	}
9077
9078	/*
9079	* Lazy-update of the host MSRs values in the auto-load/store MSR area.
9080	*/
9081	if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
9082	&& pVCpu->hm.s.vmx.cMsrs > 0)
9083	{
9084	hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
9085	}
9086
9087	/*
9088	* Load the host state bits as we may've been preempted (only happens when
9089	* thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
9090	* Note that the 64-on-32 switcher saves the (64-bit) host state into the VMCS and
9091	* if we change the switcher back to 32-bit, we must save the 32-bit host state here.
9092	* See @bugref{8432}.
9093	*/
9094	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
9095	{
9096	int rc = hmR0VmxSaveHostState(pVM, pVCpu);
9097	AssertRC(rc);
9098	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreemptSaveHostState);
9099	}
9100	Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT));
9101
9102	/*
9103	* Load the state shared between host and guest (FPU, debug, lazy MSRs).
9104	*/
9105	if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE))
9106	hmR0VmxLoadSharedState(pVM, pVCpu, pMixedCtx);
9107	AssertMsg(!HMCPU_CF_VALUE(pVCpu), ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
9108
9109	/* Store status of the shared guest-host state at the time of VM-entry. */
9110	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
9111	if (CPUMIsGuestInLongModeEx(pMixedCtx))
9112	{
9113	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
9114	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
9115	}
9116	else
9117	#endif
9118	{
9119	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
9120	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
9121	}
9122	pVmxTransient->fWasGuestFPUStateActive = CPUMIsGuestFPUStateActive(pVCpu);
9123
9124	/*
9125	* Cache the TPR-shadow for checking on every VM-exit if it might have changed.
9126	*/
9127	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
9128	pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR];
9129
9130	PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
9131	RTCPUID idCurrentCpu = pCpu->idCpu;
9132	if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
9133	\|\| idCurrentCpu != pVCpu->hm.s.idLastCpu)
9134	{
9135	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVM, pVCpu);
9136	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
9137	}
9138
9139	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
9140	hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
9141	Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
9142	pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
9143
9144	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
9145
9146	TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
9147	to start executing. */
9148
9149	/*
9150	* Load the TSC_AUX MSR when we are not intercepting RDTSCP.
9151	*/
9152	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
9153	{
9154	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
9155	{
9156	bool fMsrUpdated;
9157	int rc2 = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
9158	AssertRC(rc2);
9159	Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS));
9160
9161	rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, CPUMR0GetGuestTscAux(pVCpu), true /* fUpdateHostMsr */,
9162	&fMsrUpdated);
9163	AssertRC(rc2);
9164	Assert(fMsrUpdated \|\| pVCpu->hm.s.vmx.fUpdatedHostMsrs);
9165
9166	/* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
9167	pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
9168	}
9169	else
9170	{
9171	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
9172	Assert(!pVCpu->hm.s.vmx.cMsrs \|\| pVCpu->hm.s.vmx.fUpdatedHostMsrs);
9173	}
9174	}
9175
9176	#ifdef VBOX_STRICT
9177	hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
9178	hmR0VmxCheckHostEferMsr(pVCpu);
9179	AssertRC(hmR0VmxCheckVmcsCtls(pVCpu));
9180	#endif
9181	#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
9182	uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVM, pVCpu, pMixedCtx);
9183	if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
9184	Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
9185	#endif
9186	}
9187
9188
9189	/**
9190	* Performs some essential restoration of state after running guest code in
9191	* VT-x.
9192	*
9193	* @param pVM The cross context VM structure.
9194	* @param pVCpu The cross context virtual CPU structure.
9195	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
9196	* out-of-sync. Make sure to update the required fields
9197	* before using them.
9198	* @param pVmxTransient Pointer to the VMX transient structure.
9199	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
9200	*
9201	* @remarks Called with interrupts disabled, and returns with interrupts enabled!
9202	*
9203	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
9204	* unconditionally when it is safe to do so.
9205	*/
9206	static void hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
9207	{
9208	NOREF(pVM);
9209
9210	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9211
9212	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
9213	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
9214	HMVMXCPU_GST_RESET_TO(pVCpu, 0); /* Exits/longjmps to ring-3 requires saving the guest state. */
9215	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
9216	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
9217	pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
9218
9219	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
9220	TMCpuTickSetLastSeen(pVCpu, ASMReadTSC() + pVCpu->hm.s.vmx.u64TSCOffset);
9221
9222	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
9223	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
9224	Assert(!ASMIntAreEnabled());
9225	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
9226
9227	#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
9228	if (CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVM, pVCpu))
9229	{
9230	hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9231	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9232	}
9233	#endif
9234
9235	#if HC_ARCH_BITS == 64
9236	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
9237	#endif
9238	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
9239	/* The 64-on-32 switcher maintains uVmcsState on its own and we need to leave it alone here. */
9240	if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
9241	pVCpu->hm.s.vmx.uVmcsState \|= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
9242	#else
9243	pVCpu->hm.s.vmx.uVmcsState \|= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
9244	#endif
9245	#ifdef VBOX_STRICT
9246	hmR0VmxCheckHostEferMsr(pVCpu); /* Verify that VMRUN/VMLAUNCH didn't modify host EFER. */
9247	#endif
9248	ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
9249
9250	/* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
9251	uint32_t uExitReason;
9252	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
9253	rc \|= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
9254	AssertRC(rc);
9255	pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
9256	pVmxTransient->fVMEntryFailed = VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uEntryIntInfo);
9257
9258	if (rcVMRun == VINF_SUCCESS)
9259	{
9260	/*
9261	* Update the VM-exit history array here even if the VM-entry failed due to:
9262	* - Invalid guest state.
9263	* - MSR loading.
9264	* - Machine-check event.
9265	*
9266	* In any of the above cases we will still have a "valid" VM-exit reason
9267	* despite @a fVMEntryFailed being false.
9268	*
9269	* See Intel spec. 26.7 "VM-Entry failures during or after loading guest state".
9270	*/
9271	HMCPU_EXIT_HISTORY_ADD(pVCpu, pVmxTransient->uExitReason);
9272
9273	if (!pVmxTransient->fVMEntryFailed)
9274	{
9275	/** @todo We can optimize this by only syncing with our force-flags when
9276	* really needed and keeping the VMCS state as it is for most
9277	* VM-exits. */
9278	/* Update the guest interruptibility-state from the VMCS. */
9279	hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
9280
9281	/*
9282	* Allow longjmps to ring-3 -after- saving the guest-interruptibility state
9283	* as it's not part of hmR0VmxSaveGuestState() and thus would trigger an assertion
9284	* on the longjmp path to ring-3 while saving the (rest of) the guest state,
9285	* see @bugref{6208#c63}.
9286	*/
9287	VMMRZCallRing3Enable(pVCpu);
9288
9289	#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) \|\| defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
9290	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9291	AssertRC(rc);
9292	#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
9293	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9294	AssertRC(rc);
9295	#endif
9296
9297	/*
9298	* Sync the TPR shadow with our APIC state.
9299	*/
9300	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
9301	&& pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR])
9302	{
9303	rc = APICSetTpr(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR]);
9304	AssertRC(rc);
9305	HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
9306	}
9307
9308	return;
9309	}
9310	}
9311	else
9312	{
9313	Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
9314	pVmxTransient->fVMEntryFailed));
9315	}
9316
9317	VMMRZCallRing3Enable(pVCpu);
9318	}
9319
9320
9321	/**
9322	* Runs the guest code using VT-x the normal way.
9323	*
9324	* @returns VBox status code.
9325	* @param pVM The cross context VM structure.
9326	* @param pVCpu The cross context virtual CPU structure.
9327	* @param pCtx Pointer to the guest-CPU context.
9328	*
9329	* @note Mostly the same as hmR0VmxRunGuestCodeStep().
9330	*/
9331	static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
9332	{
9333	VMXTRANSIENT VmxTransient;
9334	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
9335	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
9336	uint32_t cLoops = 0;
9337
9338	for (;; cLoops++)
9339	{
9340	Assert(!HMR0SuspendPending());
9341	HMVMX_ASSERT_CPU_SAFE();
9342
9343	/* Preparatory work for running guest code, this may force us to return
9344	to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
9345	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
9346	rcStrict = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, false /* fStepping */);
9347	if (rcStrict != VINF_SUCCESS)
9348	break;
9349
9350	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
9351	int rcRun = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
9352	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
9353
9354	/* Restore any residual host-state and save any bits shared between host
9355	and guest into the guest-CPU state. Re-enables interrupts! */
9356	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rcRun);
9357
9358	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
9359	if (RT_SUCCESS(rcRun))
9360	{ /* very likely */ }
9361	else
9362	{
9363	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
9364	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rcRun, pCtx, &VmxTransient);
9365	return rcRun;
9366	}
9367
9368	/* Profile the VM-exit. */
9369	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
9370	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
9371	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
9372	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
9373	HMVMX_START_EXIT_DISPATCH_PROF();
9374
9375	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
9376
9377	/* Handle the VM-exit. */
9378	#ifdef HMVMX_USE_FUNCTION_TABLE
9379	rcStrict = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
9380	#else
9381	rcStrict = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
9382	#endif
9383	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
9384	if (rcStrict == VINF_SUCCESS)
9385	{
9386	if (cLoops <= pVM->hm.s.cMaxResumeLoops)
9387	continue; /* likely */
9388	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
9389	rcStrict = VINF_EM_RAW_INTERRUPT;
9390	}
9391	break;
9392	}
9393
9394	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
9395	return rcStrict;
9396	}
9397
9398
9399
9400	/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
9401	* probes.
9402	*
9403	* The following few functions and associated structure contains the bloat
9404	* necessary for providing detailed debug events and dtrace probes as well as
9405	* reliable host side single stepping. This works on the principle of
9406	* "subclassing" the normal execution loop and workers. We replace the loop
9407	* method completely and override selected helpers to add necessary adjustments
9408	* to their core operation.
9409	*
9410	* The goal is to keep the "parent" code lean and mean, so as not to sacrifice
9411	* any performance for debug and analysis features.
9412	*
9413	* @{
9414	*/
9415
9416	/**
9417	* Transient per-VCPU debug state of VMCS and related info. we save/restore in
9418	* the debug run loop.
9419	*/
9420	typedef struct VMXRUNDBGSTATE
9421	{
9422	/** The RIP we started executing at. This is for detecting that we stepped. */
9423	uint64_t uRipStart;
9424	/** The CS we started executing with. */
9425	uint16_t uCsStart;
9426
9427	/** Whether we've actually modified the 1st execution control field. */
9428	bool fModifiedProcCtls : 1;
9429	/** Whether we've actually modified the 2nd execution control field. */
9430	bool fModifiedProcCtls2 : 1;
9431	/** Whether we've actually modified the exception bitmap. */
9432	bool fModifiedXcptBitmap : 1;
9433
9434	/** We desire the modified the CR0 mask to be cleared. */
9435	bool fClearCr0Mask : 1;
9436	/** We desire the modified the CR4 mask to be cleared. */
9437	bool fClearCr4Mask : 1;
9438	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
9439	uint32_t fCpe1Extra;
9440	/** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
9441	uint32_t fCpe1Unwanted;
9442	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
9443	uint32_t fCpe2Extra;
9444	/** Extra stuff we need in VMX_VMCS32_CTRL_EXCEPTION_BITMAP. */
9445	uint32_t bmXcptExtra;
9446	/** The sequence number of the Dtrace provider settings the state was
9447	* configured against. */
9448	uint32_t uDtraceSettingsSeqNo;
9449	/** VM-exits to check (one bit per VM-exit). */
9450	uint32_t bmExitsToCheck[3];
9451
9452	/** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
9453	uint32_t fProcCtlsInitial;
9454	/** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
9455	uint32_t fProcCtls2Initial;
9456	/** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
9457	uint32_t bmXcptInitial;
9458	} VMXRUNDBGSTATE;
9459	AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
9460	typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
9461
9462
9463	/**
9464	* Initializes the VMXRUNDBGSTATE structure.
9465	*
9466	* @param pVCpu The cross context virtual CPU structure of the
9467	* calling EMT.
9468	* @param pCtx The CPU register context to go with @a pVCpu.
9469	* @param pDbgState The structure to initialize.
9470	*/
9471	DECLINLINE(void) hmR0VmxRunDebugStateInit(PVMCPU pVCpu, PCCPUMCTX pCtx, PVMXRUNDBGSTATE pDbgState)
9472	{
9473	pDbgState->uRipStart = pCtx->rip;
9474	pDbgState->uCsStart = pCtx->cs.Sel;
9475
9476	pDbgState->fModifiedProcCtls = false;
9477	pDbgState->fModifiedProcCtls2 = false;
9478	pDbgState->fModifiedXcptBitmap = false;
9479	pDbgState->fClearCr0Mask = false;
9480	pDbgState->fClearCr4Mask = false;
9481	pDbgState->fCpe1Extra = 0;
9482	pDbgState->fCpe1Unwanted = 0;
9483	pDbgState->fCpe2Extra = 0;
9484	pDbgState->bmXcptExtra = 0;
9485	pDbgState->fProcCtlsInitial = pVCpu->hm.s.vmx.u32ProcCtls;
9486	pDbgState->fProcCtls2Initial = pVCpu->hm.s.vmx.u32ProcCtls2;
9487	pDbgState->bmXcptInitial = pVCpu->hm.s.vmx.u32XcptBitmap;
9488	}
9489
9490
9491	/**
9492	* Updates the VMSC fields with changes requested by @a pDbgState.
9493	*
9494	* This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
9495	* immediately before executing guest code, i.e. when interrupts are disabled.
9496	* We don't check status codes here as we cannot easily assert or return in the
9497	* latter case.
9498	*
9499	* @param pVCpu The cross context virtual CPU structure.
9500	* @param pDbgState The debug state.
9501	*/
9502	DECLINLINE(void) hmR0VmxPreRunGuestDebugStateApply(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState)
9503	{
9504	/*
9505	* Ensure desired flags in VMCS control fields are set.
9506	* (Ignoring write failure here, as we're committed and it's just debug extras.)
9507	*
9508	* Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
9509	* there should be no stale data in pCtx at this point.
9510	*/
9511	if ( (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
9512	\|\| (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Unwanted))
9513	{
9514	pVCpu->hm.s.vmx.u32ProcCtls \|= pDbgState->fCpe1Extra;
9515	pVCpu->hm.s.vmx.u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
9516	VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
9517	Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls));
9518	pDbgState->fModifiedProcCtls = true;
9519	}
9520
9521	if ((pVCpu->hm.s.vmx.u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
9522	{
9523	pVCpu->hm.s.vmx.u32ProcCtls2 \|= pDbgState->fCpe2Extra;
9524	VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pVCpu->hm.s.vmx.u32ProcCtls2);
9525	Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls2));
9526	pDbgState->fModifiedProcCtls2 = true;
9527	}
9528
9529	if ((pVCpu->hm.s.vmx.u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
9530	{
9531	pVCpu->hm.s.vmx.u32XcptBitmap \|= pDbgState->bmXcptExtra;
9532	VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
9533	Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVCpu->hm.s.vmx.u32XcptBitmap));
9534	pDbgState->fModifiedXcptBitmap = true;
9535	}
9536
9537	if (pDbgState->fClearCr0Mask && pVCpu->hm.s.vmx.u32CR0Mask != 0)
9538	{
9539	pVCpu->hm.s.vmx.u32CR0Mask = 0;
9540	VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, 0);
9541	Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR0_MASK: 0\n"));
9542	}
9543
9544	if (pDbgState->fClearCr4Mask && pVCpu->hm.s.vmx.u32CR4Mask != 0)
9545	{
9546	pVCpu->hm.s.vmx.u32CR4Mask = 0;
9547	VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, 0);
9548	Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR4_MASK: 0\n"));
9549	}
9550	}
9551
9552
9553	DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugStateRevert(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState, VBOXSTRICTRC rcStrict)
9554	{
9555	/*
9556	* Restore VM-exit control settings as we may not reenter this function the
9557	* next time around.
9558	*/
9559	/* We reload the initial value, trigger what we can of recalculations the
9560	next time around. From the looks of things, that's all that's required atm. */
9561	if (pDbgState->fModifiedProcCtls)
9562	{
9563	if (!(pDbgState->fProcCtlsInitial & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
9564	pDbgState->fProcCtlsInitial \|= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
9565	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
9566	AssertRCReturn(rc2, rc2);
9567	pVCpu->hm.s.vmx.u32ProcCtls = pDbgState->fProcCtlsInitial;
9568	}
9569
9570	/* We're currently the only ones messing with this one, so just restore the
9571	cached value and reload the field. */
9572	if ( pDbgState->fModifiedProcCtls2
9573	&& pVCpu->hm.s.vmx.u32ProcCtls2 != pDbgState->fProcCtls2Initial)
9574	{
9575	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
9576	AssertRCReturn(rc2, rc2);
9577	pVCpu->hm.s.vmx.u32ProcCtls2 = pDbgState->fProcCtls2Initial;
9578	}
9579
9580	/* If we've modified the exception bitmap, we restore it and trigger
9581	reloading and partial recalculation the next time around. */
9582	if (pDbgState->fModifiedXcptBitmap)
9583	pVCpu->hm.s.vmx.u32XcptBitmap = pDbgState->bmXcptInitial;
9584
9585	return rcStrict;
9586	}
9587
9588
9589	/**
9590	* Configures VM-exit controls for current DBGF and DTrace settings.
9591	*
9592	* This updates @a pDbgState and the VMCS execution control fields to reflect
9593	* the necessary VM-exits demanded by DBGF and DTrace.
9594	*
9595	* @param pVM The cross context VM structure.
9596	* @param pVCpu The cross context virtual CPU structure.
9597	* @param pCtx Pointer to the guest-CPU context.
9598	* @param pDbgState The debug state.
9599	* @param pVmxTransient Pointer to the VMX transient structure. May update
9600	* fUpdateTscOffsettingAndPreemptTimer.
9601	*/
9602	static void hmR0VmxPreRunGuestDebugStateUpdate(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx,
9603	PVMXRUNDBGSTATE pDbgState, PVMXTRANSIENT pVmxTransient)
9604	{
9605	/*
9606	* Take down the dtrace serial number so we can spot changes.
9607	*/
9608	pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
9609	ASMCompilerBarrier();
9610
9611	/*
9612	* We'll rebuild most of the middle block of data members (holding the
9613	* current settings) as we go along here, so start by clearing it all.
9614	*/
9615	pDbgState->bmXcptExtra = 0;
9616	pDbgState->fCpe1Extra = 0;
9617	pDbgState->fCpe1Unwanted = 0;
9618	pDbgState->fCpe2Extra = 0;
9619	for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
9620	pDbgState->bmExitsToCheck[i] = 0;
9621
9622	/*
9623	* Software interrupts (INT XXh) - no idea how to trigger these...
9624	*/
9625	if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
9626	\|\| VBOXVMM_INT_SOFTWARE_ENABLED())
9627	{
9628	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9629	}
9630
9631	/*
9632	* INT3 breakpoints - triggered by #BP exceptions.
9633	*/
9634	if (pVM->dbgf.ro.cEnabledInt3Breakpoints > 0)
9635	pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
9636
9637	/*
9638	* Exception bitmap and XCPT events+probes.
9639	*/
9640	for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
9641	if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
9642	pDbgState->bmXcptExtra \|= RT_BIT_32(iXcpt);
9643
9644	if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DE);
9645	if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DB);
9646	if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
9647	if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_OF);
9648	if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BR);
9649	if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_UD);
9650	if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NM);
9651	if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DF);
9652	if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_TS);
9653	if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NP);
9654	if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SS);
9655	if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_GP);
9656	if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_PF);
9657	if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_MF);
9658	if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_AC);
9659	if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_XF);
9660	if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_VE);
9661	if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SX);
9662
9663	if (pDbgState->bmXcptExtra)
9664	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9665
9666	/*
9667	* Process events and probes for VM-exits, making sure we get the wanted VM-exits.
9668	*
9669	* Note! This is the reverse of waft hmR0VmxHandleExitDtraceEvents does.
9670	* So, when adding/changing/removing please don't forget to update it.
9671	*
9672	* Some of the macros are picking up local variables to save horizontal space,
9673	* (being able to see it in a table is the lesser evil here).
9674	*/
9675	#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
9676	( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
9677	\|\| RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
9678	#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
9679	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9680	{ AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9681	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9682	} else do { } while (0)
9683	#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
9684	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9685	{ \
9686	(pDbgState)->fCpe1Extra \|= (a_fCtrlProcExec); \
9687	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9688	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9689	} else do { } while (0)
9690	#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
9691	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9692	{ \
9693	(pDbgState)->fCpe1Unwanted \|= (a_fUnwantedCtrlProcExec); \
9694	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9695	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9696	} else do { } while (0)
9697	#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
9698	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9699	{ \
9700	(pDbgState)->fCpe2Extra \|= (a_fCtrlProcExec2); \
9701	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9702	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9703	} else do { } while (0)
9704
9705	SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
9706	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
9707	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
9708	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
9709	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
9710
9711	SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
9712	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
9713	SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
9714	SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
9715	SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT); /* paranoia */
9716	SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
9717	SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
9718	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
9719	SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9720	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
9721	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT);
9722	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
9723	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9724	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
9725	SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
9726	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
9727	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
9728	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
9729	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
9730	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
9731	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
9732	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
9733	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
9734	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
9735	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
9736	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
9737	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
9738	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
9739	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
9740	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
9741	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
9742	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
9743	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
9744	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
9745	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
9746	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
9747
9748	if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
9749	\|\| IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9750	{
9751	int rc2 = hmR0VmxSaveGuestCR0(pVCpu, pCtx);
9752	rc2 \|= hmR0VmxSaveGuestCR4(pVCpu, pCtx);
9753	rc2 \|= hmR0VmxSaveGuestApicState(pVCpu, pCtx);
9754	AssertRC(rc2);
9755
9756	#if 0 /** @todo fix me */
9757	pDbgState->fClearCr0Mask = true;
9758	pDbgState->fClearCr4Mask = true;
9759	#endif
9760	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
9761	pDbgState->fCpe1Extra \|= VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT \| VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT;
9762	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9763	pDbgState->fCpe1Extra \|= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT \| VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT;
9764	pDbgState->fCpe1Unwanted \|= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* risky? */
9765	/* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
9766	require clearing here and in the loop if we start using it. */
9767	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
9768	}
9769	else
9770	{
9771	if (pDbgState->fClearCr0Mask)
9772	{
9773	pDbgState->fClearCr0Mask = false;
9774	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9775	}
9776	if (pDbgState->fClearCr4Mask)
9777	{
9778	pDbgState->fClearCr4Mask = false;
9779	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
9780	}
9781	}
9782	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
9783	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
9784
9785	if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
9786	\|\| IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
9787	{
9788	/** @todo later, need to fix handler as it assumes this won't usually happen. */
9789	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
9790	}
9791	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
9792	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
9793
9794	SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS); /* risky clearing this? */
9795	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
9796	SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS);
9797	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
9798	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT); /* paranoia */
9799	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
9800	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT); /* paranoia */
9801	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
9802	#if 0 /** @todo too slow, fix handler. */
9803	SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT);
9804	#endif
9805	SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
9806
9807	if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
9808	\|\| IS_EITHER_ENABLED(pVM, INSTR_SIDT)
9809	\|\| IS_EITHER_ENABLED(pVM, INSTR_LGDT)
9810	\|\| IS_EITHER_ENABLED(pVM, INSTR_LIDT))
9811	{
9812	pDbgState->fCpe2Extra \|= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9813	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XDTR_ACCESS);
9814	}
9815	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_XDTR_ACCESS);
9816	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_XDTR_ACCESS);
9817	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_XDTR_ACCESS);
9818	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_XDTR_ACCESS);
9819
9820	if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
9821	\|\| IS_EITHER_ENABLED(pVM, INSTR_STR)
9822	\|\| IS_EITHER_ENABLED(pVM, INSTR_LLDT)
9823	\|\| IS_EITHER_ENABLED(pVM, INSTR_LTR))
9824	{
9825	pDbgState->fCpe2Extra \|= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9826	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_TR_ACCESS);
9827	}
9828	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_TR_ACCESS);
9829	SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_TR_ACCESS);
9830	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_TR_ACCESS);
9831	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_TR_ACCESS);
9832
9833	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
9834	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
9835	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9836	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
9837	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
9838	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
9839	SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT);
9840	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
9841	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
9842	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
9843	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT);
9844	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
9845	SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9846	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
9847	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
9848	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
9849	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_VMCS_CTRL_PROC_EXEC2_RDSEED_EXIT);
9850	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
9851	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
9852	SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
9853	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
9854	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
9855
9856	#undef IS_EITHER_ENABLED
9857	#undef SET_ONLY_XBM_IF_EITHER_EN
9858	#undef SET_CPE1_XBM_IF_EITHER_EN
9859	#undef SET_CPEU_XBM_IF_EITHER_EN
9860	#undef SET_CPE2_XBM_IF_EITHER_EN
9861
9862	/*
9863	* Sanitize the control stuff.
9864	*/
9865	pDbgState->fCpe2Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1;
9866	if (pDbgState->fCpe2Extra)
9867	pDbgState->fCpe1Extra \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
9868	pDbgState->fCpe1Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1;
9869	pDbgState->fCpe1Unwanted &= ~pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0;
9870	if (pVCpu->hm.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
9871	{
9872	pVCpu->hm.s.fDebugWantRdTscExit ^= true;
9873	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9874	}
9875
9876	Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
9877	pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
9878	pDbgState->fClearCr0Mask ? " clr-cr0" : "",
9879	pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
9880	}
9881
9882
9883	/**
9884	* Fires off DBGF events and dtrace probes for a VM-exit, when it's
9885	* appropriate.
9886	*
9887	* The caller has checked the VM-exit against the
9888	* VMXRUNDBGSTATE::bmExitsToCheck bitmap. The caller has checked for NMIs
9889	* already, so we don't have to do that either.
9890	*
9891	* @returns Strict VBox status code (i.e. informational status codes too).
9892	* @param pVM The cross context VM structure.
9893	* @param pVCpu The cross context virtual CPU structure.
9894	* @param pMixedCtx Pointer to the guest-CPU context.
9895	* @param pVmxTransient Pointer to the VMX-transient structure.
9896	* @param uExitReason The VM-exit reason.
9897	*
9898	* @remarks The name of this function is displayed by dtrace, so keep it short
9899	* and to the point. No longer than 33 chars long, please.
9900	*/
9901	static VBOXSTRICTRC hmR0VmxHandleExitDtraceEvents(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx,
9902	PVMXTRANSIENT pVmxTransient, uint32_t uExitReason)
9903	{
9904	/*
9905	* Translate the event into a DBGF event (enmEvent + uEventArg) and at the
9906	* same time check whether any corresponding Dtrace event is enabled (fDtrace).
9907	*
9908	* Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
9909	* does. Must add/change/remove both places. Same ordering, please.
9910	*
9911	* Added/removed events must also be reflected in the next section
9912	* where we dispatch dtrace events.
9913	*/
9914	bool fDtrace1 = false;
9915	bool fDtrace2 = false;
9916	DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
9917	DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
9918	uint32_t uEventArg = 0;
9919	#define SET_EXIT(a_EventSubName) \
9920	do { \
9921	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9922	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9923	} while (0)
9924	#define SET_BOTH(a_EventSubName) \
9925	do { \
9926	enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
9927	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9928	fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
9929	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9930	} while (0)
9931	switch (uExitReason)
9932	{
9933	case VMX_EXIT_MTF:
9934	return hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient);
9935
9936	case VMX_EXIT_XCPT_OR_NMI:
9937	{
9938	uint8_t const idxVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
9939	switch (VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo))
9940	{
9941	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
9942	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT:
9943	case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT:
9944	if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
9945	{
9946	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uExitIntInfo))
9947	{
9948	hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
9949	uEventArg = pVmxTransient->uExitIntErrorCode;
9950	}
9951	enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
9952	switch (enmEvent1)
9953	{
9954	case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
9955	case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
9956	case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
9957	case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
9958	case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
9959	case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
9960	case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
9961	case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
9962	case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
9963	case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
9964	case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
9965	case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
9966	case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
9967	case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
9968	case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
9969	case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
9970	case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
9971	case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
9972	default: break;
9973	}
9974	}
9975	else
9976	AssertFailed();
9977	break;
9978
9979	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT:
9980	uEventArg = idxVector;
9981	enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
9982	fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
9983	break;
9984	}
9985	break;
9986	}
9987
9988	case VMX_EXIT_TRIPLE_FAULT:
9989	enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
9990	//fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
9991	break;
9992	case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
9993	case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
9994	case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
9995	case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
9996	case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
9997
9998	/* Instruction specific VM-exits: */
9999	case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
10000	case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
10001	case VMX_EXIT_HLT: SET_BOTH(HALT); break;
10002	case VMX_EXIT_INVD: SET_BOTH(INVD); break;
10003	case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
10004	case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
10005	case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
10006	case VMX_EXIT_RSM: SET_BOTH(RSM); break;
10007	case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
10008	case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
10009	case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
10010	case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
10011	case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
10012	case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
10013	case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
10014	case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
10015	case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
10016	case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
10017	case VMX_EXIT_MOV_CRX:
10018	hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10019	/** @todo r=bird: I feel these macros aren't very descriptive and needs to be at least 30 chars longer! ;-)
10020	* Sensible abbreviations strongly recommended here because even with 130 columns this stuff get too wide! */
10021	if ( VMX_EXIT_QUALIFICATION_CRX_ACCESS(pVmxTransient->uExitQualification)
10022	== VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ)
10023	SET_BOTH(CRX_READ);
10024	else
10025	SET_BOTH(CRX_WRITE);
10026	uEventArg = VMX_EXIT_QUALIFICATION_CRX_REGISTER(pVmxTransient->uExitQualification);
10027	break;
10028	case VMX_EXIT_MOV_DRX:
10029	hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10030	if ( VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification)
10031	== VMX_EXIT_QUALIFICATION_DRX_DIRECTION_READ)
10032	SET_BOTH(DRX_READ);
10033	else
10034	SET_BOTH(DRX_WRITE);
10035	uEventArg = VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification);
10036	break;
10037	case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
10038	case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
10039	case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
10040	case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
10041	case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
10042	case VMX_EXIT_XDTR_ACCESS:
10043	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
10044	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_XDTR_INSINFO_INSTR_ID))
10045	{
10046	case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
10047	case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
10048	case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
10049	case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
10050	}
10051	break;
10052
10053	case VMX_EXIT_TR_ACCESS:
10054	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
10055	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_YYTR_INSINFO_INSTR_ID))
10056	{
10057	case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
10058	case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
10059	case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
10060	case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
10061	}
10062	break;
10063
10064	case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
10065	case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
10066	case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
10067	case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
10068	case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
10069	case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
10070	case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
10071	case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
10072	case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
10073	case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
10074	case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
10075
10076	/* Events that aren't relevant at this point. */
10077	case VMX_EXIT_EXT_INT:
10078	case VMX_EXIT_INT_WINDOW:
10079	case VMX_EXIT_NMI_WINDOW:
10080	case VMX_EXIT_TPR_BELOW_THRESHOLD:
10081	case VMX_EXIT_PREEMPT_TIMER:
10082	case VMX_EXIT_IO_INSTR:
10083	break;
10084
10085	/* Errors and unexpected events. */
10086	case VMX_EXIT_INIT_SIGNAL:
10087	case VMX_EXIT_SIPI:
10088	case VMX_EXIT_IO_SMI:
10089	case VMX_EXIT_SMI:
10090	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
10091	case VMX_EXIT_ERR_MSR_LOAD:
10092	case VMX_EXIT_ERR_MACHINE_CHECK:
10093	break;
10094
10095	default:
10096	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
10097	break;
10098	}
10099	#undef SET_BOTH
10100	#undef SET_EXIT
10101
10102	/*
10103	* Dtrace tracepoints go first. We do them here at once so we don't
10104	* have to copy the guest state saving and stuff a few dozen times.
10105	* Down side is that we've got to repeat the switch, though this time
10106	* we use enmEvent since the probes are a subset of what DBGF does.
10107	*/
10108	if (fDtrace1 \|\| fDtrace2)
10109	{
10110	hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10111	hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10112	switch (enmEvent1)
10113	{
10114	/** @todo consider which extra parameters would be helpful for each probe. */
10115	case DBGFEVENT_END: break;
10116	case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pMixedCtx); break;
10117	case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pMixedCtx, pMixedCtx->dr[6]); break;
10118	case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pMixedCtx); break;
10119	case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pMixedCtx); break;
10120	case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pMixedCtx); break;
10121	case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pMixedCtx); break;
10122	case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pMixedCtx); break;
10123	case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pMixedCtx); break;
10124	case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pMixedCtx, uEventArg); break;
10125	case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pMixedCtx, uEventArg); break;
10126	case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pMixedCtx, uEventArg); break;
10127	case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pMixedCtx, uEventArg); break;
10128	case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pMixedCtx, uEventArg, pMixedCtx->cr2); break;
10129	case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pMixedCtx); break;
10130	case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pMixedCtx); break;
10131	case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pMixedCtx); break;
10132	case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pMixedCtx); break;
10133	case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pMixedCtx, uEventArg); break;
10134	case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10135	case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pMixedCtx, pMixedCtx->eax, pMixedCtx->ecx); break;
10136	case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pMixedCtx); break;
10137	case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pMixedCtx); break;
10138	case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pMixedCtx); break;
10139	case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pMixedCtx); break;
10140	case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pMixedCtx); break;
10141	case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pMixedCtx); break;
10142	case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pMixedCtx); break;
10143	case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10144	case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10145	case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10146	case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10147	case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pMixedCtx, pMixedCtx->ecx); break;
10148	case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pMixedCtx, pMixedCtx->ecx,
10149	RT_MAKE_U64(pMixedCtx->eax, pMixedCtx->edx)); break;
10150	case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pMixedCtx); break;
10151	case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pMixedCtx); break;
10152	case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pMixedCtx); break;
10153	case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pMixedCtx); break;
10154	case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pMixedCtx); break;
10155	case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pMixedCtx); break;
10156	case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pMixedCtx); break;
10157	case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pMixedCtx); break;
10158	case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pMixedCtx); break;
10159	case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pMixedCtx); break;
10160	case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pMixedCtx); break;
10161	case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pMixedCtx); break;
10162	case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pMixedCtx); break;
10163	case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pMixedCtx); break;
10164	case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pMixedCtx); break;
10165	case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pMixedCtx); break;
10166	case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pMixedCtx); break;
10167	case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pMixedCtx); break;
10168	case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pMixedCtx); break;
10169	case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pMixedCtx); break;
10170	case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pMixedCtx); break;
10171	case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pMixedCtx); break;
10172	case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pMixedCtx); break;
10173	case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pMixedCtx); break;
10174	case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pMixedCtx); break;
10175	case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pMixedCtx); break;
10176	case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pMixedCtx); break;
10177	case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pMixedCtx); break;
10178	case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pMixedCtx); break;
10179	case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pMixedCtx); break;
10180	case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pMixedCtx); break;
10181	case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pMixedCtx); break;
10182	default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
10183	}
10184	switch (enmEvent2)
10185	{
10186	/** @todo consider which extra parameters would be helpful for each probe. */
10187	case DBGFEVENT_END: break;
10188	case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pMixedCtx); break;
10189	case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pMixedCtx, pMixedCtx->eax, pMixedCtx->ecx); break;
10190	case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pMixedCtx); break;
10191	case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pMixedCtx); break;
10192	case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pMixedCtx); break;
10193	case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pMixedCtx); break;
10194	case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pMixedCtx); break;
10195	case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pMixedCtx); break;
10196	case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pMixedCtx); break;
10197	case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10198	case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10199	case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10200	case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10201	case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pMixedCtx, pMixedCtx->ecx); break;
10202	case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pMixedCtx, pMixedCtx->ecx,
10203	RT_MAKE_U64(pMixedCtx->eax, pMixedCtx->edx)); break;
10204	case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pMixedCtx); break;
10205	case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pMixedCtx); break;
10206	case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pMixedCtx); break;
10207	case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pMixedCtx); break;
10208	case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pMixedCtx); break;
10209	case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pMixedCtx); break;
10210	case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pMixedCtx); break;
10211	case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pMixedCtx); break;
10212	case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pMixedCtx); break;
10213	case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pMixedCtx); break;
10214	case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pMixedCtx); break;
10215	case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pMixedCtx); break;
10216	case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pMixedCtx); break;
10217	case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pMixedCtx); break;
10218	case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pMixedCtx); break;
10219	case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pMixedCtx); break;
10220	case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pMixedCtx); break;
10221	case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pMixedCtx); break;
10222	case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pMixedCtx); break;
10223	case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pMixedCtx); break;
10224	case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pMixedCtx); break;
10225	case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pMixedCtx); break;
10226	case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pMixedCtx); break;
10227	case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pMixedCtx); break;
10228	case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pMixedCtx); break;
10229	case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pMixedCtx); break;
10230	case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pMixedCtx); break;
10231	case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pMixedCtx); break;
10232	case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pMixedCtx); break;
10233	case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pMixedCtx); break;
10234	case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pMixedCtx); break;
10235	case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pMixedCtx); break;
10236	case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pMixedCtx); break;
10237	case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pMixedCtx); break;
10238	case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pMixedCtx); break;
10239	case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pMixedCtx); break;
10240	default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
10241	}
10242	}
10243
10244	/*
10245	* Fire of the DBGF event, if enabled (our check here is just a quick one,
10246	* the DBGF call will do a full check).
10247	*
10248	* Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
10249	* Note! If we have to events, we prioritize the first, i.e. the instruction
10250	* one, in order to avoid event nesting.
10251	*/
10252	if ( enmEvent1 != DBGFEVENT_END
10253	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
10254	{
10255	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent1, uEventArg, DBGFEVENTCTX_HM);
10256	if (rcStrict != VINF_SUCCESS)
10257	return rcStrict;
10258	}
10259	else if ( enmEvent2 != DBGFEVENT_END
10260	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
10261	{
10262	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent2, uEventArg, DBGFEVENTCTX_HM);
10263	if (rcStrict != VINF_SUCCESS)
10264	return rcStrict;
10265	}
10266
10267	return VINF_SUCCESS;
10268	}
10269
10270
10271	/**
10272	* Single-stepping VM-exit filtering.
10273	*
10274	* This is preprocessing the VM-exits and deciding whether we've gotten far
10275	* enough to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit
10276	* handling is performed.
10277	*
10278	* @returns Strict VBox status code (i.e. informational status codes too).
10279	* @param pVM The cross context VM structure.
10280	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
10281	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
10282	* out-of-sync. Make sure to update the required
10283	* fields before using them.
10284	* @param pVmxTransient Pointer to the VMX-transient structure.
10285	* @param uExitReason The VM-exit reason.
10286	* @param pDbgState The debug state.
10287	*/
10288	DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugHandleExit(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient,
10289	uint32_t uExitReason, PVMXRUNDBGSTATE pDbgState)
10290	{
10291	/*
10292	* Expensive (saves context) generic dtrace VM-exit probe.
10293	*/
10294	if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
10295	{ /* more likely */ }
10296	else
10297	{
10298	hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10299	hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10300	VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, pMixedCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQualification);
10301	}
10302
10303	/*
10304	* Check for host NMI, just to get that out of the way.
10305	*/
10306	if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
10307	{ /* normally likely */ }
10308	else
10309	{
10310	int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
10311	AssertRCReturn(rc2, rc2);
10312	uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
10313	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
10314	return hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient);
10315	}
10316
10317	/*
10318	* Check for single stepping event if we're stepping.
10319	*/
10320	if (pVCpu->hm.s.fSingleInstruction)
10321	{
10322	switch (uExitReason)
10323	{
10324	case VMX_EXIT_MTF:
10325	return hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient);
10326
10327	/* Various events: */
10328	case VMX_EXIT_XCPT_OR_NMI:
10329	case VMX_EXIT_EXT_INT:
10330	case VMX_EXIT_TRIPLE_FAULT:
10331	case VMX_EXIT_INT_WINDOW:
10332	case VMX_EXIT_NMI_WINDOW:
10333	case VMX_EXIT_TASK_SWITCH:
10334	case VMX_EXIT_TPR_BELOW_THRESHOLD:
10335	case VMX_EXIT_APIC_ACCESS:
10336	case VMX_EXIT_EPT_VIOLATION:
10337	case VMX_EXIT_EPT_MISCONFIG:
10338	case VMX_EXIT_PREEMPT_TIMER:
10339
10340	/* Instruction specific VM-exits: */
10341	case VMX_EXIT_CPUID:
10342	case VMX_EXIT_GETSEC:
10343	case VMX_EXIT_HLT:
10344	case VMX_EXIT_INVD:
10345	case VMX_EXIT_INVLPG:
10346	case VMX_EXIT_RDPMC:
10347	case VMX_EXIT_RDTSC:
10348	case VMX_EXIT_RSM:
10349	case VMX_EXIT_VMCALL:
10350	case VMX_EXIT_VMCLEAR:
10351	case VMX_EXIT_VMLAUNCH:
10352	case VMX_EXIT_VMPTRLD:
10353	case VMX_EXIT_VMPTRST:
10354	case VMX_EXIT_VMREAD:
10355	case VMX_EXIT_VMRESUME:
10356	case VMX_EXIT_VMWRITE:
10357	case VMX_EXIT_VMXOFF:
10358	case VMX_EXIT_VMXON:
10359	case VMX_EXIT_MOV_CRX:
10360	case VMX_EXIT_MOV_DRX:
10361	case VMX_EXIT_IO_INSTR:
10362	case VMX_EXIT_RDMSR:
10363	case VMX_EXIT_WRMSR:
10364	case VMX_EXIT_MWAIT:
10365	case VMX_EXIT_MONITOR:
10366	case VMX_EXIT_PAUSE:
10367	case VMX_EXIT_XDTR_ACCESS:
10368	case VMX_EXIT_TR_ACCESS:
10369	case VMX_EXIT_INVEPT:
10370	case VMX_EXIT_RDTSCP:
10371	case VMX_EXIT_INVVPID:
10372	case VMX_EXIT_WBINVD:
10373	case VMX_EXIT_XSETBV:
10374	case VMX_EXIT_RDRAND:
10375	case VMX_EXIT_INVPCID:
10376	case VMX_EXIT_VMFUNC:
10377	case VMX_EXIT_RDSEED:
10378	case VMX_EXIT_XSAVES:
10379	case VMX_EXIT_XRSTORS:
10380	{
10381	int rc2 = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10382	rc2 \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10383	AssertRCReturn(rc2, rc2);
10384	if ( pMixedCtx->rip != pDbgState->uRipStart
10385	\|\| pMixedCtx->cs.Sel != pDbgState->uCsStart)
10386	return VINF_EM_DBG_STEPPED;
10387	break;
10388	}
10389
10390	/* Errors and unexpected events: */
10391	case VMX_EXIT_INIT_SIGNAL:
10392	case VMX_EXIT_SIPI:
10393	case VMX_EXIT_IO_SMI:
10394	case VMX_EXIT_SMI:
10395	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
10396	case VMX_EXIT_ERR_MSR_LOAD:
10397	case VMX_EXIT_ERR_MACHINE_CHECK:
10398	case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
10399	break;
10400
10401	default:
10402	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
10403	break;
10404	}
10405	}
10406
10407	/*
10408	* Check for debugger event breakpoints and dtrace probes.
10409	*/
10410	if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
10411	&& ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
10412	{
10413	VBOXSTRICTRC rcStrict = hmR0VmxHandleExitDtraceEvents(pVM, pVCpu, pMixedCtx, pVmxTransient, uExitReason);
10414	if (rcStrict != VINF_SUCCESS)
10415	return rcStrict;
10416	}
10417
10418	/*
10419	* Normal processing.
10420	*/
10421	#ifdef HMVMX_USE_FUNCTION_TABLE
10422	return g_apfnVMExitHandlers[uExitReason](pVCpu, pMixedCtx, pVmxTransient);
10423	#else
10424	return hmR0VmxHandleExit(pVCpu, pMixedCtx, pVmxTransient, uExitReason);
10425	#endif
10426	}
10427
10428
10429	/**
10430	* Single steps guest code using VT-x.
10431	*
10432	* @returns Strict VBox status code (i.e. informational status codes too).
10433	* @param pVM The cross context VM structure.
10434	* @param pVCpu The cross context virtual CPU structure.
10435	* @param pCtx Pointer to the guest-CPU context.
10436	*
10437	* @note Mostly the same as hmR0VmxRunGuestCodeNormal().
10438	*/
10439	static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10440	{
10441	VMXTRANSIENT VmxTransient;
10442	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
10443
10444	/* Set HMCPU indicators. */
10445	bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
10446	pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction \|\| DBGFIsStepping(pVCpu);
10447	pVCpu->hm.s.fDebugWantRdTscExit = false;
10448	pVCpu->hm.s.fUsingDebugLoop = true;
10449
10450	/* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
10451	VMXRUNDBGSTATE DbgState;
10452	hmR0VmxRunDebugStateInit(pVCpu, pCtx, &DbgState);
10453	hmR0VmxPreRunGuestDebugStateUpdate(pVM, pVCpu, pCtx, &DbgState, &VmxTransient);
10454
10455	/*
10456	* The loop.
10457	*/
10458	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
10459	for (uint32_t cLoops = 0; ; cLoops++)
10460	{
10461	Assert(!HMR0SuspendPending());
10462	HMVMX_ASSERT_CPU_SAFE();
10463	bool fStepping = pVCpu->hm.s.fSingleInstruction;
10464
10465	/*
10466	* Preparatory work for running guest code, this may force us to return
10467	* to ring-3. This bugger disables interrupts on VINF_SUCCESS!
10468	*/
10469	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
10470	hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Set up execute controls the next to can respond to. */
10471	rcStrict = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, fStepping);
10472	if (rcStrict != VINF_SUCCESS)
10473	break;
10474
10475	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
10476	hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Override any obnoxious code in the above two calls. */
10477
10478	/*
10479	* Now we can run the guest code.
10480	*/
10481	int rcRun = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
10482
10483	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
10484
10485	/*
10486	* Restore any residual host-state and save any bits shared between host
10487	* and guest into the guest-CPU state. Re-enables interrupts!
10488	*/
10489	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rcRun);
10490
10491	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
10492	if (RT_SUCCESS(rcRun))
10493	{ /* very likely */ }
10494	else
10495	{
10496	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
10497	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rcRun, pCtx, &VmxTransient);
10498	return rcRun;
10499	}
10500
10501	/* Profile the VM-exit. */
10502	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
10503	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
10504	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
10505	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
10506	HMVMX_START_EXIT_DISPATCH_PROF();
10507
10508	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
10509
10510	/*
10511	* Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitDebug().
10512	*/
10513	rcStrict = hmR0VmxRunDebugHandleExit(pVM, pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason, &DbgState);
10514	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
10515	if (rcStrict != VINF_SUCCESS)
10516	break;
10517	if (cLoops > pVM->hm.s.cMaxResumeLoops)
10518	{
10519	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
10520	rcStrict = VINF_EM_RAW_INTERRUPT;
10521	break;
10522	}
10523
10524	/*
10525	* Stepping: Did the RIP change, if so, consider it a single step.
10526	* Otherwise, make sure one of the TFs gets set.
10527	*/
10528	if (fStepping)
10529	{
10530	int rc2 = hmR0VmxSaveGuestRip(pVCpu, pCtx);
10531	rc2 \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pCtx);
10532	AssertRCReturn(rc2, rc2);
10533	if ( pCtx->rip != DbgState.uRipStart
10534	\|\| pCtx->cs.Sel != DbgState.uCsStart)
10535	{
10536	rcStrict = VINF_EM_DBG_STEPPED;
10537	break;
10538	}
10539	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
10540	}
10541
10542	/*
10543	* Update when dtrace settings changes (DBGF kicks us, so no need to check).
10544	*/
10545	if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
10546	hmR0VmxPreRunGuestDebugStateUpdate(pVM, pVCpu, pCtx, &DbgState, &VmxTransient);
10547	}
10548
10549	/*
10550	* Clear the X86_EFL_TF if necessary.
10551	*/
10552	if (pVCpu->hm.s.fClearTrapFlag)
10553	{
10554	int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pCtx);
10555	AssertRCReturn(rc2, rc2);
10556	pVCpu->hm.s.fClearTrapFlag = false;
10557	pCtx->eflags.Bits.u1TF = 0;
10558	}
10559	/** @todo there seems to be issues with the resume flag when the monitor trap
10560	* flag is pending without being used. Seen early in bios init when
10561	* accessing APIC page in protected mode. */
10562
10563	/*
10564	* Restore VM-exit control settings as we may not reenter this function the
10565	* next time around.
10566	*/
10567	rcStrict = hmR0VmxRunDebugStateRevert(pVCpu, &DbgState, rcStrict);
10568
10569	/* Restore HMCPU indicators. */
10570	pVCpu->hm.s.fUsingDebugLoop = false;
10571	pVCpu->hm.s.fDebugWantRdTscExit = false;
10572	pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
10573
10574	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
10575	return rcStrict;
10576	}
10577
10578
10579	/** @} */
10580
10581
10582	/**
10583	* Checks if any expensive dtrace probes are enabled and we should go to the
10584	* debug loop.
10585	*
10586	* @returns true if we should use debug loop, false if not.
10587	*/
10588	static bool hmR0VmxAnyExpensiveProbesEnabled(void)
10589	{
10590	/* It's probably faster to OR the raw 32-bit counter variables together.
10591	Since the variables are in an array and the probes are next to one
10592	another (more or less), we have good locality. So, better read
10593	eight-nine cache lines ever time and only have one conditional, than
10594	128+ conditionals, right? */
10595	return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED_RAW() /* expensive too due to context */
10596	\| VBOXVMM_XCPT_DE_ENABLED_RAW()
10597	\| VBOXVMM_XCPT_DB_ENABLED_RAW()
10598	\| VBOXVMM_XCPT_BP_ENABLED_RAW()
10599	\| VBOXVMM_XCPT_OF_ENABLED_RAW()
10600	\| VBOXVMM_XCPT_BR_ENABLED_RAW()
10601	\| VBOXVMM_XCPT_UD_ENABLED_RAW()
10602	\| VBOXVMM_XCPT_NM_ENABLED_RAW()
10603	\| VBOXVMM_XCPT_DF_ENABLED_RAW()
10604	\| VBOXVMM_XCPT_TS_ENABLED_RAW()
10605	\| VBOXVMM_XCPT_NP_ENABLED_RAW()
10606	\| VBOXVMM_XCPT_SS_ENABLED_RAW()
10607	\| VBOXVMM_XCPT_GP_ENABLED_RAW()
10608	\| VBOXVMM_XCPT_PF_ENABLED_RAW()
10609	\| VBOXVMM_XCPT_MF_ENABLED_RAW()
10610	\| VBOXVMM_XCPT_AC_ENABLED_RAW()
10611	\| VBOXVMM_XCPT_XF_ENABLED_RAW()
10612	\| VBOXVMM_XCPT_VE_ENABLED_RAW()
10613	\| VBOXVMM_XCPT_SX_ENABLED_RAW()
10614	\| VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
10615	\| VBOXVMM_INT_HARDWARE_ENABLED_RAW()
10616	) != 0
10617	\|\| ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
10618	\| VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
10619	\| VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
10620	\| VBOXVMM_INSTR_CPUID_ENABLED_RAW()
10621	\| VBOXVMM_INSTR_INVD_ENABLED_RAW()
10622	\| VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
10623	\| VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
10624	\| VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
10625	\| VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
10626	\| VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
10627	\| VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
10628	\| VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
10629	\| VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
10630	\| VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
10631	\| VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
10632	\| VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
10633	\| VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
10634	\| VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
10635	\| VBOXVMM_INSTR_SIDT_ENABLED_RAW()
10636	\| VBOXVMM_INSTR_LIDT_ENABLED_RAW()
10637	\| VBOXVMM_INSTR_SGDT_ENABLED_RAW()
10638	\| VBOXVMM_INSTR_LGDT_ENABLED_RAW()
10639	\| VBOXVMM_INSTR_SLDT_ENABLED_RAW()
10640	\| VBOXVMM_INSTR_LLDT_ENABLED_RAW()
10641	\| VBOXVMM_INSTR_STR_ENABLED_RAW()
10642	\| VBOXVMM_INSTR_LTR_ENABLED_RAW()
10643	\| VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
10644	\| VBOXVMM_INSTR_RSM_ENABLED_RAW()
10645	\| VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
10646	\| VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
10647	\| VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
10648	\| VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
10649	\| VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
10650	\| VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED_RAW()
10651	\| VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED_RAW()
10652	\| VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED_RAW()
10653	\| VBOXVMM_INSTR_VMX_VMPTRST_ENABLED_RAW()
10654	\| VBOXVMM_INSTR_VMX_VMREAD_ENABLED_RAW()
10655	\| VBOXVMM_INSTR_VMX_VMRESUME_ENABLED_RAW()
10656	\| VBOXVMM_INSTR_VMX_VMWRITE_ENABLED_RAW()
10657	\| VBOXVMM_INSTR_VMX_VMXOFF_ENABLED_RAW()
10658	\| VBOXVMM_INSTR_VMX_VMXON_ENABLED_RAW()
10659	\| VBOXVMM_INSTR_VMX_VMFUNC_ENABLED_RAW()
10660	\| VBOXVMM_INSTR_VMX_INVEPT_ENABLED_RAW()
10661	\| VBOXVMM_INSTR_VMX_INVVPID_ENABLED_RAW()
10662	\| VBOXVMM_INSTR_VMX_INVPCID_ENABLED_RAW()
10663	) != 0
10664	\|\| ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
10665	\| VBOXVMM_EXIT_HALT_ENABLED_RAW()
10666	\| VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
10667	\| VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
10668	\| VBOXVMM_EXIT_CPUID_ENABLED_RAW()
10669	\| VBOXVMM_EXIT_INVD_ENABLED_RAW()
10670	\| VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
10671	\| VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
10672	\| VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
10673	\| VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
10674	\| VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
10675	\| VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
10676	\| VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
10677	\| VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
10678	\| VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
10679	\| VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
10680	\| VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
10681	\| VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
10682	\| VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
10683	\| VBOXVMM_EXIT_SIDT_ENABLED_RAW()
10684	\| VBOXVMM_EXIT_LIDT_ENABLED_RAW()
10685	\| VBOXVMM_EXIT_SGDT_ENABLED_RAW()
10686	\| VBOXVMM_EXIT_LGDT_ENABLED_RAW()
10687	\| VBOXVMM_EXIT_SLDT_ENABLED_RAW()
10688	\| VBOXVMM_EXIT_LLDT_ENABLED_RAW()
10689	\| VBOXVMM_EXIT_STR_ENABLED_RAW()
10690	\| VBOXVMM_EXIT_LTR_ENABLED_RAW()
10691	\| VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
10692	\| VBOXVMM_EXIT_RSM_ENABLED_RAW()
10693	\| VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
10694	\| VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
10695	\| VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
10696	\| VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
10697	\| VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
10698	\| VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED_RAW()
10699	\| VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED_RAW()
10700	\| VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED_RAW()
10701	\| VBOXVMM_EXIT_VMX_VMPTRST_ENABLED_RAW()
10702	\| VBOXVMM_EXIT_VMX_VMREAD_ENABLED_RAW()
10703	\| VBOXVMM_EXIT_VMX_VMRESUME_ENABLED_RAW()
10704	\| VBOXVMM_EXIT_VMX_VMWRITE_ENABLED_RAW()
10705	\| VBOXVMM_EXIT_VMX_VMXOFF_ENABLED_RAW()
10706	\| VBOXVMM_EXIT_VMX_VMXON_ENABLED_RAW()
10707	\| VBOXVMM_EXIT_VMX_VMFUNC_ENABLED_RAW()
10708	\| VBOXVMM_EXIT_VMX_INVEPT_ENABLED_RAW()
10709	\| VBOXVMM_EXIT_VMX_INVVPID_ENABLED_RAW()
10710	\| VBOXVMM_EXIT_VMX_INVPCID_ENABLED_RAW()
10711	\| VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED_RAW()
10712	\| VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED_RAW()
10713	\| VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED_RAW()
10714	\| VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED_RAW()
10715	) != 0;
10716	}
10717
10718
10719	/**
10720	* Runs the guest code using VT-x.
10721	*
10722	* @returns Strict VBox status code (i.e. informational status codes too).
10723	* @param pVM The cross context VM structure.
10724	* @param pVCpu The cross context virtual CPU structure.
10725	* @param pCtx Pointer to the guest-CPU context.
10726	*/
10727	VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10728	{
10729	Assert(VMMRZCallRing3IsEnabled(pVCpu));
10730	Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
10731	HMVMX_ASSERT_PREEMPT_SAFE();
10732
10733	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
10734
10735	VBOXSTRICTRC rcStrict;
10736	if ( !pVCpu->hm.s.fUseDebugLoop
10737	&& (!VBOXVMM_ANY_PROBES_ENABLED() \|\| !hmR0VmxAnyExpensiveProbesEnabled())
10738	&& !DBGFIsStepping(pVCpu)
10739	&& !pVM->dbgf.ro.cEnabledInt3Breakpoints)
10740	rcStrict = hmR0VmxRunGuestCodeNormal(pVM, pVCpu, pCtx);
10741	else
10742	rcStrict = hmR0VmxRunGuestCodeDebug(pVM, pVCpu, pCtx);
10743
10744	if (rcStrict == VERR_EM_INTERPRETER)
10745	rcStrict = VINF_EM_RAW_EMULATE_INSTR;
10746	else if (rcStrict == VINF_EM_RESET)
10747	rcStrict = VINF_EM_TRIPLE_FAULT;
10748
10749	int rc2 = hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rcStrict);
10750	if (RT_FAILURE(rc2))
10751	{
10752	pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
10753	rcStrict = rc2;
10754	}
10755	Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
10756	return rcStrict;
10757	}
10758
10759
10760	#ifndef HMVMX_USE_FUNCTION_TABLE
10761	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
10762	{
10763	# ifdef DEBUG_ramshankar
10764	# define RETURN_EXIT_CALL(a_CallExpr) \
10765	do { \
10766	int rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx); AssertRC(rc2); \
10767	VBOXSTRICTRC rcStrict = a_CallExpr; \
10768	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST); \
10769	return rcStrict; \
10770	} while (0)
10771	# else
10772	# define RETURN_EXIT_CALL(a_CallExpr) return a_CallExpr
10773	# endif
10774	switch (rcReason)
10775	{
10776	case VMX_EXIT_EPT_MISCONFIG: RETURN_EXIT_CALL(hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient));
10777	case VMX_EXIT_EPT_VIOLATION: RETURN_EXIT_CALL(hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient));
10778	case VMX_EXIT_IO_INSTR: RETURN_EXIT_CALL(hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient));
10779	case VMX_EXIT_CPUID: RETURN_EXIT_CALL(hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient));
10780	case VMX_EXIT_RDTSC: RETURN_EXIT_CALL(hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient));
10781	case VMX_EXIT_RDTSCP: RETURN_EXIT_CALL(hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient));
10782	case VMX_EXIT_APIC_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient));
10783	case VMX_EXIT_XCPT_OR_NMI: RETURN_EXIT_CALL(hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient));
10784	case VMX_EXIT_MOV_CRX: RETURN_EXIT_CALL(hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient));
10785	case VMX_EXIT_EXT_INT: RETURN_EXIT_CALL(hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient));
10786	case VMX_EXIT_INT_WINDOW: RETURN_EXIT_CALL(hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient));
10787	case VMX_EXIT_MWAIT: RETURN_EXIT_CALL(hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient));
10788	case VMX_EXIT_MONITOR: RETURN_EXIT_CALL(hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient));
10789	case VMX_EXIT_TASK_SWITCH: RETURN_EXIT_CALL(hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient));
10790	case VMX_EXIT_PREEMPT_TIMER: RETURN_EXIT_CALL(hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient));
10791	case VMX_EXIT_RDMSR: RETURN_EXIT_CALL(hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient));
10792	case VMX_EXIT_WRMSR: RETURN_EXIT_CALL(hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient));
10793	case VMX_EXIT_MOV_DRX: RETURN_EXIT_CALL(hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient));
10794	case VMX_EXIT_TPR_BELOW_THRESHOLD: RETURN_EXIT_CALL(hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient));
10795	case VMX_EXIT_HLT: RETURN_EXIT_CALL(hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient));
10796	case VMX_EXIT_INVD: RETURN_EXIT_CALL(hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient));
10797	case VMX_EXIT_INVLPG: RETURN_EXIT_CALL(hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient));
10798	case VMX_EXIT_RSM: RETURN_EXIT_CALL(hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient));
10799	case VMX_EXIT_MTF: RETURN_EXIT_CALL(hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient));
10800	case VMX_EXIT_PAUSE: RETURN_EXIT_CALL(hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient));
10801	case VMX_EXIT_XDTR_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient));
10802	case VMX_EXIT_TR_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient));
10803	case VMX_EXIT_WBINVD: RETURN_EXIT_CALL(hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient));
10804	case VMX_EXIT_XSETBV: RETURN_EXIT_CALL(hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient));
10805	case VMX_EXIT_RDRAND: RETURN_EXIT_CALL(hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient));
10806	case VMX_EXIT_INVPCID: RETURN_EXIT_CALL(hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient));
10807	case VMX_EXIT_GETSEC: RETURN_EXIT_CALL(hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient));
10808	case VMX_EXIT_RDPMC: RETURN_EXIT_CALL(hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient));
10809	case VMX_EXIT_VMCALL: RETURN_EXIT_CALL(hmR0VmxExitVmcall(pVCpu, pMixedCtx, pVmxTransient));
10810
10811	case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient);
10812	case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient);
10813	case VMX_EXIT_INIT_SIGNAL: return hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient);
10814	case VMX_EXIT_SIPI: return hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient);
10815	case VMX_EXIT_IO_SMI: return hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient);
10816	case VMX_EXIT_SMI: return hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient);
10817	case VMX_EXIT_ERR_MSR_LOAD: return hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient);
10818	case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient);
10819	case VMX_EXIT_ERR_MACHINE_CHECK: return hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient);
10820
10821	case VMX_EXIT_VMCLEAR:
10822	case VMX_EXIT_VMLAUNCH:
10823	case VMX_EXIT_VMPTRLD:
10824	case VMX_EXIT_VMPTRST:
10825	case VMX_EXIT_VMREAD:
10826	case VMX_EXIT_VMRESUME:
10827	case VMX_EXIT_VMWRITE:
10828	case VMX_EXIT_VMXOFF:
10829	case VMX_EXIT_VMXON:
10830	case VMX_EXIT_INVEPT:
10831	case VMX_EXIT_INVVPID:
10832	case VMX_EXIT_VMFUNC:
10833	case VMX_EXIT_XSAVES:
10834	case VMX_EXIT_XRSTORS:
10835	return hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
10836	case VMX_EXIT_ENCLS:
10837	case VMX_EXIT_RDSEED: /* only spurious VM-exits, so undefined */
10838	case VMX_EXIT_PML_FULL:
10839	default:
10840	return hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
10841	}
10842	#undef RETURN_EXIT_CALL
10843	}
10844	#endif /* !HMVMX_USE_FUNCTION_TABLE */
10845
10846
10847	#ifdef VBOX_STRICT
10848	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
10849	# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
10850	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
10851
10852	# define HMVMX_ASSERT_PREEMPT_CPUID() \
10853	do { \
10854	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
10855	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
10856	} while (0)
10857
10858	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
10859	do { \
10860	AssertPtr(pVCpu); \
10861	AssertPtr(pMixedCtx); \
10862	AssertPtr(pVmxTransient); \
10863	Assert(pVmxTransient->fVMEntryFailed == false); \
10864	Assert(ASMIntAreEnabled()); \
10865	HMVMX_ASSERT_PREEMPT_SAFE(); \
10866	HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
10867	Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
10868	HMVMX_ASSERT_PREEMPT_SAFE(); \
10869	if (VMMR0IsLogFlushDisabled(pVCpu)) \
10870	HMVMX_ASSERT_PREEMPT_CPUID(); \
10871	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10872	} while (0)
10873
10874	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
10875	do { \
10876	Log4Func(("\n")); \
10877	} while (0)
10878	#else /* nonstrict builds: */
10879	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
10880	do { \
10881	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10882	NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient); \
10883	} while (0)
10884	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while (0)
10885	#endif
10886
10887
10888	/**
10889	* Advances the guest RIP by the specified number of bytes.
10890	*
10891	* @param pVCpu The cross context virtual CPU structure.
10892	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
10893	* out-of-sync. Make sure to update the required fields
10894	* before using them.
10895	* @param cbInstr Number of bytes to advance the RIP by.
10896	*
10897	* @remarks No-long-jump zone!!!
10898	*/
10899	DECLINLINE(void) hmR0VmxAdvanceGuestRipBy(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
10900	{
10901	/* Advance the RIP. */
10902	pMixedCtx->rip += cbInstr;
10903	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10904
10905	/* Update interrupt inhibition. */
10906	if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
10907	&& pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
10908	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
10909	}
10910
10911
10912	/**
10913	* Advances the guest RIP after reading it from the VMCS.
10914	*
10915	* @returns VBox status code, no informational status codes.
10916	* @param pVCpu The cross context virtual CPU structure.
10917	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
10918	* out-of-sync. Make sure to update the required fields
10919	* before using them.
10920	* @param pVmxTransient Pointer to the VMX transient structure.
10921	*
10922	* @remarks No-long-jump zone!!!
10923	*/
10924	static int hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10925	{
10926	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10927	rc \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10928	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10929	AssertRCReturn(rc, rc);
10930
10931	hmR0VmxAdvanceGuestRipBy(pVCpu, pMixedCtx, pVmxTransient->cbInstr);
10932
10933	/*
10934	* Deliver a debug exception to the guest if it is single-stepping. Don't directly inject a #DB but use the
10935	* pending debug exception field as it takes care of priority of events.
10936	*
10937	* See Intel spec. 32.2.1 "Debug Exceptions".
10938	*/
10939	if ( !pVCpu->hm.s.fSingleInstruction
10940	&& pMixedCtx->eflags.Bits.u1TF)
10941	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
10942
10943	return VINF_SUCCESS;
10944	}
10945
10946
10947	/**
10948	* Tries to determine what part of the guest-state VT-x has deemed as invalid
10949	* and update error record fields accordingly.
10950	*
10951	* @return VMX_IGS_* return codes.
10952	* @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
10953	* wrong with the guest state.
10954	*
10955	* @param pVM The cross context VM structure.
10956	* @param pVCpu The cross context virtual CPU structure.
10957	* @param pCtx Pointer to the guest-CPU state.
10958	*
10959	* @remarks This function assumes our cache of the VMCS controls
10960	* are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
10961	*/
10962	static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10963	{
10964	#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
10965	#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
10966	uError = (err); \
10967	break; \
10968	} else do { } while (0)
10969
10970	int rc;
10971	uint32_t uError = VMX_IGS_ERROR;
10972	uint32_t u32Val;
10973	bool fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
10974
10975	do
10976	{
10977	/*
10978	* CR0.
10979	*/
10980	uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10981	uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10982	/* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
10983	See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
10984	if (fUnrestrictedGuest)
10985	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
10986
10987	uint32_t u32GuestCR0;
10988	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCR0);
10989	AssertRCBreak(rc);
10990	HMVMX_CHECK_BREAK((u32GuestCR0 & uSetCR0) == uSetCR0, VMX_IGS_CR0_FIXED1);
10991	HMVMX_CHECK_BREAK(!(u32GuestCR0 & ~uZapCR0), VMX_IGS_CR0_FIXED0);
10992	if ( !fUnrestrictedGuest
10993	&& (u32GuestCR0 & X86_CR0_PG)
10994	&& !(u32GuestCR0 & X86_CR0_PE))
10995	{
10996	HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
10997	}
10998
10999	/*
11000	* CR4.
11001	*/
11002	uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
11003	uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
11004
11005	uint32_t u32GuestCR4;
11006	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCR4);
11007	AssertRCBreak(rc);
11008	HMVMX_CHECK_BREAK((u32GuestCR4 & uSetCR4) == uSetCR4, VMX_IGS_CR4_FIXED1);
11009	HMVMX_CHECK_BREAK(!(u32GuestCR4 & ~uZapCR4), VMX_IGS_CR4_FIXED0);
11010
11011	/*
11012	* IA32_DEBUGCTL MSR.
11013	*/
11014	uint64_t u64Val;
11015	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
11016	AssertRCBreak(rc);
11017	if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
11018	&& (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
11019	{
11020	HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
11021	}
11022	uint64_t u64DebugCtlMsr = u64Val;
11023
11024	#ifdef VBOX_STRICT
11025	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
11026	AssertRCBreak(rc);
11027	Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
11028	#endif
11029	bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
11030
11031	/*
11032	* RIP and RFLAGS.
11033	*/
11034	uint32_t u32Eflags;
11035	#if HC_ARCH_BITS == 64
11036	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
11037	AssertRCBreak(rc);
11038	/* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
11039	if ( !fLongModeGuest
11040	\|\| !pCtx->cs.Attr.n.u1Long)
11041	{
11042	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
11043	}
11044	/** @todo If the processor supports N < 64 linear-address bits, bits 63:N
11045	* must be identical if the "IA-32e mode guest" VM-entry
11046	* control is 1 and CS.L is 1. No check applies if the
11047	* CPU supports 64 linear-address bits. */
11048
11049	/* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
11050	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
11051	AssertRCBreak(rc);
11052	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
11053	VMX_IGS_RFLAGS_RESERVED);
11054	HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
11055	u32Eflags = u64Val;
11056	#else
11057	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
11058	AssertRCBreak(rc);
11059	HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
11060	HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
11061	#endif
11062
11063	if ( fLongModeGuest
11064	\|\| ( fUnrestrictedGuest
11065	&& !(u32GuestCR0 & X86_CR0_PE)))
11066	{
11067	HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
11068	}
11069
11070	uint32_t u32EntryInfo;
11071	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
11072	AssertRCBreak(rc);
11073	if ( VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
11074	&& VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
11075	{
11076	HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
11077	}
11078
11079	/*
11080	* 64-bit checks.
11081	*/
11082	#if HC_ARCH_BITS == 64
11083	if (fLongModeGuest)
11084	{
11085	HMVMX_CHECK_BREAK(u32GuestCR0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
11086	HMVMX_CHECK_BREAK(u32GuestCR4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
11087	}
11088
11089	if ( !fLongModeGuest
11090	&& (u32GuestCR4 & X86_CR4_PCIDE))
11091	{
11092	HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
11093	}
11094
11095	/** @todo CR3 field must be such that bits 63:52 and bits in the range
11096	* 51:32 beyond the processor's physical-address width are 0. */
11097
11098	if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
11099	&& (pCtx->dr[7] & X86_DR7_MBZ_MASK))
11100	{
11101	HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
11102	}
11103
11104	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
11105	AssertRCBreak(rc);
11106	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
11107
11108	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
11109	AssertRCBreak(rc);
11110	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
11111	#endif
11112
11113	/*
11114	* PERF_GLOBAL MSR.
11115	*/
11116	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
11117	{
11118	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
11119	AssertRCBreak(rc);
11120	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
11121	VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
11122	}
11123
11124	/*
11125	* PAT MSR.
11126	*/
11127	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
11128	{
11129	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
11130	AssertRCBreak(rc);
11131	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
11132	for (unsigned i = 0; i < 8; i++)
11133	{
11134	uint8_t u8Val = (u64Val & 0xff);
11135	if ( u8Val != 0 /* UC */
11136	&& u8Val != 1 /* WC */
11137	&& u8Val != 4 /* WT */
11138	&& u8Val != 5 /* WP */
11139	&& u8Val != 6 /* WB */
11140	&& u8Val != 7 /* UC- */)
11141	{
11142	HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
11143	}
11144	u64Val >>= 8;
11145	}
11146	}
11147
11148	/*
11149	* EFER MSR.
11150	*/
11151	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
11152	{
11153	Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
11154	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
11155	AssertRCBreak(rc);
11156	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
11157	VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
11158	HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVCpu->hm.s.vmx.u32EntryCtls
11159	& VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
11160	VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
11161	HMVMX_CHECK_BREAK( fUnrestrictedGuest
11162	\|\| !(u32GuestCR0 & X86_CR0_PG)
11163	\|\| RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
11164	VMX_IGS_EFER_LMA_LME_MISMATCH);
11165	}
11166
11167	/*
11168	* Segment registers.
11169	*/
11170	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
11171	\|\| !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
11172	if (!(u32Eflags & X86_EFL_VM))
11173	{
11174	/* CS */
11175	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
11176	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
11177	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
11178	HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
11179	\|\| !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
11180	HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
11181	\|\| (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
11182	/* CS cannot be loaded with NULL in protected mode. */
11183	HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
11184	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
11185	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
11186	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
11187	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
11188	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
11189	else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
11190	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
11191	else
11192	HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
11193
11194	/* SS */
11195	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
11196	\|\| (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
11197	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
11198	if ( !(pCtx->cr0 & X86_CR0_PE)
11199	\|\| pCtx->cs.Attr.n.u4Type == 3)
11200	{
11201	HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
11202	}
11203	if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
11204	{
11205	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
11206	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
11207	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
11208	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
11209	HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
11210	\|\| !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
11211	HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
11212	\|\| (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
11213	}
11214
11215	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
11216	if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
11217	{
11218	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
11219	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
11220	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
11221	\|\| pCtx->ds.Attr.n.u4Type > 11
11222	\|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
11223	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
11224	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
11225	HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
11226	\|\| !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
11227	HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
11228	\|\| (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
11229	HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
11230	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
11231	}
11232	if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
11233	{
11234	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
11235	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
11236	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
11237	\|\| pCtx->es.Attr.n.u4Type > 11
11238	\|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
11239	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
11240	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
11241	HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
11242	\|\| !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
11243	HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
11244	\|\| (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
11245	HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
11246	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
11247	}
11248	if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
11249	{
11250	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
11251	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
11252	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
11253	\|\| pCtx->fs.Attr.n.u4Type > 11
11254	\|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
11255	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
11256	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
11257	HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
11258	\|\| !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
11259	HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
11260	\|\| (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
11261	HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
11262	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
11263	}
11264	if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
11265	{
11266	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
11267	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
11268	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
11269	\|\| pCtx->gs.Attr.n.u4Type > 11
11270	\|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
11271	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
11272	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
11273	HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
11274	\|\| !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
11275	HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
11276	\|\| (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
11277	HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
11278	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
11279	}
11280	/* 64-bit capable CPUs. */
11281	#if HC_ARCH_BITS == 64
11282	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
11283	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
11284	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
11285	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
11286	HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
11287	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ss.u64Base >> 32),
11288	VMX_IGS_LONGMODE_SS_BASE_INVALID);
11289	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ds.u64Base >> 32),
11290	VMX_IGS_LONGMODE_DS_BASE_INVALID);
11291	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->es.u64Base >> 32),
11292	VMX_IGS_LONGMODE_ES_BASE_INVALID);
11293	#endif
11294	}
11295	else
11296	{
11297	/* V86 mode checks. */
11298	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
11299	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11300	{
11301	u32CSAttr = 0xf3; u32SSAttr = 0xf3;
11302	u32DSAttr = 0xf3; u32ESAttr = 0xf3;
11303	u32FSAttr = 0xf3; u32GSAttr = 0xf3;
11304	}
11305	else
11306	{
11307	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
11308	u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
11309	u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
11310	}
11311
11312	/* CS */
11313	HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
11314	HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
11315	HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
11316	/* SS */
11317	HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
11318	HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
11319	HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
11320	/* DS */
11321	HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
11322	HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
11323	HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
11324	/* ES */
11325	HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
11326	HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
11327	HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
11328	/* FS */
11329	HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
11330	HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
11331	HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
11332	/* GS */
11333	HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
11334	HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
11335	HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
11336	/* 64-bit capable CPUs. */
11337	#if HC_ARCH_BITS == 64
11338	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
11339	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
11340	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
11341	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
11342	HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
11343	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ss.u64Base >> 32),
11344	VMX_IGS_LONGMODE_SS_BASE_INVALID);
11345	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->ds.u64Base >> 32),
11346	VMX_IGS_LONGMODE_DS_BASE_INVALID);
11347	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !(pCtx->es.u64Base >> 32),
11348	VMX_IGS_LONGMODE_ES_BASE_INVALID);
11349	#endif
11350	}
11351
11352	/*
11353	* TR.
11354	*/
11355	HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
11356	/* 64-bit capable CPUs. */
11357	#if HC_ARCH_BITS == 64
11358	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
11359	#endif
11360	if (fLongModeGuest)
11361	{
11362	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
11363	VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
11364	}
11365	else
11366	{
11367	HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
11368	\|\| pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
11369	VMX_IGS_TR_ATTR_TYPE_INVALID);
11370	}
11371	HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
11372	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
11373	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
11374	HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
11375	\|\| !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
11376	HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
11377	\|\| (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
11378	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
11379
11380	/*
11381	* GDTR and IDTR.
11382	*/
11383	#if HC_ARCH_BITS == 64
11384	rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
11385	AssertRCBreak(rc);
11386	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
11387
11388	rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
11389	AssertRCBreak(rc);
11390	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
11391	#endif
11392
11393	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
11394	AssertRCBreak(rc);
11395	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
11396
11397	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
11398	AssertRCBreak(rc);
11399	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
11400
11401	/*
11402	* Guest Non-Register State.
11403	*/
11404	/* Activity State. */
11405	uint32_t u32ActivityState;
11406	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
11407	AssertRCBreak(rc);
11408	HMVMX_CHECK_BREAK( !u32ActivityState
11409	\|\| (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
11410	VMX_IGS_ACTIVITY_STATE_INVALID);
11411	HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
11412	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
11413	uint32_t u32IntrState;
11414	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
11415	AssertRCBreak(rc);
11416	if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
11417	\|\| u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
11418	{
11419	HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
11420	}
11421
11422	/** @todo Activity state and injecting interrupts. Left as a todo since we
11423	* currently don't use activity states but ACTIVE. */
11424
11425	HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
11426	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
11427
11428	/* Guest interruptibility-state. */
11429	HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
11430	HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
11431	\| VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
11432	!= ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
11433	\| VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
11434	VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
11435	HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
11436	\|\| !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
11437	VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
11438	if (VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo))
11439	{
11440	if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
11441	{
11442	HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
11443	&& !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
11444	VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
11445	}
11446	else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
11447	{
11448	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
11449	VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
11450	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
11451	VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
11452	}
11453	}
11454	/** @todo Assumes the processor is not in SMM. */
11455	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
11456	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
11457	HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
11458	\|\| (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
11459	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
11460	if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
11461	&& VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
11462	&& VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
11463	{
11464	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
11465	VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
11466	}
11467
11468	/* Pending debug exceptions. */
11469	#if HC_ARCH_BITS == 64
11470	rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
11471	AssertRCBreak(rc);
11472	/* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
11473	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
11474	u32Val = u64Val; /* For pending debug exceptions checks below. */
11475	#else
11476	rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
11477	AssertRCBreak(rc);
11478	/* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
11479	HMVMX_CHECK_BREAK(!(u32Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
11480	#endif
11481
11482	if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
11483	\|\| (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
11484	\|\| u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
11485	{
11486	if ( (u32Eflags & X86_EFL_TF)
11487	&& !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
11488	{
11489	/* Bit 14 is PendingDebug.BS. */
11490	HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
11491	}
11492	if ( !(u32Eflags & X86_EFL_TF)
11493	\|\| (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
11494	{
11495	/* Bit 14 is PendingDebug.BS. */
11496	HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
11497	}
11498	}
11499
11500	/* VMCS link pointer. */
11501	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
11502	AssertRCBreak(rc);
11503	if (u64Val != UINT64_C(0xffffffffffffffff))
11504	{
11505	HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
11506	/** @todo Bits beyond the processor's physical-address width MBZ. */
11507	/** @todo 32-bit located in memory referenced by value of this field (as a
11508	* physical address) must contain the processor's VMCS revision ID. */
11509	/** @todo SMM checks. */
11510	}
11511
11512	/** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
11513	* not using Nested Paging? */
11514	if ( pVM->hm.s.fNestedPaging
11515	&& !fLongModeGuest
11516	&& CPUMIsGuestInPAEModeEx(pCtx))
11517	{
11518	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
11519	AssertRCBreak(rc);
11520	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11521
11522	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
11523	AssertRCBreak(rc);
11524	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11525
11526	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
11527	AssertRCBreak(rc);
11528	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11529
11530	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
11531	AssertRCBreak(rc);
11532	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11533	}
11534
11535	/* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
11536	if (uError == VMX_IGS_ERROR)
11537	uError = VMX_IGS_REASON_NOT_FOUND;
11538	} while (0);
11539
11540	pVCpu->hm.s.u32HMError = uError;
11541	return uError;
11542
11543	#undef HMVMX_ERROR_BREAK
11544	#undef HMVMX_CHECK_BREAK
11545	}
11546
11547	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11548	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
11549	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11550
11551	/** @name VM-exit handlers.
11552	* @{
11553	*/
11554
11555	/**
11556	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
11557	*/
11558	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11559	{
11560	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11561	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
11562	/* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
11563	if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
11564	return VINF_SUCCESS;
11565	return VINF_EM_RAW_INTERRUPT;
11566	}
11567
11568
11569	/**
11570	* VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
11571	*/
11572	HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11573	{
11574	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11575	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
11576
11577	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11578	AssertRCReturn(rc, rc);
11579
11580	uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
11581	Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
11582	&& uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
11583	Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
11584
11585	if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
11586	{
11587	/*
11588	* This cannot be a guest NMI as the only way for the guest to receive an NMI is if we injected it ourselves and
11589	* anything we inject is not going to cause a VM-exit directly for the event being injected.
11590	* See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
11591	*
11592	* Dispatch the NMI to the host. See Intel spec. 27.5.5 "Updating Non-Register State".
11593	*/
11594	VMXDispatchHostNmi();
11595	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
11596	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11597	return VINF_SUCCESS;
11598	}
11599
11600	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11601	VBOXSTRICTRC rcStrictRc1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11602	if (RT_UNLIKELY(rcStrictRc1 == VINF_SUCCESS))
11603	{ /* likely */ }
11604	else
11605	{
11606	if (rcStrictRc1 == VINF_HM_DOUBLE_FAULT)
11607	rcStrictRc1 = VINF_SUCCESS;
11608	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11609	return rcStrictRc1;
11610	}
11611
11612	uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
11613	uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntInfo);
11614	switch (uIntType)
11615	{
11616	case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
11617	Assert(uVector == X86_XCPT_DB);
11618	RT_FALL_THRU();
11619	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
11620	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF \|\| uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT);
11621	RT_FALL_THRU();
11622	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
11623	{
11624	/*
11625	* If there's any exception caused as a result of event injection, the resulting
11626	* secondary/final execption will be pending, we shall continue guest execution
11627	* after injecting the event. The page-fault case is complicated and we manually
11628	* handle any currently pending event in hmR0VmxExitXcptPF.
11629	*/
11630	if (!pVCpu->hm.s.Event.fPending)
11631	{ /* likely */ }
11632	else if (uVector != X86_XCPT_PF)
11633	{
11634	rc = VINF_SUCCESS;
11635	break;
11636	}
11637
11638	switch (uVector)
11639	{
11640	case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
11641	case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
11642	case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
11643	case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
11644	case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
11645	case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
11646	case X86_XCPT_AC: rc = hmR0VmxExitXcptAC(pVCpu, pMixedCtx, pVmxTransient); break;
11647
11648	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
11649	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11650	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
11651	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11652	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
11653	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11654	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
11655	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11656	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
11657	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11658	case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
11659	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11660	default:
11661	{
11662	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11663	AssertRCReturn(rc, rc);
11664
11665	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
11666	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11667	{
11668	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
11669	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
11670	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
11671
11672	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11673	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11674	AssertRCReturn(rc, rc);
11675	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
11676	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
11677	0 /* GCPtrFaultAddress */);
11678	AssertRCReturn(rc, rc);
11679	}
11680	else
11681	{
11682	AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
11683	pVCpu->hm.s.u32HMError = uVector;
11684	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
11685	}
11686	break;
11687	}
11688	}
11689	break;
11690	}
11691
11692	default:
11693	{
11694	pVCpu->hm.s.u32HMError = uExitIntInfo;
11695	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
11696	AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntInfo)));
11697	break;
11698	}
11699	}
11700	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11701	return rc;
11702	}
11703
11704
11705	/**
11706	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
11707	*/
11708	HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11709	{
11710	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11711
11712	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
11713	hmR0VmxClearIntWindowExitVmcs(pVCpu);
11714
11715	/* Deliver the pending interrupts via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11716	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
11717	return VINF_SUCCESS;
11718	}
11719
11720
11721	/**
11722	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
11723	*/
11724	HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11725	{
11726	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11727	if (RT_UNLIKELY(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)))
11728	{
11729	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
11730	HMVMX_RETURN_UNEXPECTED_EXIT();
11731	}
11732
11733	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS));
11734
11735	/*
11736	* If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
11737	* It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
11738	*/
11739	uint32_t uIntrState = 0;
11740	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
11741	AssertRCReturn(rc, rc);
11742
11743	bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
11744	if ( fBlockSti
11745	&& VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
11746	{
11747	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
11748	}
11749
11750	/* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
11751	hmR0VmxClearNmiWindowExitVmcs(pVCpu);
11752
11753	/* Deliver the pending NMI via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11754	return VINF_SUCCESS;
11755	}
11756
11757
11758	/**
11759	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
11760	*/
11761	HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11762	{
11763	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11764	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
11765	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11766	}
11767
11768
11769	/**
11770	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
11771	*/
11772	HMVMX_EXIT_NSRC_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11773	{
11774	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11775	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
11776	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11777	}
11778
11779
11780	/**
11781	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
11782	*/
11783	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11784	{
11785	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11786	PVM pVM = pVCpu->CTX_SUFF(pVM);
11787	int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11788	if (RT_LIKELY(rc == VINF_SUCCESS))
11789	{
11790	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11791	Assert(pVmxTransient->cbInstr == 2);
11792	}
11793	else
11794	{
11795	AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
11796	rc = VERR_EM_INTERPRETER;
11797	}
11798	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
11799	return rc;
11800	}
11801
11802
11803	/**
11804	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
11805	*/
11806	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11807	{
11808	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11809	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11810	AssertRCReturn(rc, rc);
11811
11812	if (pMixedCtx->cr4 & X86_CR4_SMXE)
11813	return VINF_EM_RAW_EMULATE_INSTR;
11814
11815	AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
11816	HMVMX_RETURN_UNEXPECTED_EXIT();
11817	}
11818
11819
11820	/**
11821	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
11822	*/
11823	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11824	{
11825	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11826	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11827	AssertRCReturn(rc, rc);
11828
11829	PVM pVM = pVCpu->CTX_SUFF(pVM);
11830	rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11831	if (RT_LIKELY(rc == VINF_SUCCESS))
11832	{
11833	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11834	Assert(pVmxTransient->cbInstr == 2);
11835	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
11836	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11837	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11838	}
11839	else
11840	rc = VERR_EM_INTERPRETER;
11841	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
11842	return rc;
11843	}
11844
11845
11846	/**
11847	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
11848	*/
11849	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11850	{
11851	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11852	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11853	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
11854	AssertRCReturn(rc, rc);
11855
11856	PVM pVM = pVCpu->CTX_SUFF(pVM);
11857	rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
11858	if (RT_SUCCESS(rc))
11859	{
11860	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11861	Assert(pVmxTransient->cbInstr == 3);
11862	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
11863	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11864	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11865	}
11866	else
11867	{
11868	AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
11869	rc = VERR_EM_INTERPRETER;
11870	}
11871	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
11872	return rc;
11873	}
11874
11875
11876	/**
11877	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
11878	*/
11879	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11880	{
11881	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11882	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11883	rc \|= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11884	AssertRCReturn(rc, rc);
11885
11886	PVM pVM = pVCpu->CTX_SUFF(pVM);
11887	rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11888	if (RT_LIKELY(rc == VINF_SUCCESS))
11889	{
11890	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11891	Assert(pVmxTransient->cbInstr == 2);
11892	}
11893	else
11894	{
11895	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
11896	rc = VERR_EM_INTERPRETER;
11897	}
11898	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
11899	return rc;
11900	}
11901
11902
11903	/**
11904	* VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
11905	*/
11906	HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11907	{
11908	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11909	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitVmcall);
11910
11911	VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
11912	if (pVCpu->hm.s.fHypercallsEnabled)
11913	{
11914	#if 0
11915	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11916	#else
11917	/* Aggressive state sync. for now. */
11918	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
11919	rc \|= hmR0VmxSaveGuestRflags(pVCpu,pMixedCtx); /* For CPL checks in gimHvHypercall() & gimKvmHypercall() */
11920	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* For long-mode checks in gimKvmHypercall(). */
11921	AssertRCReturn(rc, rc);
11922	#endif
11923
11924	/* Perform the hypercall. */
11925	rcStrict = GIMHypercall(pVCpu, pMixedCtx);
11926	if (rcStrict == VINF_SUCCESS)
11927	{
11928	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11929	AssertRCReturn(rc, rc);
11930	}
11931	else
11932	Assert( rcStrict == VINF_GIM_R3_HYPERCALL
11933	\|\| rcStrict == VINF_GIM_HYPERCALL_CONTINUING
11934	\|\| RT_FAILURE(VBOXSTRICTRC_VAL(rcStrict)));
11935
11936	/* If the hypercall changes anything other than guest's general-purpose registers,
11937	we would need to reload the guest changed bits here before VM-entry. */
11938	}
11939	else
11940	Log4(("hmR0VmxExitVmcall: Hypercalls not enabled\n"));
11941
11942	/* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
11943	if (RT_FAILURE(VBOXSTRICTRC_VAL(rcStrict)))
11944	{
11945	hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
11946	rcStrict = VINF_SUCCESS;
11947	}
11948
11949	return rcStrict;
11950	}
11951
11952
11953	/**
11954	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
11955	*/
11956	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11957	{
11958	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11959	PVM pVM = pVCpu->CTX_SUFF(pVM);
11960	Assert(!pVM->hm.s.fNestedPaging \|\| pVCpu->hm.s.fUsingDebugLoop);
11961
11962	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11963	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11964	AssertRCReturn(rc, rc);
11965
11966	VBOXSTRICTRC rcStrict = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
11967	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11968	rcStrict = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11969	else
11970	AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
11971	pVmxTransient->uExitQualification, VBOXSTRICTRC_VAL(rcStrict)));
11972	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
11973	return rcStrict;
11974	}
11975
11976
11977	/**
11978	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
11979	*/
11980	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11981	{
11982	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11983	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11984	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
11985	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11986	AssertRCReturn(rc, rc);
11987
11988	PVM pVM = pVCpu->CTX_SUFF(pVM);
11989	rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11990	if (RT_LIKELY(rc == VINF_SUCCESS))
11991	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11992	else
11993	{
11994	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
11995	rc = VERR_EM_INTERPRETER;
11996	}
11997	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
11998	return rc;
11999	}
12000
12001
12002	/**
12003	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
12004	*/
12005	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12006	{
12007	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12008	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
12009	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
12010	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12011	AssertRCReturn(rc, rc);
12012
12013	PVM pVM = pVCpu->CTX_SUFF(pVM);
12014	VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
12015	rc = VBOXSTRICTRC_VAL(rc2);
12016	if (RT_LIKELY( rc == VINF_SUCCESS
12017	\|\| rc == VINF_EM_HALT))
12018	{
12019	int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12020	AssertRCReturn(rc3, rc3);
12021
12022	if ( rc == VINF_EM_HALT
12023	&& EMMonitorWaitShouldContinue(pVCpu, pMixedCtx))
12024	{
12025	rc = VINF_SUCCESS;
12026	}
12027	}
12028	else
12029	{
12030	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
12031	rc = VERR_EM_INTERPRETER;
12032	}
12033	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_EM_HALT \|\| rc == VERR_EM_INTERPRETER,
12034	("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
12035	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
12036	return rc;
12037	}
12038
12039
12040	/**
12041	* VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
12042	*/
12043	HMVMX_EXIT_NSRC_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12044	{
12045	/*
12046	* Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
12047	* get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
12048	* executing VMCALL in VMX root operation. If we get here, something funny is going on.
12049	* See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
12050	*/
12051	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12052	AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12053	HMVMX_RETURN_UNEXPECTED_EXIT();
12054	}
12055
12056
12057	/**
12058	* VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
12059	*/
12060	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12061	{
12062	/*
12063	* This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
12064	* root operation. Only an STM (SMM transfer monitor) would get this VM-exit when we (the executive monitor) execute a VMCALL
12065	* in VMX root mode or receive an SMI. If we get here, something funny is going on.
12066	* See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
12067	*/
12068	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12069	AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12070	HMVMX_RETURN_UNEXPECTED_EXIT();
12071	}
12072
12073
12074	/**
12075	* VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
12076	*/
12077	HMVMX_EXIT_NSRC_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12078	{
12079	/* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
12080	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12081	AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12082	HMVMX_RETURN_UNEXPECTED_EXIT();
12083	}
12084
12085
12086	/**
12087	* VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
12088	*/
12089	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12090	{
12091	/*
12092	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
12093	* don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
12094	* See Intel spec. 25.3 "Other Causes of VM-exits".
12095	*/
12096	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12097	AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12098	HMVMX_RETURN_UNEXPECTED_EXIT();
12099	}
12100
12101
12102	/**
12103	* VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
12104	* VM-exit.
12105	*/
12106	HMVMX_EXIT_NSRC_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12107	{
12108	/*
12109	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
12110	* See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
12111	*
12112	* It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
12113	* See Intel spec. "23.8 Restrictions on VMX operation".
12114	*/
12115	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12116	return VINF_SUCCESS;
12117	}
12118
12119
12120	/**
12121	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
12122	* VM-exit.
12123	*/
12124	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12125	{
12126	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12127	return VINF_EM_RESET;
12128	}
12129
12130
12131	/**
12132	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
12133	*/
12134	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12135	{
12136	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12137	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
12138
12139	int rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12140	AssertRCReturn(rc, rc);
12141
12142	if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
12143	rc = VINF_SUCCESS;
12144	else
12145	rc = VINF_EM_HALT;
12146
12147	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
12148	if (rc != VINF_SUCCESS)
12149	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
12150	return rc;
12151	}
12152
12153
12154	/**
12155	* VM-exit handler for instructions that result in a \#UD exception delivered to
12156	* the guest.
12157	*/
12158	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12159	{
12160	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12161	hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
12162	return VINF_SUCCESS;
12163	}
12164
12165
12166	/**
12167	* VM-exit handler for expiry of the VMX preemption timer.
12168	*/
12169	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12170	{
12171	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12172
12173	/* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
12174	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
12175
12176	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
12177	PVM pVM = pVCpu->CTX_SUFF(pVM);
12178	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
12179	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
12180	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
12181	}
12182
12183
12184	/**
12185	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
12186	*/
12187	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12188	{
12189	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12190
12191	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12192	rc \|= hmR0VmxSaveGuestRegsForIemExec(pVCpu, pMixedCtx, false /fMemory/, false /fNeedRsp/);
12193	rc \|= hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
12194	AssertRCReturn(rc, rc);
12195
12196	VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbInstr);
12197	HMCPU_CF_SET(pVCpu, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS : HM_CHANGED_ALL_GUEST);
12198
12199	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
12200
12201	return rcStrict;
12202	}
12203
12204
12205	/**
12206	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
12207	*/
12208	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12209	{
12210	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12211	/** @todo Use VM-exit instruction information. */
12212	return VERR_EM_INTERPRETER;
12213	}
12214
12215
12216	/**
12217	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
12218	* Error VM-exit.
12219	*/
12220	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12221	{
12222	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12223	AssertRCReturn(rc, rc);
12224
12225	rc = hmR0VmxCheckVmcsCtls(pVCpu);
12226	AssertRCReturn(rc, rc);
12227
12228	uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
12229	NOREF(uInvalidReason);
12230
12231	#ifdef VBOX_STRICT
12232	uint32_t uIntrState;
12233	RTHCUINTREG uHCReg;
12234	uint64_t u64Val;
12235	uint32_t u32Val;
12236
12237	rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
12238	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
12239	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
12240	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
12241	AssertRCReturn(rc, rc);
12242
12243	Log4(("uInvalidReason %u\n", uInvalidReason));
12244	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
12245	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
12246	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
12247	Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
12248
12249	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
12250	Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
12251	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
12252	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
12253	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
12254	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
12255	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
12256	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
12257	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
12258	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
12259	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
12260	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
12261	#else
12262	NOREF(pVmxTransient);
12263	#endif
12264
12265	hmR0DumpRegs(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
12266	return VERR_VMX_INVALID_GUEST_STATE;
12267	}
12268
12269
12270	/**
12271	* VM-exit handler for VM-entry failure due to an MSR-load
12272	* (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
12273	*/
12274	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12275	{
12276	NOREF(pVmxTransient);
12277	AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
12278	HMVMX_RETURN_UNEXPECTED_EXIT();
12279	}
12280
12281
12282	/**
12283	* VM-exit handler for VM-entry failure due to a machine-check event
12284	* (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
12285	*/
12286	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12287	{
12288	NOREF(pVmxTransient);
12289	AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
12290	HMVMX_RETURN_UNEXPECTED_EXIT();
12291	}
12292
12293
12294	/**
12295	* VM-exit handler for all undefined reasons. Should never ever happen.. in
12296	* theory.
12297	*/
12298	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12299	{
12300	AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
12301	NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient);
12302	return VERR_VMX_UNDEFINED_EXIT_CODE;
12303	}
12304
12305
12306	/**
12307	* VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
12308	* (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
12309	* Conditional VM-exit.
12310	*/
12311	HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12312	{
12313	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12314
12315	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
12316	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
12317	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
12318	return VERR_EM_INTERPRETER;
12319	AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12320	HMVMX_RETURN_UNEXPECTED_EXIT();
12321	}
12322
12323
12324	/**
12325	* VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
12326	*/
12327	HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12328	{
12329	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12330
12331	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
12332	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
12333	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
12334	return VERR_EM_INTERPRETER;
12335	AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12336	HMVMX_RETURN_UNEXPECTED_EXIT();
12337	}
12338
12339
12340	/**
12341	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
12342	*/
12343	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12344	{
12345	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12346
12347	/* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
12348	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
12349	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
12350	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12351	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
12352	{
12353	rc \|= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
12354	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
12355	}
12356	AssertRCReturn(rc, rc);
12357	Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
12358
12359	#ifdef VBOX_STRICT
12360	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
12361	{
12362	if ( hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx)
12363	&& pMixedCtx->ecx != MSR_K6_EFER)
12364	{
12365	AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
12366	pMixedCtx->ecx));
12367	HMVMX_RETURN_UNEXPECTED_EXIT();
12368	}
12369	if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
12370	{
12371	VMXMSREXITREAD enmRead;
12372	VMXMSREXITWRITE enmWrite;
12373	int rc2 = hmR0VmxGetMsrPermission(pVCpu, pMixedCtx->ecx, &enmRead, &enmWrite);
12374	AssertRCReturn(rc2, rc2);
12375	if (enmRead == VMXMSREXIT_PASSTHRU_READ)
12376	{
12377	AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
12378	HMVMX_RETURN_UNEXPECTED_EXIT();
12379	}
12380	}
12381	}
12382	#endif
12383
12384	PVM pVM = pVCpu->CTX_SUFF(pVM);
12385	rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
12386	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER,
12387	("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
12388	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
12389	if (RT_SUCCESS(rc))
12390	{
12391	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12392	Assert(pVmxTransient->cbInstr == 2);
12393	}
12394	return rc;
12395	}
12396
12397
12398	/**
12399	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
12400	*/
12401	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12402	{
12403	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12404	PVM pVM = pVCpu->CTX_SUFF(pVM);
12405	int rc = VINF_SUCCESS;
12406
12407	/* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
12408	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
12409	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
12410	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12411	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
12412	{
12413	rc \|= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
12414	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
12415	}
12416	AssertRCReturn(rc, rc);
12417	Log4(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", pMixedCtx->ecx, pMixedCtx->edx, pMixedCtx->eax));
12418
12419	rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
12420	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
12421	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
12422
12423	if (RT_SUCCESS(rc))
12424	{
12425	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12426
12427	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
12428	if ( pMixedCtx->ecx == MSR_IA32_APICBASE
12429	\|\| ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
12430	&& pMixedCtx->ecx <= MSR_IA32_X2APIC_END))
12431	{
12432	/*
12433	* We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
12434	* virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
12435	* EMInterpretWrmsr() changes it.
12436	*/
12437	HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
12438	}
12439	else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
12440	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
12441	else if (pMixedCtx->ecx == MSR_K6_EFER)
12442	{
12443	/*
12444	* If the guest touches EFER we need to update the VM-Entry and VM-Exit controls as well,
12445	* even if it is -not- touching bits that cause paging mode changes (LMA/LME). We care about
12446	* the other bits as well, SCE and NXE. See @bugref{7368}.
12447	*/
12448	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_EFER_MSR \| HM_CHANGED_VMX_ENTRY_CTLS \| HM_CHANGED_VMX_EXIT_CTLS);
12449	}
12450
12451	/* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
12452	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
12453	{
12454	switch (pMixedCtx->ecx)
12455	{
12456	/*
12457	* For SYSENTER CS, EIP, ESP MSRs, we set both the flags here so we don't accidentally
12458	* overwrite the changed guest-CPU context value while going to ring-3, see @bufref{8745}.
12459	*/
12460	case MSR_IA32_SYSENTER_CS:
12461	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR);
12462	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR);
12463	break;
12464	case MSR_IA32_SYSENTER_EIP:
12465	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
12466	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR);
12467	break;
12468	case MSR_IA32_SYSENTER_ESP:
12469	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
12470	HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR);
12471	break;
12472	case MSR_K8_FS_BASE: RT_FALL_THRU();
12473	case MSR_K8_GS_BASE: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS); break;
12474	case MSR_K6_EFER: /* already handled above */ break;
12475	default:
12476	{
12477	if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
12478	HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
12479	else if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
12480	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
12481	break;
12482	}
12483	}
12484	}
12485	#ifdef VBOX_STRICT
12486	else
12487	{
12488	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
12489	switch (pMixedCtx->ecx)
12490	{
12491	case MSR_IA32_SYSENTER_CS:
12492	case MSR_IA32_SYSENTER_EIP:
12493	case MSR_IA32_SYSENTER_ESP:
12494	case MSR_K8_FS_BASE:
12495	case MSR_K8_GS_BASE:
12496	{
12497	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
12498	HMVMX_RETURN_UNEXPECTED_EXIT();
12499	}
12500
12501	/* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
12502	default:
12503	{
12504	if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
12505	{
12506	/* EFER writes are always intercepted, see hmR0VmxLoadGuestMsrs(). */
12507	if (pMixedCtx->ecx != MSR_K6_EFER)
12508	{
12509	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
12510	pMixedCtx->ecx));
12511	HMVMX_RETURN_UNEXPECTED_EXIT();
12512	}
12513	}
12514
12515	if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
12516	{
12517	VMXMSREXITREAD enmRead;
12518	VMXMSREXITWRITE enmWrite;
12519	int rc2 = hmR0VmxGetMsrPermission(pVCpu, pMixedCtx->ecx, &enmRead, &enmWrite);
12520	AssertRCReturn(rc2, rc2);
12521	if (enmWrite == VMXMSREXIT_PASSTHRU_WRITE)
12522	{
12523	AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
12524	HMVMX_RETURN_UNEXPECTED_EXIT();
12525	}
12526	}
12527	break;
12528	}
12529	}
12530	}
12531	#endif /* VBOX_STRICT */
12532	}
12533	return rc;
12534	}
12535
12536
12537	/**
12538	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
12539	*/
12540	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12541	{
12542	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12543
12544	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
12545	return VINF_EM_RAW_INTERRUPT;
12546	}
12547
12548
12549	/**
12550	* VM-exit handler for when the TPR value is lowered below the specified
12551	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
12552	*/
12553	HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12554	{
12555	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12556	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
12557
12558	/*
12559	* The TPR shadow would've been synced with the APIC TPR in hmR0VmxPostRunGuest(). We'll re-evaluate
12560	* pending interrupts and inject them before the next VM-entry so we can just continue execution here.
12561	*/
12562	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
12563	return VINF_SUCCESS;
12564	}
12565
12566
12567	/**
12568	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
12569	* VM-exit.
12570	*
12571	* @retval VINF_SUCCESS when guest execution can continue.
12572	* @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
12573	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
12574	* @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
12575	* interpreter.
12576	*/
12577	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12578	{
12579	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12580	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
12581	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12582	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12583	AssertRCReturn(rc, rc);
12584
12585	RTGCUINTPTR const uExitQualification = pVmxTransient->uExitQualification;
12586	uint32_t const uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
12587	PVM pVM = pVCpu->CTX_SUFF(pVM);
12588	VBOXSTRICTRC rcStrict;
12589	rc = hmR0VmxSaveGuestRegsForIemExec(pVCpu, pMixedCtx, false /fMemory/, true /fNeedRsp/);
12590	switch (uAccessType)
12591	{
12592	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
12593	{
12594	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12595	AssertRCReturn(rc, rc);
12596
12597	rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, pVmxTransient->cbInstr,
12598	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
12599	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
12600	AssertMsg( rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_IEM_RAISED_XCPT \|\| rcStrict == VINF_PGM_CHANGE_MODE
12601	\|\| rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12602	switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
12603	{
12604	case 0: /* CR0 */
12605	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
12606	Log4(("CRX CR0 write rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr0));
12607	break;
12608	case 2: /* CR2 */
12609	/* Nothing to do here, CR2 it's not part of the VMCS. */
12610	break;
12611	case 3: /* CR3 */
12612	Assert(!pVM->hm.s.fNestedPaging \|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx) \|\| pVCpu->hm.s.fUsingDebugLoop);
12613	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR3);
12614	Log4(("CRX CR3 write rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr3));
12615	break;
12616	case 4: /* CR4 */
12617	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
12618	Log4(("CRX CR4 write rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n",
12619	VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
12620	break;
12621	case 8: /* CR8 */
12622	Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
12623	/* CR8 contains the APIC TPR. Was updated by IEMExecDecodedMovCRxWrite(). */
12624	HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
12625	break;
12626	default:
12627	AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
12628	break;
12629	}
12630
12631	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
12632	break;
12633	}
12634
12635	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
12636	{
12637	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12638	AssertRCReturn(rc, rc);
12639
12640	Assert( !pVM->hm.s.fNestedPaging
12641	\|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx)
12642	\|\| pVCpu->hm.s.fUsingDebugLoop
12643	\|\| VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
12644
12645	/* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
12646	Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
12647	\|\| !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
12648
12649	rcStrict = IEMExecDecodedMovCRxRead(pVCpu, pVmxTransient->cbInstr,
12650	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
12651	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
12652	AssertMsg(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12653	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
12654	Log4(("CRX CR%d Read access rcStrict=%Rrc\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
12655	VBOXSTRICTRC_VAL(rcStrict)));
12656	if (VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification) == X86_GREG_xSP)
12657	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RSP);
12658	break;
12659	}
12660
12661	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
12662	{
12663	AssertRCReturn(rc, rc);
12664	rcStrict = IEMExecDecodedClts(pVCpu, pVmxTransient->cbInstr);
12665	AssertMsg(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12666	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
12667	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
12668	Log4(("CRX CLTS rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12669	break;
12670	}
12671
12672	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
12673	{
12674	AssertRCReturn(rc, rc);
12675	rcStrict = IEMExecDecodedLmsw(pVCpu, pVmxTransient->cbInstr,
12676	VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
12677	AssertMsg(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_IEM_RAISED_XCPT \|\| rcStrict == VINF_PGM_CHANGE_MODE,
12678	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12679	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
12680	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
12681	Log4(("CRX LMSW rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12682	break;
12683	}
12684
12685	default:
12686	AssertMsgFailedReturn(("Invalid access-type in Mov CRx VM-exit qualification %#x\n", uAccessType),
12687	VERR_VMX_UNEXPECTED_EXCEPTION);
12688	}
12689
12690	HMCPU_CF_SET(pVCpu, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS : HM_CHANGED_ALL_GUEST);
12691	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
12692	NOREF(pVM);
12693	return rcStrict;
12694	}
12695
12696
12697	/**
12698	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
12699	* VM-exit.
12700	*/
12701	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12702	{
12703	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12704	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
12705
12706	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12707	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12708	rc \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
12709	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
12710	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
12711	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
12712	/* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
12713	AssertRCReturn(rc, rc);
12714
12715	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
12716	uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
12717	uint8_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
12718	bool fIOWrite = ( VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
12719	== VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
12720	bool fIOString = VMX_EXIT_QUALIFICATION_IO_IS_STRING(pVmxTransient->uExitQualification);
12721	bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
12722	bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
12723	AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_VMX_IPE_1);
12724
12725	/* I/O operation lookup arrays. */
12726	static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
12727	static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
12728
12729	VBOXSTRICTRC rcStrict;
12730	uint32_t const cbValue = s_aIOSizes[uIOWidth];
12731	uint32_t const cbInstr = pVmxTransient->cbInstr;
12732	bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
12733	PVM pVM = pVCpu->CTX_SUFF(pVM);
12734	if (fIOString)
12735	{
12736	#ifdef VBOX_WITH_2ND_IEM_STEP /* This used to gurus with debian 32-bit guest without NP (on ATA reads).
12737	See @bugref{5752#c158}. Should work now. */
12738	/*
12739	* INS/OUTS - I/O String instruction.
12740	*
12741	* Use instruction-information if available, otherwise fall back on
12742	* interpreting the instruction.
12743	*/
12744	Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue,
12745	fIOWrite ? 'w' : 'r'));
12746	AssertReturn(pMixedCtx->dx == uIOPort, VERR_VMX_IPE_2);
12747	if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
12748	{
12749	int rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
12750	/** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
12751	rc2 \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12752	AssertRCReturn(rc2, rc2);
12753	AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
12754	AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
12755	IEMMODE enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
12756	bool fRep = VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification);
12757	if (fIOWrite)
12758	{
12759	rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
12760	pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /fIoChecked/);
12761	}
12762	else
12763	{
12764	/*
12765	* The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
12766	* Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
12767	* See Intel Instruction spec. for "INS".
12768	* See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
12769	*/
12770	rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /fIoChecked/);
12771	}
12772	}
12773	else
12774	{
12775	/** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
12776	int rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12777	AssertRCReturn(rc2, rc2);
12778	rcStrict = IEMExecOne(pVCpu);
12779	}
12780	/** @todo IEM needs to be setting these flags somehow. */
12781	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
12782	fUpdateRipAlready = true;
12783	#else
12784	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
12785	rcStrict = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL /* pcbInstr */);
12786	if (RT_SUCCESS(rcStrict))
12787	{
12788	if (fIOWrite)
12789	{
12790	rcStrict = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
12791	(DISCPUMODE)pDis->uAddrMode, cbValue);
12792	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
12793	}
12794	else
12795	{
12796	rcStrict = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
12797	(DISCPUMODE)pDis->uAddrMode, cbValue);
12798	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
12799	}
12800	}
12801	else
12802	{
12803	AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("rcStrict=%Rrc RIP=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict),
12804	pMixedCtx->rip));
12805	rcStrict = VINF_EM_RAW_EMULATE_INSTR;
12806	}
12807	#endif
12808	}
12809	else
12810	{
12811	/*
12812	* IN/OUT - I/O instruction.
12813	*/
12814	Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
12815	uint32_t const uAndVal = s_aIOOpAnd[uIOWidth];
12816	Assert(!VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification));
12817	if (fIOWrite)
12818	{
12819	rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
12820	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
12821	}
12822	else
12823	{
12824	uint32_t u32Result = 0;
12825	rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
12826	if (IOM_SUCCESS(rcStrict))
12827	{
12828	/* Save result of I/O IN instr. in AL/AX/EAX. */
12829	pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
12830	}
12831	else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
12832	HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
12833	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
12834	}
12835	}
12836
12837	if (IOM_SUCCESS(rcStrict))
12838	{
12839	if (!fUpdateRipAlready)
12840	{
12841	hmR0VmxAdvanceGuestRipBy(pVCpu, pMixedCtx, cbInstr);
12842	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
12843	}
12844
12845	/*
12846	* INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru while booting Fedora 17 64-bit guest.
12847	* See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
12848	*/
12849	if (fIOString)
12850	{
12851	/** @todo Single-step for INS/OUTS with REP prefix? */
12852	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
12853	}
12854	else if ( !fDbgStepping
12855	&& fGstStepping)
12856	{
12857	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
12858	}
12859
12860	/*
12861	* If any I/O breakpoints are armed, we need to check if one triggered
12862	* and take appropriate action.
12863	* Note that the I/O breakpoint type is undefined if CR4.DE is 0.
12864	*/
12865	int rc2 = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
12866	AssertRCReturn(rc2, rc2);
12867
12868	/** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
12869	* execution engines about whether hyper BPs and such are pending. */
12870	uint32_t const uDr7 = pMixedCtx->dr[7];
12871	if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
12872	&& X86_DR7_ANY_RW_IO(uDr7)
12873	&& (pMixedCtx->cr4 & X86_CR4_DE))
12874	\|\| DBGFBpIsHwIoArmed(pVM)))
12875	{
12876	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
12877
12878	/* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
12879	VMMRZCallRing3Disable(pVCpu);
12880	HM_DISABLE_PREEMPT();
12881
12882	bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
12883
12884	VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
12885	if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
12886	{
12887	/* Raise #DB. */
12888	if (fIsGuestDbgActive)
12889	ASMSetDR6(pMixedCtx->dr[6]);
12890	if (pMixedCtx->dr[7] != uDr7)
12891	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
12892
12893	hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
12894	}
12895	/* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
12896	however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
12897	else if ( rcStrict2 != VINF_SUCCESS
12898	&& (rcStrict == VINF_SUCCESS \|\| rcStrict2 < rcStrict))
12899	rcStrict = rcStrict2;
12900	AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
12901
12902	HM_RESTORE_PREEMPT();
12903	VMMRZCallRing3Enable(pVCpu);
12904	}
12905	}
12906
12907	#ifdef VBOX_STRICT
12908	if (rcStrict == VINF_IOM_R3_IOPORT_READ)
12909	Assert(!fIOWrite);
12910	else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE \|\| rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE)
12911	Assert(fIOWrite);
12912	else
12913	{
12914	#if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
12915	* statuses, that the VMM device and some others may return. See
12916	* IOM_SUCCESS() for guidance. */
12917	AssertMsg( RT_FAILURE(rcStrict)
12918	\|\| rcStrict == VINF_SUCCESS
12919	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
12920	\|\| rcStrict == VINF_EM_DBG_BREAKPOINT
12921	\|\| rcStrict == VINF_EM_RAW_GUEST_TRAP
12922	\|\| rcStrict == VINF_EM_RAW_TO_R3
12923	\|\| rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12924	#endif
12925	}
12926	#endif
12927
12928	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
12929	return rcStrict;
12930	}
12931
12932
12933	/**
12934	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
12935	* VM-exit.
12936	*/
12937	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12938	{
12939	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12940
12941	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
12942	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12943	AssertRCReturn(rc, rc);
12944	if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
12945	{
12946	rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
12947	AssertRCReturn(rc, rc);
12948	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
12949	{
12950	uint32_t uErrCode;
12951	RTGCUINTPTR GCPtrFaultAddress;
12952	uint32_t const uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
12953	uint32_t const uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
12954	bool const fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
12955	if (fErrorCodeValid)
12956	{
12957	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
12958	AssertRCReturn(rc, rc);
12959	uErrCode = pVmxTransient->uIdtVectoringErrorCode;
12960	}
12961	else
12962	uErrCode = 0;
12963
12964	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
12965	&& uVector == X86_XCPT_PF)
12966	GCPtrFaultAddress = pMixedCtx->cr2;
12967	else
12968	GCPtrFaultAddress = 0;
12969
12970	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
12971	0 /* cbInstr */, uErrCode, GCPtrFaultAddress);
12972
12973	Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
12974	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12975	return VINF_EM_RAW_INJECT_TRPM_EVENT;
12976	}
12977	}
12978
12979	/* Fall back to the interpreter to emulate the task-switch. */
12980	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12981	return VERR_EM_INTERPRETER;
12982	}
12983
12984
12985	/**
12986	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
12987	*/
12988	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12989	{
12990	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12991	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
12992	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
12993	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12994	AssertRCReturn(rc, rc);
12995	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
12996	return VINF_EM_DBG_STEPPED;
12997	}
12998
12999
13000	/**
13001	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
13002	*/
13003	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13004	{
13005	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
13006
13007	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
13008
13009	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
13010	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
13011	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
13012	{
13013	/* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
13014	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
13015	{
13016	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
13017	return VINF_EM_RAW_INJECT_TRPM_EVENT;
13018	}
13019	}
13020	else
13021	{
13022	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
13023	rcStrict1 = VINF_SUCCESS;
13024	return rcStrict1;
13025	}
13026
13027	#if 0
13028	/** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
13029	* just sync the whole thing. */
13030	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13031	#else
13032	/* Aggressive state sync. for now. */
13033	int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
13034	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
13035	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
13036	#endif
13037	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13038	AssertRCReturn(rc, rc);
13039
13040	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
13041	uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
13042	VBOXSTRICTRC rcStrict2;
13043	switch (uAccessType)
13044	{
13045	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
13046	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
13047	{
13048	AssertMsg( !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
13049	\|\| VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) != XAPIC_OFF_TPR,
13050	("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
13051
13052	RTGCPHYS GCPhys = pVCpu->hm.s.vmx.u64MsrApicBase; /* Always up-to-date, u64MsrApicBase is not part of the VMCS. */
13053	GCPhys &= PAGE_BASE_GC_MASK;
13054	GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
13055	PVM pVM = pVCpu->CTX_SUFF(pVM);
13056	Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
13057	VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
13058
13059	rcStrict2 = IOMMMIOPhysHandler(pVM, pVCpu,
13060	uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
13061	CPUMCTX2CORE(pMixedCtx), GCPhys);
13062	Log4(("ApicAccess rcStrict2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
13063	if ( rcStrict2 == VINF_SUCCESS
13064	\|\| rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
13065	\|\| rcStrict2 == VERR_PAGE_NOT_PRESENT)
13066	{
13067	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13068	\| HM_CHANGED_GUEST_RSP
13069	\| HM_CHANGED_GUEST_RFLAGS
13070	\| HM_CHANGED_VMX_GUEST_APIC_STATE);
13071	rcStrict2 = VINF_SUCCESS;
13072	}
13073	break;
13074	}
13075
13076	default:
13077	Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
13078	rcStrict2 = VINF_EM_RAW_EMULATE_INSTR;
13079	break;
13080	}
13081
13082	if (rcStrict2 != VINF_SUCCESS)
13083	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
13084	return rcStrict2;
13085	}
13086
13087
13088	/**
13089	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
13090	* VM-exit.
13091	*/
13092	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13093	{
13094	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
13095
13096	/* We should -not- get this VM-exit if the guest's debug registers were active. */
13097	if (pVmxTransient->fWasGuestDebugStateActive)
13098	{
13099	AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
13100	HMVMX_RETURN_UNEXPECTED_EXIT();
13101	}
13102
13103	if ( !pVCpu->hm.s.fSingleInstruction
13104	&& !pVmxTransient->fWasHyperDebugStateActive)
13105	{
13106	Assert(!DBGFIsStepping(pVCpu));
13107	Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
13108
13109	/* Don't intercept MOV DRx any more. */
13110	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
13111	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
13112	AssertRCReturn(rc, rc);
13113
13114	/* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
13115	VMMRZCallRing3Disable(pVCpu);
13116	HM_DISABLE_PREEMPT();
13117
13118	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
13119	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
13120	Assert(CPUMIsGuestDebugStateActive(pVCpu) \|\| HC_ARCH_BITS == 32);
13121
13122	HM_RESTORE_PREEMPT();
13123	VMMRZCallRing3Enable(pVCpu);
13124
13125	#ifdef VBOX_WITH_STATISTICS
13126	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13127	AssertRCReturn(rc, rc);
13128	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
13129	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
13130	else
13131	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
13132	#endif
13133	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
13134	return VINF_SUCCESS;
13135	}
13136
13137	/*
13138	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
13139	* Update the segment registers and DR7 from the CPU.
13140	*/
13141	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13142	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
13143	rc \|= hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
13144	AssertRCReturn(rc, rc);
13145	Log4(("CS:RIP=%04x:%08RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
13146
13147	PVM pVM = pVCpu->CTX_SUFF(pVM);
13148	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
13149	{
13150	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
13151	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
13152	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
13153	if (RT_SUCCESS(rc))
13154	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
13155	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
13156	}
13157	else
13158	{
13159	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
13160	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
13161	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
13162	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
13163	}
13164
13165	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
13166	if (RT_SUCCESS(rc))
13167	{
13168	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
13169	AssertRCReturn(rc2, rc2);
13170	return VINF_SUCCESS;
13171	}
13172	return rc;
13173	}
13174
13175
13176	/**
13177	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
13178	* Conditional VM-exit.
13179	*/
13180	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13181	{
13182	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
13183	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
13184
13185	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
13186	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
13187	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
13188	{
13189	/* If event delivery causes an EPT misconfig (MMIO), go back to instruction emulation as otherwise
13190	injecting the original pending event would most likely cause the same EPT misconfig VM-exit. */
13191	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
13192	{
13193	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
13194	return VINF_EM_RAW_INJECT_TRPM_EVENT;
13195	}
13196	}
13197	else
13198	{
13199	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
13200	rcStrict1 = VINF_SUCCESS;
13201	return rcStrict1;
13202	}
13203
13204	RTGCPHYS GCPhys = 0;
13205	int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
13206
13207	#if 0
13208	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
13209	#else
13210	/* Aggressive state sync. for now. */
13211	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
13212	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
13213	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
13214	#endif
13215	AssertRCReturn(rc, rc);
13216
13217	/*
13218	* If we succeed, resume guest execution.
13219	* If we fail in interpreting the instruction because we couldn't get the guest physical address
13220	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
13221	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
13222	* weird case. See @bugref{6043}.
13223	*/
13224	PVM pVM = pVCpu->CTX_SUFF(pVM);
13225	VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
13226	Log4(("EPT misconfig at %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pMixedCtx->rip, VBOXSTRICTRC_VAL(rcStrict2)));
13227	if ( rcStrict2 == VINF_SUCCESS
13228	\|\| rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
13229	\|\| rcStrict2 == VERR_PAGE_NOT_PRESENT)
13230	{
13231	/* Successfully handled MMIO operation. */
13232	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13233	\| HM_CHANGED_GUEST_RSP
13234	\| HM_CHANGED_GUEST_RFLAGS
13235	\| HM_CHANGED_VMX_GUEST_APIC_STATE);
13236	return VINF_SUCCESS;
13237	}
13238	return rcStrict2;
13239	}
13240
13241
13242	/**
13243	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
13244	* VM-exit.
13245	*/
13246	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13247	{
13248	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
13249	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
13250
13251	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
13252	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
13253	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
13254	{
13255	/* In the unlikely case that the EPT violation happened as a result of delivering an event, log it. */
13256	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
13257	Log4(("EPT violation with an event pending u64IntInfo=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo));
13258	}
13259	else
13260	{
13261	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
13262	rcStrict1 = VINF_SUCCESS;
13263	return rcStrict1;
13264	}
13265
13266	RTGCPHYS GCPhys = 0;
13267	int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
13268	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13269	#if 0
13270	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
13271	#else
13272	/* Aggressive state sync. for now. */
13273	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
13274	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
13275	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
13276	#endif
13277	AssertRCReturn(rc, rc);
13278
13279	/* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
13280	AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
13281
13282	RTGCUINT uErrorCode = 0;
13283	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
13284	uErrorCode \|= X86_TRAP_PF_ID;
13285	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
13286	uErrorCode \|= X86_TRAP_PF_RW;
13287	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
13288	uErrorCode \|= X86_TRAP_PF_P;
13289
13290	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
13291
13292	Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:RIP=%04x:%08RX64\n", pVmxTransient->uExitQualification, GCPhys,
13293	uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
13294
13295	/* Handle the pagefault trap for the nested shadow table. */
13296	PVM pVM = pVCpu->CTX_SUFF(pVM);
13297	VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
13298	TRPMResetTrap(pVCpu);
13299
13300	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
13301	if ( rcStrict2 == VINF_SUCCESS
13302	\|\| rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
13303	\|\| rcStrict2 == VERR_PAGE_NOT_PRESENT)
13304	{
13305	/* Successfully synced our nested page tables. */
13306	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
13307	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13308	\| HM_CHANGED_GUEST_RSP
13309	\| HM_CHANGED_GUEST_RFLAGS);
13310	return VINF_SUCCESS;
13311	}
13312
13313	Log4(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
13314	return rcStrict2;
13315	}
13316
13317	/** @} */
13318
13319	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13320	/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
13321	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13322
13323	/** @name VM-exit exception handlers.
13324	* @{
13325	*/
13326
13327	/**
13328	* VM-exit exception handler for \#MF (Math Fault: floating point exception).
13329	*/
13330	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13331	{
13332	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13333	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
13334
13335	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
13336	AssertRCReturn(rc, rc);
13337
13338	if (!(pMixedCtx->cr0 & X86_CR0_NE))
13339	{
13340	/* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
13341	rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
13342
13343	/** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
13344	* provides VM-exit instruction length. If this causes problem later,
13345	* disassemble the instruction like it's done on AMD-V. */
13346	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
13347	AssertRCReturn(rc2, rc2);
13348	return rc;
13349	}
13350
13351	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13352	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13353	return rc;
13354	}
13355
13356
13357	/**
13358	* VM-exit exception handler for \#BP (Breakpoint exception).
13359	*/
13360	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13361	{
13362	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13363	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
13364
13365	/** @todo Try optimize this by not saving the entire guest state unless
13366	* really needed. */
13367	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13368	AssertRCReturn(rc, rc);
13369
13370	PVM pVM = pVCpu->CTX_SUFF(pVM);
13371	rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
13372	if (rc == VINF_EM_RAW_GUEST_TRAP)
13373	{
13374	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13375	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13376	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13377	AssertRCReturn(rc, rc);
13378
13379	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13380	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13381	}
13382
13383	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_EM_DBG_BREAKPOINT);
13384	return rc;
13385	}
13386
13387
13388	/**
13389	* VM-exit exception handler for \#AC (alignment check exception).
13390	*/
13391	static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13392	{
13393	RT_NOREF_PV(pMixedCtx);
13394	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13395
13396	/*
13397	* Re-inject it. We'll detect any nesting before getting here.
13398	*/
13399	int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13400	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13401	AssertRCReturn(rc, rc);
13402	Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
13403
13404	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13405	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13406	return VINF_SUCCESS;
13407	}
13408
13409
13410	/**
13411	* VM-exit exception handler for \#DB (Debug exception).
13412	*/
13413	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13414	{
13415	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13416	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
13417	Log6(("XcptDB\n"));
13418
13419	/*
13420	* Get the DR6-like values from the VM-exit qualification and pass it to DBGF
13421	* for processing.
13422	*/
13423	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13424	AssertRCReturn(rc, rc);
13425
13426	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
13427	uint64_t uDR6 = X86_DR6_INIT_VAL;
13428	uDR6 \|= ( pVmxTransient->uExitQualification
13429	& (X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3 \| X86_DR6_BD \| X86_DR6_BS));
13430
13431	rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
13432	if (rc == VINF_EM_RAW_GUEST_TRAP)
13433	{
13434	/*
13435	* The exception was for the guest. Update DR6, DR7.GD and
13436	* IA32_DEBUGCTL.LBR before forwarding it.
13437	* (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
13438	*/
13439	VMMRZCallRing3Disable(pVCpu);
13440	HM_DISABLE_PREEMPT();
13441
13442	pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
13443	pMixedCtx->dr[6] \|= uDR6;
13444	if (CPUMIsGuestDebugStateActive(pVCpu))
13445	ASMSetDR6(pMixedCtx->dr[6]);
13446
13447	HM_RESTORE_PREEMPT();
13448	VMMRZCallRing3Enable(pVCpu);
13449
13450	rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
13451	AssertRCReturn(rc, rc);
13452
13453	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
13454	pMixedCtx->dr[7] &= ~X86_DR7_GD;
13455
13456	/* Paranoia. */
13457	pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
13458	pMixedCtx->dr[7] \|= X86_DR7_RA1_MASK;
13459
13460	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
13461	AssertRCReturn(rc, rc);
13462
13463	/*
13464	* Raise #DB in the guest.
13465	*
13466	* It is important to reflect what the VM-exit gave us (preserving the interruption-type) rather than use
13467	* hmR0VmxSetPendingXcptDB() as the #DB could've been raised while executing ICEBP and not the 'normal' #DB.
13468	* Thus it -may- trigger different handling in the CPU (like skipped DPL checks). See @bugref{6398}.
13469	*
13470	* Since ICEBP isn't documented on Intel, see AMD spec. 15.20 "Event Injection".
13471	*/
13472	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13473	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13474	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13475	AssertRCReturn(rc, rc);
13476	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13477	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13478	return VINF_SUCCESS;
13479	}
13480
13481	/*
13482	* Not a guest trap, must be a hypervisor related debug event then.
13483	* Update DR6 in case someone is interested in it.
13484	*/
13485	AssertMsg(rc == VINF_EM_DBG_STEPPED \|\| rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
13486	AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
13487	CPUMSetHyperDR6(pVCpu, uDR6);
13488
13489	return rc;
13490	}
13491
13492
13493	/**
13494	* VM-exit exception handler for \#NM (Device-not-available exception: floating
13495	* point exception).
13496	*/
13497	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13498	{
13499	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13500
13501	/* We require CR0 and EFER. EFER is always up-to-date. */
13502	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
13503	AssertRCReturn(rc, rc);
13504
13505	/* We're playing with the host CPU state here, have to disable preemption or longjmp. */
13506	VMMRZCallRing3Disable(pVCpu);
13507	HM_DISABLE_PREEMPT();
13508
13509	/* If the guest FPU was active at the time of the #NM VM-exit, then it's a guest fault. */
13510	if (pVmxTransient->fWasGuestFPUStateActive)
13511	{
13512	rc = VINF_EM_RAW_GUEST_TRAP;
13513	Assert(CPUMIsGuestFPUStateActive(pVCpu) \|\| HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0));
13514	}
13515	else
13516	{
13517	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13518	Assert(!pVmxTransient->fWasGuestFPUStateActive \|\| pVCpu->hm.s.fUsingDebugLoop);
13519	#endif
13520	rc = CPUMR0Trap07Handler(pVCpu->CTX_SUFF(pVM), pVCpu);
13521	Assert( rc == VINF_EM_RAW_GUEST_TRAP
13522	\|\| ((rc == VINF_SUCCESS \|\| rc == VINF_CPUM_HOST_CR0_MODIFIED) && CPUMIsGuestFPUStateActive(pVCpu)));
13523	if (rc == VINF_CPUM_HOST_CR0_MODIFIED)
13524	HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT);
13525	}
13526
13527	HM_RESTORE_PREEMPT();
13528	VMMRZCallRing3Enable(pVCpu);
13529
13530	if (rc == VINF_SUCCESS \|\| rc == VINF_CPUM_HOST_CR0_MODIFIED)
13531	{
13532	/* Guest FPU state was activated, we'll want to change CR0 FPU intercepts before the next VM-reentry. */
13533	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
13534	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
13535	pVCpu->hm.s.fPreloadGuestFpu = true;
13536	}
13537	else
13538	{
13539	/* Forward #NM to the guest. */
13540	Assert(rc == VINF_EM_RAW_GUEST_TRAP);
13541	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13542	AssertRCReturn(rc, rc);
13543	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13544	pVmxTransient->cbInstr, 0 /* error code /, 0 / GCPtrFaultAddress */);
13545	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
13546	}
13547
13548	return VINF_SUCCESS;
13549	}
13550
13551
13552	/**
13553	* VM-exit exception handler for \#GP (General-protection exception).
13554	*
13555	* @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
13556	*/
13557	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13558	{
13559	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13560	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
13561
13562	int rc;
13563	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
13564	{ /* likely */ }
13565	else
13566	{
13567	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13568	Assert(pVCpu->hm.s.fUsingDebugLoop);
13569	#endif
13570	/* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
13571	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13572	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13573	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13574	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13575	AssertRCReturn(rc, rc);
13576	Log4(("#GP Gst: CS:RIP %04x:%08RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u TR=%#04x\n", pMixedCtx->cs.Sel, pMixedCtx->rip,
13577	pVmxTransient->uExitIntErrorCode, pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu), pMixedCtx->tr.Sel));
13578	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13579	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13580	return rc;
13581	}
13582
13583	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
13584	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
13585
13586	/* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
13587	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13588	AssertRCReturn(rc, rc);
13589
13590	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
13591	uint32_t cbOp = 0;
13592	PVM pVM = pVCpu->CTX_SUFF(pVM);
13593	bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
13594	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
13595	if (RT_SUCCESS(rc))
13596	{
13597	rc = VINF_SUCCESS;
13598	Assert(cbOp == pDis->cbInstr);
13599	Log4(("#GP Disas OpCode=%u CS:EIP %04x:%04RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
13600	switch (pDis->pCurInstr->uOpcode)
13601	{
13602	case OP_CLI:
13603	{
13604	pMixedCtx->eflags.Bits.u1IF = 0;
13605	pMixedCtx->eflags.Bits.u1RF = 0;
13606	pMixedCtx->rip += pDis->cbInstr;
13607	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13608	if ( !fDbgStepping
13609	&& pMixedCtx->eflags.Bits.u1TF)
13610	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13611	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
13612	break;
13613	}
13614
13615	case OP_STI:
13616	{
13617	bool fOldIF = pMixedCtx->eflags.Bits.u1IF;
13618	pMixedCtx->eflags.Bits.u1IF = 1;
13619	pMixedCtx->eflags.Bits.u1RF = 0;
13620	pMixedCtx->rip += pDis->cbInstr;
13621	if (!fOldIF)
13622	{
13623	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
13624	Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
13625	}
13626	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13627	if ( !fDbgStepping
13628	&& pMixedCtx->eflags.Bits.u1TF)
13629	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13630	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
13631	break;
13632	}
13633
13634	case OP_HLT:
13635	{
13636	rc = VINF_EM_HALT;
13637	pMixedCtx->rip += pDis->cbInstr;
13638	pMixedCtx->eflags.Bits.u1RF = 0;
13639	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13640	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
13641	break;
13642	}
13643
13644	case OP_POPF:
13645	{
13646	Log4(("POPF CS:EIP %04x:%04RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
13647	uint32_t cbParm;
13648	uint32_t uMask;
13649	bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
13650	if (pDis->fPrefix & DISPREFIX_OPSIZE)
13651	{
13652	cbParm = 4;
13653	uMask = 0xffffffff;
13654	}
13655	else
13656	{
13657	cbParm = 2;
13658	uMask = 0xffff;
13659	}
13660
13661	/* Get the stack pointer & pop the contents of the stack onto Eflags. */
13662	RTGCPTR GCPtrStack = 0;
13663	X86EFLAGS Eflags;
13664	Eflags.u32 = 0;
13665	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
13666	&GCPtrStack);
13667	if (RT_SUCCESS(rc))
13668	{
13669	Assert(sizeof(Eflags.u32) >= cbParm);
13670	rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm, PGMACCESSORIGIN_HM));
13671	AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13672	}
13673	if (RT_FAILURE(rc))
13674	{
13675	rc = VERR_EM_INTERPRETER;
13676	break;
13677	}
13678	Log4(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
13679	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~((X86_EFL_POPF_BITS & uMask) \| X86_EFL_RF))
13680	\| (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
13681	pMixedCtx->esp += cbParm;
13682	pMixedCtx->esp &= uMask;
13683	pMixedCtx->rip += pDis->cbInstr;
13684	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13685	\| HM_CHANGED_GUEST_RSP
13686	\| HM_CHANGED_GUEST_RFLAGS);
13687	/* Generate a pending-debug exception when the guest stepping over POPF regardless of how
13688	POPF restores EFLAGS.TF. */
13689	if ( !fDbgStepping
13690	&& fGstStepping)
13691	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13692	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
13693	break;
13694	}
13695
13696	case OP_PUSHF:
13697	{
13698	uint32_t cbParm;
13699	uint32_t uMask;
13700	if (pDis->fPrefix & DISPREFIX_OPSIZE)
13701	{
13702	cbParm = 4;
13703	uMask = 0xffffffff;
13704	}
13705	else
13706	{
13707	cbParm = 2;
13708	uMask = 0xffff;
13709	}
13710
13711	/* Get the stack pointer & push the contents of eflags onto the stack. */
13712	RTGCPTR GCPtrStack = 0;
13713	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
13714	SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
13715	if (RT_FAILURE(rc))
13716	{
13717	rc = VERR_EM_INTERPRETER;
13718	break;
13719	}
13720	X86EFLAGS Eflags = pMixedCtx->eflags;
13721	/* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
13722	Eflags.Bits.u1RF = 0;
13723	Eflags.Bits.u1VM = 0;
13724
13725	rc = VBOXSTRICTRC_TODO(PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm, PGMACCESSORIGIN_HM));
13726	if (RT_UNLIKELY(rc != VINF_SUCCESS))
13727	{
13728	AssertMsgFailed(("%Rrc\n", rc)); /** @todo allow strict return codes here */
13729	rc = VERR_EM_INTERPRETER;
13730	break;
13731	}
13732	Log4(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
13733	pMixedCtx->esp -= cbParm;
13734	pMixedCtx->esp &= uMask;
13735	pMixedCtx->rip += pDis->cbInstr;
13736	pMixedCtx->eflags.Bits.u1RF = 0;
13737	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13738	\| HM_CHANGED_GUEST_RSP
13739	\| HM_CHANGED_GUEST_RFLAGS);
13740	if ( !fDbgStepping
13741	&& pMixedCtx->eflags.Bits.u1TF)
13742	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13743	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
13744	break;
13745	}
13746
13747	case OP_IRET:
13748	{
13749	/** @todo Handle 32-bit operand sizes and check stack limits. See Intel
13750	* instruction reference. */
13751	RTGCPTR GCPtrStack = 0;
13752	uint32_t uMask = 0xffff;
13753	bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
13754	uint16_t aIretFrame[3];
13755	if (pDis->fPrefix & (DISPREFIX_OPSIZE \| DISPREFIX_ADDRSIZE))
13756	{
13757	rc = VERR_EM_INTERPRETER;
13758	break;
13759	}
13760	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
13761	&GCPtrStack);
13762	if (RT_SUCCESS(rc))
13763	{
13764	rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame),
13765	PGMACCESSORIGIN_HM));
13766	AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13767	}
13768	if (RT_FAILURE(rc))
13769	{
13770	rc = VERR_EM_INTERPRETER;
13771	break;
13772	}
13773	pMixedCtx->eip = 0;
13774	pMixedCtx->ip = aIretFrame[0];
13775	pMixedCtx->cs.Sel = aIretFrame[1];
13776	pMixedCtx->cs.ValidSel = aIretFrame[1];
13777	pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
13778	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ((UINT32_C(0xffff0000) \| X86_EFL_1) & ~X86_EFL_RF))
13779	\| (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
13780	pMixedCtx->sp += sizeof(aIretFrame);
13781	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13782	\| HM_CHANGED_GUEST_SEGMENT_REGS
13783	\| HM_CHANGED_GUEST_RSP
13784	\| HM_CHANGED_GUEST_RFLAGS);
13785	/* Generate a pending-debug exception when stepping over IRET regardless of how IRET modifies EFLAGS.TF. */
13786	if ( !fDbgStepping
13787	&& fGstStepping)
13788	hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13789	Log4(("IRET %#RX32 to %04x:%04x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
13790	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
13791	break;
13792	}
13793
13794	case OP_INT:
13795	{
13796	uint16_t uVector = pDis->Param1.uValue & 0xff;
13797	hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
13798	/* INT clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13799	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13800	break;
13801	}
13802
13803	case OP_INTO:
13804	{
13805	if (pMixedCtx->eflags.Bits.u1OF)
13806	{
13807	hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
13808	/* INTO clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13809	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13810	}
13811	else
13812	{
13813	pMixedCtx->eflags.Bits.u1RF = 0;
13814	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
13815	}
13816	break;
13817	}
13818
13819	default:
13820	{
13821	pMixedCtx->eflags.Bits.u1RF = 0; /* This is correct most of the time... */
13822	VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
13823	EMCODETYPE_SUPERVISOR);
13824	rc = VBOXSTRICTRC_VAL(rc2);
13825	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
13826	/** @todo We have to set pending-debug exceptions here when the guest is
13827	* single-stepping depending on the instruction that was interpreted. */
13828	Log4(("#GP rc=%Rrc\n", rc));
13829	break;
13830	}
13831	}
13832	}
13833	else
13834	rc = VERR_EM_INTERPRETER;
13835
13836	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_EM_HALT,
13837	("#GP Unexpected rc=%Rrc\n", rc));
13838	return rc;
13839	}
13840
13841
13842	/**
13843	* VM-exit exception handler wrapper for generic exceptions. Simply re-injects
13844	* the exception reported in the VMX transient structure back into the VM.
13845	*
13846	* @remarks Requires uExitIntInfo in the VMX transient structure to be
13847	* up-to-date.
13848	*/
13849	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13850	{
13851	RT_NOREF_PV(pMixedCtx);
13852	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13853	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13854	AssertMsg(pVCpu->hm.s.fUsingDebugLoop \|\| pVCpu->hm.s.vmx.RealMode.fRealOnV86Active,
13855	("uVector=%#04x u32XcptBitmap=%#010RX32\n",
13856	VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo), pVCpu->hm.s.vmx.u32XcptBitmap));
13857	#endif
13858
13859	/* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
13860	hmR0VmxCheckExitDueToEventDelivery(). */
13861	int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13862	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13863	AssertRCReturn(rc, rc);
13864	Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
13865
13866	#ifdef DEBUG_ramshankar
13867	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
13868	uint8_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
13869	Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
13870	#endif
13871
13872	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13873	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13874	return VINF_SUCCESS;
13875	}
13876
13877
13878	/**
13879	* VM-exit exception handler for \#PF (Page-fault exception).
13880	*/
13881	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13882	{
13883	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13884	PVM pVM = pVCpu->CTX_SUFF(pVM);
13885	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13886	rc \|= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13887	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13888	AssertRCReturn(rc, rc);
13889
13890	if (!pVM->hm.s.fNestedPaging)
13891	{ /* likely */ }
13892	else
13893	{
13894	#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF)
13895	Assert(pVCpu->hm.s.fUsingDebugLoop);
13896	#endif
13897	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
13898	if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
13899	{
13900	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13901	0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
13902	}
13903	else
13904	{
13905	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13906	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
13907	Log4(("Pending #DF due to vectoring #PF. NP\n"));
13908	}
13909	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13910	return rc;
13911	}
13912
13913	/* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
13914	of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
13915	if (pVmxTransient->fVectoringPF)
13916	{
13917	Assert(pVCpu->hm.s.Event.fPending);
13918	return VINF_EM_RAW_INJECT_TRPM_EVENT;
13919	}
13920
13921	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13922	AssertRCReturn(rc, rc);
13923
13924	Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
13925	pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntErrorCode, pMixedCtx->cr3));
13926
13927	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
13928	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pMixedCtx),
13929	(RTGCPTR)pVmxTransient->uExitQualification);
13930
13931	Log4(("#PF: rc=%Rrc\n", rc));
13932	if (rc == VINF_SUCCESS)
13933	{
13934	#if 0
13935	/* Successfully synced shadow pages tables or emulated an MMIO instruction. */
13936	/** @todo this isn't quite right, what if guest does lgdt with some MMIO
13937	* memory? We don't update the whole state here... */
13938	HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13939	\| HM_CHANGED_GUEST_RSP
13940	\| HM_CHANGED_GUEST_RFLAGS
13941	\| HM_CHANGED_VMX_GUEST_APIC_STATE);
13942	#else
13943	/*
13944	* This is typically a shadow page table sync or a MMIO instruction. But we may have
13945	* emulated something like LTR or a far jump. Any part of the CPU context may have changed.
13946	*/
13947	/** @todo take advantage of CPUM changed flags instead of brute forcing. */
13948	HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
13949	#endif
13950	TRPMResetTrap(pVCpu);
13951	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
13952	return rc;
13953	}
13954
13955	if (rc == VINF_EM_RAW_GUEST_TRAP)
13956	{
13957	if (!pVmxTransient->fVectoringDoublePF)
13958	{
13959	/* It's a guest page fault and needs to be reflected to the guest. */
13960	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
13961	TRPMResetTrap(pVCpu);
13962	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
13963	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13964	0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
13965	}
13966	else
13967	{
13968	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13969	TRPMResetTrap(pVCpu);
13970	pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
13971	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
13972	Log4(("#PF: Pending #DF due to vectoring #PF\n"));
13973	}
13974
13975	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13976	return VINF_SUCCESS;
13977	}
13978
13979	TRPMResetTrap(pVCpu);
13980	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
13981	return rc;
13982	}
13983
13984	/** @} */
13985

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 70687

Download in other formats: